Skip to content

Ears for coding agents: let AI hear, review and make sound. CPU-only, open source.

85.1 KB
Raw Blame
{
 "n_eval": 219,
 "n_ref": 148,
 "n_vocab": 322,
 "chosen": {
  "pad": "repeatpad",
  "templates": "t5",
  "method": "both",
  "all": {
   "cat_map": 0.7775335919864723,
   "cat_p5": 0.85,
   "cat_p10": 0.85,
   "class_map": 0.6424589753467237,
   "class_top3": 0.8857142857142857,
   "tag_acc": 0.5799086757990868,
   "tag_mrr": 0.712406871471145,
   "pooled_auc": 0.9656601066427873,
   "pooled_auc_strict": 0.9508040386214509,
   "spec_top1": 0.7916666666666666,
   "spec_top3": 0.9583333333333334
  },
  "dev": {
   "cat_map": 0.777586606292167,
   "cat_p5": 0.85,
   "cat_p10": 0.75,
   "class_map": 0.6779892152094823,
   "class_top3": 0.8870967741935484,
   "tag_acc": 0.6008064516129032,
   "tag_mrr": 0.7313790616536111,
   "pooled_auc": 0.9731773544405689,
   "pooled_auc_strict": 0.9595833926453143,
   "spec_top1": 0.9,
   "spec_top3": 1.0
  },
  "test": {
   "cat_map": 0.7932570388183038,
   "cat_p5": 0.825,
   "cat_p10": 0.7125,
   "class_map": 0.6390066424489291,
   "class_top3": 0.8793103448275862,
   "tag_acc": 0.5526315789473685,
   "tag_mrr": 0.6876431706014001,
   "pooled_auc": 0.9558856317750858,
   "pooled_auc_strict": 0.9393954636620294,
   "spec_top1": 0.7619047619047619,
   "spec_top3": 0.9047619047619048
  },
  "detail": {
   "category": {
    "map": 0.7775335919864723,
    "p5": 0.85,
    "p10": 0.85,
    "rows": [
     {
      "cat": "ambience",
      "ap": 0.9464513817222796,
      "p5": 1.0,
      "p10": 1.0
     },
     {
      "cat": "foley",
      "ap": 0.23496220076980903,
      "p5": 0.0,
      "p10": 0.2
     },
     {
      "cat": "garden",
      "ap": 0.7607732443514181,
      "p5": 1.0,
      "p10": 0.9
     },
     {
      "cat": "keyboard",
      "ap": 0.9541322251479479,
      "p5": 1.0,
      "p10": 1.0
     },
     {
      "cat": "magic",
      "ap": 1.0,
      "p5": 1.0,
      "p10": 1.0
     },
     {
      "cat": "motion",
      "ap": 0.671879351245011,
      "p5": 1.0,
      "p10": 1.0
     },
     {
      "cat": "music",
      "ap": 1.0,
      "p5": 1.0,
      "p10": 0.8
     },
     {
      "cat": "toy",
      "ap": 0.6520703326553127,
      "p5": 0.8,
      "p10": 0.9
     }
    ]
   },
   "class": {
    "map": 0.6424589753467237,
    "top3": 0.8857142857142857,
    "n_queries": 70
   },
   "specific": {
    "top1": 0.7916666666666666,
    "top3": 0.9583333333333334,
    "map": 0.7557323637715152,
    "rows": [
     {
      "q": "a deep thock of a mechanical keyboard key",
      "top1": true,
      "top3": true,
      "ap": 0.65227130473939,
      "best": "fs-789628_keychron-linear-individual-keys.ogg"
     },
     {
      "q": "a whoosh",
      "top1": true,
      "top3": true,
      "ap": 0.960530045351474,
      "best": "mk-1489_air-woosh.wav"
     },
     {
      "q": "a glass ting",
      "top1": true,
      "top3": true,
      "ap": 0.6752136752136751,
      "best": "fs-549899_small-wine-glass-tap.ogg"
     },
     {
      "q": "footsteps in grass",
      "top1": true,
      "top3": true,
      "ap": 0.8168168880455408,
      "best": "mk-1921_walking-on-grass.wav"
     },
     {
      "q": "rain on a window",
      "top1": true,
      "top3": true,
      "ap": 0.9761904761904762,
      "best": "fs-346642_rain-on-windows-interior.ogg"
     },
     {
      "q": "a cork pop",
      "top1": false,
      "top3": false,
      "ap": 0.25,
      "best": "fs-181907_mouth-plops-c214.ogg"
     },
     {
      "q": "a pop",
      "top1": true,
      "top3": true,
      "ap": 0.8692956349206349,
      "best": "fs-181907_mouth-plops-c214.ogg"
     },
     {
      "q": "a light switch click",
      "top1": false,
      "top3": true,
      "ap": 0.210921912624699,
      "best": "fs-151778_clip-in.ogg"
     },
     {
      "q": "lego bricks clicking",
      "top1": false,
      "top3": true,
      "ap": 0.43934645601312267,
      "best": "fs-326470_dominos-falling.ogg"
     },
     {
      "q": "dominoes falling",
      "top1": true,
      "top3": true,
      "ap": 1.0,
      "best": "fs-873306_various-domino-falling.ogg"
     },
     {
      "q": "a celesta note",
      "top1": true,
      "top3": true,
      "ap": 1.0,
      "best": "fs-410664_celesta-B5.ogg"
     },
     {
      "q": "a kalimba note",
      "top1": true,
      "top3": true,
      "ap": 0.8761904761904761,
      "best": "fs-681791_kalimba-D-high.ogg"
     },
     {
      "q": "birds singing in the morning",
      "top1": true,
      "top3": true,
      "ap": 1.0,
      "best": "mk-2472_morning-birds.wav"
     },
     {
      "q": "an object dropping into water",
      "top1": true,
      "top3": true,
      "ap": 0.8441043083900226,
      "best": "fs-272392_object-drops-in-water.ogg"
     },
     {
      "q": "a cymbal swell",
      "top1": true,
      "top3": true,
      "ap": 0.8928571428571428,
      "best": "fs-346023_violin-bow-on-cymbal.ogg"
     },
     {
      "q": "typing on a keyboard",
      "top1": true,
      "top3": true,
      "ap": 0.9166666666666666,
      "best": "fs-789627_keychron-linear-typing-66wpm.ogg"
     },
     {
      "q": "a creaking wooden sound",
      "top1": true,
      "top3": true,
      "ap": 0.649133022774327,
      "best": "fs-123712_creaking-wood-screws.ogg"
     },
     {
      "q": "digging in soil",
      "top1": true,
      "top3": true,
      "ap": 0.6083333333333334,
      "best": "fs-696531_trowel-soft-soil.ogg"
     },
     {
      "q": "an apple falling on grass",
      "top1": false,
      "top3": true,
      "ap": 0.6472222222222223,
      "best": "fs-559082_high-grass-movement.ogg"
     },
     {
      "q": "a finger squeaking on glass",
      "top1": true,
      "top3": true,
      "ap": 0.7784090909090909,
      "best": "fs-119916_glass-squeaking.ogg"
     },
     {
      "q": "a computer hum",
      "top1": true,
      "top3": true,
      "ap": 0.7333333333333334,
      "best": "fs-381258_computer-hum-gentle.ogg"
     },
     {
      "q": "a camera shutter",
      "top1": true,
      "top3": true,
      "ap": 1.0,
      "best": "fs-270435_shutter-click-canon.ogg"
     },
     {
      "q": "a suction cup pop",
      "top1": false,
      "top3": true,
      "ap": 0.34074074074074073,
      "best": "fs-181907_mouth-plops-c214.ogg"
     },
     {
      "q": "a deep cinematic hit",
      "top1": true,
      "top3": true,
      "ap": 1.0,
      "best": "mk-563_drum-deep-impact.wav"
     }
    ]
   },
   "tagging": {
    "acc": 0.5799086757990868,
    "mrr": 0.712406871471145
   },
   "pooled_auc": 0.9656601066427873,
   "pooled_auc_strict": 0.9508040386214509
  }
 },
 "variants": [
  {
   "pad": "repeatpad",
   "templates": "t1",
   "method": "raw",
   "all": {
    "cat_map": 0.6980522113170913,
    "cat_p5": 0.8500000000000001,
    "cat_p10": 0.7875,
    "class_map": 0.5970493925255879,
    "class_top3": 0.8428571428571429,
    "tag_acc": 0.5525114155251142,
    "tag_mrr": 0.6852196436140946,
    "pooled_auc": 0.9435436000759796,
    "pooled_auc_strict": 0.9273817348182524,
    "spec_top1": 0.7083333333333334,
    "spec_top3": 0.9583333333333334
   },
   "dev": {
    "cat_map": 0.6856773137576235,
    "cat_p5": 0.75,
    "cat_p10": 0.6625,
    "class_map": 0.6238167679338643,
    "class_top3": 0.8225806451612904,
    "tag_acc": 0.5806451612903226,
    "tag_mrr": 0.7081935069387318,
    "pooled_auc": 0.9537306144352026,
    "pooled_auc_strict": 0.9380697508896797,
    "spec_top1": 0.7,
    "spec_top3": 0.9
   },
   "test": {
    "cat_map": 0.697124334270687,
    "cat_p5": 0.75,
    "cat_p10": 0.6499999999999999,
    "class_map": 0.5876740068290082,
    "class_top3": 0.7758620689655172,
    "tag_acc": 0.5157894736842106,
    "tag_mrr": 0.6552327062219364,
    "pooled_auc": 0.9308220171927477,
    "pooled_auc_strict": 0.9139352508335569,
    "spec_top1": 0.5714285714285714,
    "spec_top3": 0.8571428571428571
   }
  },
  {
   "pad": "repeatpad",
   "templates": "t1",
   "method": "sound",
   "all": {
    "cat_map": 0.7439785814201825,
    "cat_p5": 0.8500000000000001,
    "cat_p10": 0.8,
    "class_map": 0.6094180530398979,
    "class_top3": 0.8857142857142857,
    "tag_acc": 0.5525114155251142,
    "tag_mrr": 0.6852196436140946,
    "pooled_auc": 0.9564496160316943,
    "pooled_auc_strict": 0.9402166676955068,
    "spec_top1": 0.875,
    "spec_top3": 0.9166666666666666
   },
   "dev": {
    "cat_map": 0.7456934125575034,
    "cat_p5": 0.825,
    "cat_p10": 0.7250000000000001,
    "class_map": 0.6627912840247255,
    "class_top3": 0.8709677419354839,
    "tag_acc": 0.5806451612903226,
    "tag_mrr": 0.7081935069387318,
    "pooled_auc": 0.965224040783472,
    "pooled_auc_strict": 0.9494149466192171,
    "spec_top1": 0.9,
    "spec_top3": 0.95
   },
   "test": {
    "cat_map": 0.759221115646978,
    "cat_p5": 0.8,
    "cat_p10": 0.7,
    "class_map": 0.6008358348396999,
    "class_top3": 0.8103448275862069,
    "tag_acc": 0.5157894736842106,
    "tag_mrr": 0.6552327062219364,
    "pooled_auc": 0.9450525727188112,
    "pooled_auc_strict": 0.9283196962148214,
    "spec_top1": 0.8571428571428571,
    "spec_top3": 0.9047619047619048
   }
  },
  {
   "pad": "repeatpad",
   "templates": "t1",
   "method": "center",
   "all": {
    "cat_map": 0.6980522113170913,
    "cat_p5": 0.8500000000000001,
    "cat_p10": 0.7875,
    "class_map": 0.5970493925255879,
    "class_top3": 0.8428571428571429,
    "tag_acc": 0.5684931506849316,
    "tag_mrr": 0.7059231884033613,
    "pooled_auc": 0.9489340809182677,
    "pooled_auc_strict": 0.9332191925904139,
    "spec_top1": 0.7083333333333334,
    "spec_top3": 0.9583333333333334
   },
   "dev": {
    "cat_map": 0.6856773137576235,
    "cat_p5": 0.75,
    "cat_p10": 0.6625,
    "class_map": 0.6238167679338643,
    "class_top3": 0.8225806451612904,
    "tag_acc": 0.6008064516129032,
    "tag_mrr": 0.730970754156238,
    "pooled_auc": 0.9599833646364369,
    "pooled_auc_strict": 0.9452052194543298,
    "spec_top1": 0.7,
    "spec_top3": 0.9
   },
   "test": {
    "cat_map": 0.697124334270687,
    "cat_p5": 0.75,
    "cat_p10": 0.6499999999999999,
    "class_map": 0.5876740068290082,
    "class_top3": 0.7758620689655172,
    "tag_acc": 0.5263157894736842,
    "tag_mrr": 0.6732295236311856,
    "pooled_auc": 0.9349122844270794,
    "pooled_auc_strict": 0.9179002350566563,
    "spec_top1": 0.5714285714285714,
    "spec_top3": 0.8571428571428571
   }
  },
  {
   "pad": "repeatpad",
   "templates": "t1",
   "method": "both",
   "all": {
    "cat_map": 0.7610068662109295,
    "cat_p5": 0.85,
    "cat_p10": 0.825,
    "class_map": 0.6130333886405536,
    "class_top3": 0.9,
    "tag_acc": 0.5684931506849316,
    "tag_mrr": 0.7059231884033613,
    "pooled_auc": 0.9602390982850321,
    "pooled_auc_strict": 0.9444559432467534,
    "spec_top1": 0.8333333333333334,
    "spec_top3": 0.9166666666666666
   },
   "dev": {
    "cat_map": 0.7534607877045285,
    "cat_p5": 0.825,
    "cat_p10": 0.7374999999999999,
    "class_map": 0.6746554502564531,
    "class_top3": 0.8709677419354839,
    "tag_acc": 0.6008064516129032,
    "tag_mrr": 0.730970754156238,
    "pooled_auc": 0.9693265360880065,
    "pooled_auc_strict": 0.9548673784104389,
    "spec_top1": 0.9,
    "spec_top3": 0.95
   },
   "test": {
    "cat_map": 0.7673374351307555,
    "cat_p5": 0.8,
    "cat_p10": 0.725,
    "class_map": 0.6112840868653101,
    "class_top3": 0.8275862068965517,
    "tag_acc": 0.5263157894736842,
    "tag_mrr": 0.6732295236311856,
    "pooled_auc": 0.9484651101885965,
    "pooled_auc_strict": 0.9309656804506956,
    "spec_top1": 0.8571428571428571,
    "spec_top3": 0.9047619047619048
   }
  },
  {
   "pad": "repeatpad",
   "templates": "t1",
   "method": "term_sd",
   "all": {
    "cat_map": 0.6980522113170913,
    "cat_p5": 0.8500000000000001,
    "cat_p10": 0.7875,
    "class_map": 0.5970493925255879,
    "class_top3": 0.8428571428571429,
    "tag_acc": 0.5365296803652968,
    "tag_mrr": 0.6865289901992316,
    "pooled_auc": 0.946785452621296,
    "pooled_auc_strict": 0.9314520082355633,
    "spec_top1": 0.7083333333333334,
    "spec_top3": 0.9583333333333334
   },
   "dev": {
    "cat_map": 0.6856773137576235,
    "cat_p5": 0.75,
    "cat_p10": 0.6625,
    "class_map": 0.6238167679338643,
    "class_top3": 0.8225806451612904,
    "tag_acc": 0.5806451612903226,
    "tag_mrr": 0.7158915345407281,
    "pooled_auc": 0.9578196941239603,
    "pooled_auc_strict": 0.9434927639383155,
    "spec_top1": 0.7,
    "spec_top3": 0.9
   },
   "test": {
    "cat_map": 0.697124334270687,
    "cat_p5": 0.75,
    "cat_p10": 0.6499999999999999,
    "class_map": 0.5876740068290082,
    "class_top3": 0.7758620689655172,
    "tag_acc": 0.4789473684210526,
    "tag_mrr": 0.648203142848226,
    "pooled_auc": 0.9328622846040324,
    "pooled_auc_strict": 0.9161987889472272,
    "spec_top1": 0.5714285714285714,
    "spec_top3": 0.8571428571428571
   }
  },
  {
   "pad": "repeatpad",
   "templates": "t1",
   "method": "both_sd",
   "all": {
    "cat_map": 0.7586182144969197,
    "cat_p5": 0.825,
    "cat_p10": 0.8125,
    "class_map": 0.595603722072477,
    "class_top3": 0.8714285714285714,
    "tag_acc": 0.5365296803652968,
    "tag_mrr": 0.6865289901992316,
    "pooled_auc": 0.9574583808205797,
    "pooled_auc_strict": 0.9412123731954445,
    "spec_top1": 0.8333333333333334,
    "spec_top3": 0.9583333333333334
   },
   "dev": {
    "cat_map": 0.7479252739679811,
    "cat_p5": 0.8,
    "cat_p10": 0.725,
    "class_map": 0.6719452435180059,
    "class_top3": 0.8548387096774194,
    "tag_acc": 0.5806451612903226,
    "tag_mrr": 0.7158915345407281,
    "pooled_auc": 0.9660998121813791,
    "pooled_auc_strict": 0.9513580071174377,
    "spec_top1": 0.85,
    "spec_top3": 0.95
   },
   "test": {
    "cat_map": 0.7880552005178514,
    "cat_p5": 0.8,
    "cat_p10": 0.7,
    "class_map": 0.6064812958523895,
    "class_top3": 0.8448275862068966,
    "tag_acc": 0.4789473684210526,
    "tag_mrr": 0.648203142848226,
    "pooled_auc": 0.9462222316597124,
    "pooled_auc_strict": 0.9280394486388431,
    "spec_top1": 0.7619047619047619,
    "spec_top3": 0.9047619047619048
   }
  },
  {
   "pad": "repeatpad",
   "templates": "t3",
   "method": "raw",
   "all": {
    "cat_map": 0.718118751880866,
    "cat_p5": 0.825,
    "cat_p10": 0.7875,
    "class_map": 0.6273662801196275,
    "class_top3": 0.8857142857142857,
    "tag_acc": 0.589041095890411,
    "tag_mrr": 0.7097763961710694,
    "pooled_auc": 0.9475559331922284,
    "pooled_auc_strict": 0.9320818702620396,
    "spec_top1": 0.875,
    "spec_top3": 0.9583333333333334
   },
   "dev": {
    "cat_map": 0.7153805164806042,
    "cat_p5": 0.825,
    "cat_p10": 0.675,
    "class_map": 0.6480120430856741,
    "class_top3": 0.8064516129032258,
    "tag_acc": 0.625,
    "tag_mrr": 0.7339496162100214,
    "pooled_auc": 0.9563938824792058,
    "pooled_auc_strict": 0.9414861209964412,
    "spec_top1": 0.85,
    "spec_top3": 0.9
   },
   "test": {
    "cat_map": 0.7204871594793816,
    "cat_p5": 0.75,
    "cat_p10": 0.675,
    "class_map": 0.6354754574664218,
    "class_top3": 0.8620689655172413,
    "tag_acc": 0.5421052631578948,
    "tag_mrr": 0.6782239826465425,
    "pooled_auc": 0.9363978043679064,
    "pooled_auc_strict": 0.9201422156644822,
    "spec_top1": 0.7619047619047619,
    "spec_top3": 0.9523809523809523
   }
  },
  {
   "pad": "repeatpad",
   "templates": "t3",
   "method": "sound",
   "all": {
    "cat_map": 0.7665978592943665,
    "cat_p5": 0.85,
    "cat_p10": 0.8375,
    "class_map": 0.6414165364178294,
    "class_top3": 0.9142857142857143,
    "tag_acc": 0.589041095890411,
    "tag_mrr": 0.7097763961710694,
    "pooled_auc": 0.9601956813741452,
    "pooled_auc_strict": 0.9447617206081534,
    "spec_top1": 0.8333333333333334,
    "spec_top3": 0.9583333333333334
   },
   "dev": {
    "cat_map": 0.7634370504834116,
    "cat_p5": 0.875,
    "cat_p10": 0.7375,
    "class_map": 0.6690742346670663,
    "class_top3": 0.8870967741935484,
    "tag_acc": 0.625,
    "tag_mrr": 0.7339496162100214,
    "pooled_auc": 0.9687255165012074,
    "pooled_auc_strict": 0.9539074733096086,
    "spec_top1": 0.85,
    "spec_top3": 1.0
   },
   "test": {
    "cat_map": 0.7911520237903075,
    "cat_p5": 0.8,
    "cat_p10": 0.7125,
    "class_map": 0.638673491556023,
    "class_top3": 0.8620689655172413,
    "tag_acc": 0.5421052631578948,
    "tag_mrr": 0.6782239826465425,
    "pooled_auc": 0.9492304316590046,
    "pooled_auc_strict": 0.9329641410850931,
    "spec_top1": 0.8095238095238095,
    "spec_top3": 0.9047619047619048
   }
  },
  {
   "pad": "repeatpad",
   "templates": "t3",
   "method": "center",
   "all": {
    "cat_map": 0.718118751880866,
    "cat_p5": 0.825,
    "cat_p10": 0.7875,
    "class_map": 0.6273662801196275,
    "class_top3": 0.8857142857142857,
    "tag_acc": 0.5707762557077626,
    "tag_mrr": 0.7029040570512325,
    "pooled_auc": 0.9517398987843265,
    "pooled_auc_strict": 0.9370941376685028,
    "spec_top1": 0.875,
    "spec_top3": 0.9583333333333334
   },
   "dev": {
    "cat_map": 0.7153805164806042,
    "cat_p5": 0.825,
    "cat_p10": 0.675,
    "class_map": 0.6480120430856741,
    "class_top3": 0.8064516129032258,
    "tag_acc": 0.6088709677419355,
    "tag_mrr": 0.7304551820728292,
    "pooled_auc": 0.9609621679635095,
    "pooled_auc_strict": 0.9471051008303677,
    "spec_top1": 0.85,
    "spec_top3": 0.9
   },
   "test": {
    "cat_map": 0.7204871594793816,
    "cat_p5": 0.75,
    "cat_p10": 0.675,
    "class_map": 0.6354754574664218,
    "class_top3": 0.8620689655172413,
    "tag_acc": 0.5210526315789473,
    "tag_mrr": 0.6669425886019905,
    "pooled_auc": 0.9400527673670465,
    "pooled_auc_strict": 0.9243419371734437,
    "spec_top1": 0.7619047619047619,
    "spec_top3": 0.9523809523809523
   }
  },
  {
   "pad": "repeatpad",
   "templates": "t3",
   "method": "both",
   "all": {
    "cat_map": 0.7695786878859658,
    "cat_p5": 0.85,
    "cat_p10": 0.825,
    "class_map": 0.6427246569683582,
    "class_top3": 0.9,
    "tag_acc": 0.5707762557077626,
    "tag_mrr": 0.7029040570512325,
    "pooled_auc": 0.9637309372625638,
    "pooled_auc_strict": 0.9486622353913768,
    "spec_top1": 0.8333333333333334,
    "spec_top3": 1.0
   },
   "dev": {
    "cat_map": 0.7662596317937798,
    "cat_p5": 0.825,
    "cat_p10": 0.7625,
    "class_map": 0.6750202686222422,
    "class_top3": 0.8709677419354839,
    "tag_acc": 0.6088709677419355,
    "tag_mrr": 0.7304551820728292,
    "pooled_auc": 0.9718663804668635,
    "pooled_auc_strict": 0.9581542111506525,
    "spec_top1": 0.9,
    "spec_top3": 0.95
   },
   "test": {
    "cat_map": 0.7937494731754101,
    "cat_p5": 0.8,
    "cat_p10": 0.7250000000000001,
    "class_map": 0.6407116440892073,
    "class_top3": 0.8620689655172413,
    "tag_acc": 0.5210526315789473,
    "tag_mrr": 0.6669425886019905,
    "pooled_auc": 0.9530924295992723,
    "pooled_auc_strict": 0.9362688268884375,
    "spec_top1": 0.8095238095238095,
    "spec_top3": 0.9047619047619048
   }
  },
  {
   "pad": "repeatpad",
   "templates": "t3",
   "method": "term_sd",
   "all": {
    "cat_map": 0.718118751880866,
    "cat_p5": 0.825,
    "cat_p10": 0.7875,
    "class_map": 0.6273662801196275,
    "class_top3": 0.8857142857142857,
    "tag_acc": 0.5593607305936074,
    "tag_mrr": 0.6987727253599382,
    "pooled_auc": 0.9498845042331489,
    "pooled_auc_strict": 0.9355914257667582,
    "spec_top1": 0.875,
    "spec_top3": 0.9583333333333334
   },
   "dev": {
    "cat_map": 0.7153805164806042,
    "cat_p5": 0.825,
    "cat_p10": 0.675,
    "class_map": 0.6480120430856741,
    "class_top3": 0.8064516129032258,
    "tag_acc": 0.5846774193548387,
    "tag_mrr": 0.7193196458691004,
    "pooled_auc": 0.9590576871478401,
    "pooled_auc_strict": 0.9455406880189798,
    "spec_top1": 0.85,
    "spec_top3": 0.9
   },
   "test": {
    "cat_map": 0.7204871594793816,
    "cat_p5": 0.75,
    "cat_p10": 0.675,
    "class_map": 0.6354754574664218,
    "class_top3": 0.8620689655172413,
    "tag_acc": 0.5263157894736842,
    "tag_mrr": 0.6719535870111372,
    "pooled_auc": 0.9382903160733435,
    "pooled_auc_strict": 0.9229582646686851,
    "spec_top1": 0.7619047619047619,
    "spec_top3": 0.9523809523809523
   }
  },
  {
   "pad": "repeatpad",
   "templates": "t3",
   "method": "both_sd",
   "all": {
    "cat_map": 0.7641445315624935,
    "cat_p5": 0.85,
    "cat_p10": 0.825,
    "class_map": 0.6208798291934929,
    "class_top3": 0.8857142857142857,
    "tag_acc": 0.5593607305936074,
    "tag_mrr": 0.6987727253599382,
    "pooled_auc": 0.9621942160534028,
    "pooled_auc_strict": 0.9466517605876735,
    "spec_top1": 0.875,
    "spec_top3": 1.0
   },
   "dev": {
    "cat_map": 0.7522572784268877,
    "cat_p5": 0.825,
    "cat_p10": 0.7374999999999999,
    "class_map": 0.6733560017280501,
    "class_top3": 0.8870967741935484,
    "tag_acc": 0.5846774193548387,
    "tag_mrr": 0.7193196458691004,
    "pooled_auc": 0.9696361685001341,
    "pooled_auc_strict": 0.9554733096085409,
    "spec_top1": 0.9,
    "spec_top3": 0.95
   },
   "test": {
    "cat_map": 0.7919940719791706,
    "cat_p5": 0.825,
    "cat_p10": 0.7000000000000001,
    "class_map": 0.6408222726285296,
    "class_top3": 0.8448275862068966,
    "tag_acc": 0.5263157894736842,
    "tag_mrr": 0.6719535870111372,
    "pooled_auc": 0.9524810571876515,
    "pooled_auc_strict": 0.9351709909426539,
    "spec_top1": 0.9047619047619048,
    "spec_top3": 0.9047619047619048
   }
  },
  {
   "pad": "repeatpad",
   "templates": "t5",
   "method": "raw",
   "all": {
    "cat_map": 0.7277025148739027,
    "cat_p5": 0.8500000000000001,
    "cat_p10": 0.825,
    "class_map": 0.627087687258443,
    "class_top3": 0.8857142857142857,
    "tag_acc": 0.593607305936073,
    "tag_mrr": 0.7149215845506086,
    "pooled_auc": 0.9509806116357321,
    "pooled_auc_strict": 0.9355483052580997,
    "spec_top1": 0.8333333333333334,
    "spec_top3": 0.9583333333333334
   },
   "dev": {
    "cat_map": 0.7229405850544317,
    "cat_p5": 0.85,
    "cat_p10": 0.6875,
    "class_map": 0.6431871490679331,
    "class_top3": 0.8064516129032258,
    "tag_acc": 0.6209677419354839,
    "tag_mrr": 0.7338462142037805,
    "pooled_auc": 0.9592218942849476,
    "pooled_auc_strict": 0.9444052194543298,
    "spec_top1": 0.85,
    "spec_top3": 0.9
   },
   "test": {
    "cat_map": 0.7199503932575724,
    "cat_p5": 0.75,
    "cat_p10": 0.675,
    "class_map": 0.647772223495713,
    "class_top3": 0.8448275862068966,
    "tag_acc": 0.5578947368421052,
    "tag_mrr": 0.6902199626875211,
    "pooled_auc": 0.9405800871316282,
    "pooled_auc_strict": 0.9243786647759935,
    "spec_top1": 0.7142857142857143,
    "spec_top3": 0.9523809523809523
   }
  },
  {
   "pad": "repeatpad",
   "templates": "t5",
   "method": "sound",
   "all": {
    "cat_map": 0.7702130140324235,
    "cat_p5": 0.85,
    "cat_p10": 0.8625,
    "class_map": 0.649617613261983,
    "class_top3": 0.9,
    "tag_acc": 0.593607305936073,
    "tag_mrr": 0.7149215845506086,
    "pooled_auc": 0.9633435770107457,
    "pooled_auc_strict": 0.9481022739789366,
    "spec_top1": 0.8333333333333334,
    "spec_top3": 0.9583333333333334
   },
   "dev": {
    "cat_map": 0.7706841705015869,
    "cat_p5": 0.8999999999999999,
    "cat_p10": 0.7375,
    "class_map": 0.6757453275548259,
    "class_top3": 0.9032258064516129,
    "tag_acc": 0.6209677419354839,
    "tag_mrr": 0.7338462142037805,
    "pooled_auc": 0.9712261872819963,
    "pooled_auc_strict": 0.9565115065243179,
    "spec_top1": 0.85,
    "spec_top3": 1.0
   },
   "test": {
    "cat_map": 0.7699222515537358,
    "cat_p5": 0.8,
    "cat_p10": 0.7125,
    "class_map": 0.6454645266686625,
    "class_top3": 0.8620689655172413,
    "tag_acc": 0.5578947368421052,
    "tag_mrr": 0.6902199626875211,
    "pooled_auc": 0.9532366462462972,
    "pooled_auc_strict": 0.9373035871689726,
    "spec_top1": 0.8095238095238095,
    "spec_top3": 0.9047619047619048
   }
  },
  {
   "pad": "repeatpad",
   "templates": "t5",
   "method": "center",
   "all": {
    "cat_map": 0.7277025148739027,
    "cat_p5": 0.8500000000000001,
    "cat_p10": 0.825,
    "class_map": 0.627087687258443,
    "class_top3": 0.8857142857142857,
    "tag_acc": 0.5799086757990868,
    "tag_mrr": 0.712406871471145,
    "pooled_auc": 0.9537319887658743,
    "pooled_auc_strict": 0.9391610473835383,
    "spec_top1": 0.8333333333333334,
    "spec_top3": 0.9583333333333334
   },
   "dev": {
    "cat_map": 0.7229405850544317,
    "cat_p5": 0.85,
    "cat_p10": 0.6875,
    "class_map": 0.6431871490679331,
    "class_top3": 0.8064516129032258,
    "tag_acc": 0.6008064516129032,
    "tag_mrr": 0.7313790616536111,
    "pooled_auc": 0.9622554333243896,
    "pooled_auc_strict": 0.9485385527876631,
    "spec_top1": 0.85,
    "spec_top3": 0.9
   },
   "test": {
    "cat_map": 0.7199503932575724,
    "cat_p5": 0.75,
    "cat_p10": 0.675,
    "class_map": 0.647772223495713,
    "class_top3": 0.8448275862068966,
    "tag_acc": 0.5526315789473685,
    "tag_mrr": 0.6876431706014001,
    "pooled_auc": 0.9428848991899095,
    "pooled_auc_strict": 0.9272050933200475,
    "spec_top1": 0.7142857142857143,
    "spec_top3": 0.9523809523809523
   }
  },
  {
   "pad": "repeatpad",
   "templates": "t5",
   "method": "both",
   "all": {
    "cat_map": 0.7775335919864723,
    "cat_p5": 0.85,
    "cat_p10": 0.85,
    "class_map": 0.6424589753467237,
    "class_top3": 0.8857142857142857,
    "tag_acc": 0.5799086757990868,
    "tag_mrr": 0.712406871471145,
    "pooled_auc": 0.9656601066427873,
    "pooled_auc_strict": 0.9508040386214509,
    "spec_top1": 0.7916666666666666,
    "spec_top3": 0.9583333333333334
   },
   "dev": {
    "cat_map": 0.777586606292167,
    "cat_p5": 0.85,
    "cat_p10": 0.75,
    "class_map": 0.6779892152094823,
    "class_top3": 0.8870967741935484,
    "tag_acc": 0.6008064516129032,
    "tag_mrr": 0.7313790616536111,
    "pooled_auc": 0.9731773544405689,
    "pooled_auc_strict": 0.9595833926453143,
    "spec_top1": 0.9,
    "spec_top3": 1.0
   },
   "test": {
    "cat_map": 0.7932570388183038,
    "cat_p5": 0.825,
    "cat_p10": 0.7125,
    "class_map": 0.6390066424489291,
    "class_top3": 0.8793103448275862,
    "tag_acc": 0.5526315789473685,
    "tag_mrr": 0.6876431706014001,
    "pooled_auc": 0.9558856317750858,
    "pooled_auc_strict": 0.9393954636620294,
    "spec_top1": 0.7619047619047619,
    "spec_top3": 0.9047619047619048
   }
  },
  {
   "pad": "repeatpad",
   "templates": "t5",
   "method": "term_sd",
   "all": {
    "cat_map": 0.7277025148739027,
    "cat_p5": 0.8500000000000001,
    "cat_p10": 0.825,
    "class_map": 0.627087687258443,
    "class_top3": 0.8857142857142857,
    "tag_acc": 0.5776255707762558,
    "tag_mrr": 0.7099758184972561,
    "pooled_auc": 0.9517821285140562,
    "pooled_auc_strict": 0.9377659098033947,
    "spec_top1": 0.8333333333333334,
    "spec_top3": 0.9583333333333334
   },
   "dev": {
    "cat_map": 0.7229405850544317,
    "cat_p5": 0.85,
    "cat_p10": 0.6875,
    "class_map": 0.6431871490679331,
    "class_top3": 0.8064516129032258,
    "tag_acc": 0.592741935483871,
    "tag_mrr": 0.7242494221728093,
    "pooled_auc": 0.9603209015293802,
    "pooled_auc_strict": 0.9471311981020166,
    "spec_top1": 0.85,
    "spec_top3": 0.9
   },
   "test": {
    "cat_map": 0.7199503932575724,
    "cat_p5": 0.75,
    "cat_p10": 0.675,
    "class_map": 0.647772223495713,
    "class_top3": 0.8448275862068966,
    "tag_acc": 0.5578947368421052,
    "tag_mrr": 0.6913450094891653,
    "pooled_auc": 0.9409676140727135,
    "pooled_auc_strict": 0.9258701248099745,
    "spec_top1": 0.7142857142857143,
    "spec_top3": 0.9523809523809523
   }
  },
  {
   "pad": "repeatpad",
   "templates": "t5",
   "method": "both_sd",
   "all": {
    "cat_map": 0.7684321393745663,
    "cat_p5": 0.85,
    "cat_p10": 0.8375,
    "class_map": 0.6294999047685299,
    "class_top3": 0.9,
    "tag_acc": 0.5776255707762558,
    "tag_mrr": 0.7099758184972561,
    "pooled_auc": 0.9640507977857375,
    "pooled_auc_strict": 0.9488335070257682,
    "spec_top1": 0.7916666666666666,
    "spec_top3": 1.0
   },
   "dev": {
    "cat_map": 0.7597971337607392,
    "cat_p5": 0.825,
    "cat_p10": 0.7374999999999999,
    "class_map": 0.6736543996724086,
    "class_top3": 0.9032258064516129,
    "tag_acc": 0.592741935483871,
    "tag_mrr": 0.7242494221728093,
    "pooled_auc": 0.9708870405151596,
    "pooled_auc_strict": 0.9569504151838671,
    "spec_top1": 0.9,
    "spec_top3": 1.0
   },
   "test": {
    "cat_map": 0.795543349737912,
    "cat_p5": 0.825,
    "cat_p10": 0.7124999999999999,
    "class_map": 0.6447098756668317,
    "class_top3": 0.8448275862068966,
    "tag_acc": 0.5578947368421052,
    "tag_mrr": 0.6913450094891653,
    "pooled_auc": 0.9551512770693762,
    "pooled_auc_strict": 0.9382640938182655,
    "spec_top1": 0.7619047619047619,
    "spec_top3": 0.9047619047619048
   }
  },
  {
   "pad": "zero",
   "templates": "t1",
   "method": "raw",
   "all": {
    "cat_map": 0.6966951514439714,
    "cat_p5": 0.9,
    "cat_p10": 0.75,
    "class_map": 0.6103205175324865,
    "class_top3": 0.8428571428571429,
    "tag_acc": 0.5662100456621004,
    "tag_mrr": 0.688933104494091,
    "pooled_auc": 0.9470330646911972,
    "pooled_auc_strict": 0.9317642914982698,
    "spec_top1": 0.7083333333333334,
    "spec_top3": 0.9166666666666666
   },
   "dev": {
    "cat_map": 0.6999603235345861,
    "cat_p5": 0.775,
    "cat_p10": 0.6375000000000001,
    "class_map": 0.6421799826780327,
    "class_top3": 0.8225806451612904,
    "tag_acc": 0.592741935483871,
    "tag_mrr": 0.704020998923095,
    "pooled_auc": 0.9547206868795278,
    "pooled_auc_strict": 0.9396664294187426,
    "spec_top1": 0.6,
    "spec_top3": 0.9
   },
   "test": {
    "cat_map": 0.6923689595029523,
    "cat_p5": 0.7,
    "cat_p10": 0.6375,
    "class_map": 0.6342054217314276,
    "class_top3": 0.7931034482758621,
    "tag_acc": 0.531578947368421,
    "tag_mrr": 0.6692394317657068,
    "pooled_auc": 0.9374489048382473,
    "pooled_auc_strict": 0.9218795909503187,
    "spec_top1": 0.7619047619047619,
    "spec_top3": 0.9047619047619048
   }
  },
  {
   "pad": "zero",
   "templates": "t1",
   "method": "sound",
   "all": {
    "cat_map": 0.7296165061572158,
    "cat_p5": 0.8500000000000001,
    "cat_p10": 0.825,
    "class_map": 0.5990407658743546,
    "class_top3": 0.8571428571428571,
    "tag_acc": 0.5662100456621004,
    "tag_mrr": 0.688933104494091,
    "pooled_auc": 0.9525578666015413,
    "pooled_auc_strict": 0.9376875363876573,
    "spec_top1": 0.8333333333333334,
    "spec_top3": 0.9583333333333334
   },
   "dev": {
    "cat_map": 0.7343754496700743,
    "cat_p5": 0.825,
    "cat_p10": 0.6875,
    "class_map": 0.6395514066218183,
    "class_top3": 0.8387096774193549,
    "tag_acc": 0.592741935483871,
    "tag_mrr": 0.704020998923095,
    "pooled_auc": 0.9570480279044808,
    "pooled_auc_strict": 0.9420365361803085,
    "spec_top1": 0.85,
    "spec_top3": 0.9
   },
   "test": {
    "cat_map": 0.7421549661065795,
    "cat_p5": 0.775,
    "cat_p10": 0.675,
    "class_map": 0.6013351535309124,
    "class_top3": 0.7758620689655172,
    "tag_acc": 0.531578947368421,
    "tag_mrr": 0.6692394317657068,
    "pooled_auc": 0.9466831940713687,
    "pooled_auc_strict": 0.9319892627652372,
    "spec_top1": 0.7142857142857143,
    "spec_top3": 0.8571428571428571
   }
  },
  {
   "pad": "zero",
   "templates": "t1",
   "method": "center",
   "all": {
    "cat_map": 0.6966951514439714,
    "cat_p5": 0.9,
    "cat_p10": 0.75,
    "class_map": 0.6103205175324865,
    "class_top3": 0.8428571428571429,
    "tag_acc": 0.5684931506849316,
    "tag_mrr": 0.700051267624108,
    "pooled_auc": 0.9540455741886464,
    "pooled_auc_strict": 0.9391905508894626,
    "spec_top1": 0.7083333333333334,
    "spec_top3": 0.9166666666666666
   },
   "dev": {
    "cat_map": 0.6999603235345861,
    "cat_p5": 0.775,
    "cat_p10": 0.6375000000000001,
    "class_map": 0.6421799826780327,
    "class_top3": 0.8225806451612904,
    "tag_acc": 0.5806451612903226,
    "tag_mrr": 0.711020057978075,
    "pooled_auc": 0.9625156962704589,
    "pooled_auc_strict": 0.9483321470937129,
    "spec_top1": 0.6,
    "spec_top3": 0.9
   },
   "test": {
    "cat_map": 0.6923689595029523,
    "cat_p5": 0.7,
    "cat_p10": 0.6375,
    "class_map": 0.6342054217314276,
    "class_top3": 0.7931034482758621,
    "tag_acc": 0.5526315789473685,
    "tag_mrr": 0.6857341096884034,
    "pooled_auc": 0.9431299790133811,
    "pooled_auc_strict": 0.9273655769746675,
    "spec_top1": 0.7619047619047619,
    "spec_top3": 0.9047619047619048
   }
  },
  {
   "pad": "zero",
   "templates": "t1",
   "method": "both",
   "all": {
    "cat_map": 0.7425693181695112,
    "cat_p5": 0.85,
    "cat_p10": 0.8125,
    "class_map": 0.60919922426989,
    "class_top3": 0.8571428571428571,
    "tag_acc": 0.5684931506849316,
    "tag_mrr": 0.700051267624108,
    "pooled_auc": 0.9584685023879301,
    "pooled_auc_strict": 0.9439301756411793,
    "spec_top1": 0.8333333333333334,
    "spec_top3": 0.9583333333333334
   },
   "dev": {
    "cat_map": 0.743093620970253,
    "cat_p5": 0.825,
    "cat_p10": 0.6875,
    "class_map": 0.6592606009695009,
    "class_top3": 0.8387096774193549,
    "tag_acc": 0.5806451612903226,
    "tag_mrr": 0.711020057978075,
    "pooled_auc": 0.9637552991682318,
    "pooled_auc_strict": 0.9497508896797153,
    "spec_top1": 0.8,
    "spec_top3": 0.9
   },
   "test": {
    "cat_map": 0.747603229487134,
    "cat_p5": 0.775,
    "cat_p10": 0.7125,
    "class_map": 0.6141318587911977,
    "class_top3": 0.7931034482758621,
    "tag_acc": 0.5526315789473685,
    "tag_mrr": 0.6857341096884034,
    "pooled_auc": 0.9515237417761121,
    "pooled_auc_strict": 0.9362305024336028,
    "spec_top1": 0.8095238095238095,
    "spec_top3": 0.8571428571428571
   }
  },
  {
   "pad": "zero",
   "templates": "t1",
   "method": "term_sd",
   "all": {
    "cat_map": 0.6966951514439714,
    "cat_p5": 0.9,
    "cat_p10": 0.75,
    "class_map": 0.6103205175324865,
    "class_top3": 0.8428571428571429,
    "tag_acc": 0.5662100456621004,
    "tag_mrr": 0.6955952165165974,
    "pooled_auc": 0.9520811285683274,
    "pooled_auc_strict": 0.9379881695480244,
    "spec_top1": 0.7083333333333334,
    "spec_top3": 0.9166666666666666
   },
   "dev": {
    "cat_map": 0.6999603235345861,
    "cat_p5": 0.775,
    "cat_p10": 0.6375000000000001,
    "class_map": 0.6421799826780327,
    "class_top3": 0.8225806451612904,
    "tag_acc": 0.5766129032258065,
    "tag_mrr": 0.7039788209421436,
    "pooled_auc": 0.9599737053930776,
    "pooled_auc_strict": 0.9465689205219454,
    "spec_top1": 0.6,
    "spec_top3": 0.9
   },
   "test": {
    "cat_map": 0.6923689595029523,
    "cat_p5": 0.7,
    "cat_p10": 0.6375,
    "class_map": 0.6342054217314276,
    "class_top3": 0.7931034482758621,
    "tag_acc": 0.5526315789473685,
    "tag_mrr": 0.6846524065295687,
    "pooled_auc": 0.9417612480137032,
    "pooled_auc_strict": 0.9267723463508731,
    "spec_top1": 0.7619047619047619,
    "spec_top3": 0.9047619047619048
   }
  },
  {
   "pad": "zero",
   "templates": "t1",
   "method": "both_sd",
   "all": {
    "cat_map": 0.7357901241335416,
    "cat_p5": 0.825,
    "cat_p10": 0.8,
    "class_map": 0.5762451701876967,
    "class_top3": 0.8142857142857143,
    "tag_acc": 0.5662100456621004,
    "tag_mrr": 0.6955952165165974,
    "pooled_auc": 0.9544702458482579,
    "pooled_auc_strict": 0.9397197983957355,
    "spec_top1": 0.875,
    "spec_top3": 0.9166666666666666
   },
   "dev": {
    "cat_map": 0.7328908579222572,
    "cat_p5": 0.8,
    "cat_p10": 0.675,
    "class_map": 0.6431110174811173,
    "class_top3": 0.8225806451612904,
    "tag_acc": 0.5766129032258065,
    "tag_mrr": 0.7039788209421436,
    "pooled_auc": 0.9587121008854307,
    "pooled_auc_strict": 0.9446315539739027,
    "spec_top1": 0.8,
    "spec_top3": 0.9
   },
   "test": {
    "cat_map": 0.7635682012315999,
    "cat_p5": 0.775,
    "cat_p10": 0.6749999999999999,
    "class_map": 0.6058761528294955,
    "class_top3": 0.8275862068965517,
    "tag_acc": 0.5526315789473685,
    "tag_mrr": 0.6846524065295687,
    "pooled_auc": 0.9489826975414158,
    "pooled_auc_strict": 0.9332092579108062,
    "spec_top1": 0.8095238095238095,
    "spec_top3": 0.8571428571428571
   }
  },
  {
   "pad": "zero",
   "templates": "t3",
   "method": "raw",
   "all": {
    "cat_map": 0.7113266964725198,
    "cat_p5": 0.85,
    "cat_p10": 0.7749999999999999,
    "class_map": 0.6242070802503793,
    "class_top3": 0.8428571428571429,
    "tag_acc": 0.589041095890411,
    "tag_mrr": 0.7051300820155875,
    "pooled_auc": 0.9528507611527189,
    "pooled_auc_strict": 0.9377459381993843,
    "spec_top1": 0.75,
    "spec_top3": 0.9166666666666666
   },
   "dev": {
    "cat_map": 0.7082306928343325,
    "cat_p5": 0.825,
    "cat_p10": 0.7124999999999999,
    "class_map": 0.6412667770398142,
    "class_top3": 0.8548387096774194,
    "tag_acc": 0.6209677419354839,
    "tag_mrr": 0.7232177412169984,
    "pooled_auc": 0.96023557821304,
    "pooled_auc_strict": 0.9453508896797153,
    "spec_top1": 0.6,
    "spec_top3": 0.9
   },
   "test": {
    "cat_map": 0.7129527355473899,
    "cat_p5": 0.75,
    "cat_p10": 0.6625,
    "class_map": 0.6568296279529588,
    "class_top3": 0.7931034482758621,
    "tag_acc": 0.5473684210526316,
    "tag_mrr": 0.68152092684743,
    "pooled_auc": 0.9435396250721083,
    "pooled_auc_strict": 0.9281328644975025,
    "spec_top1": 0.8571428571428571,
    "spec_top3": 0.8571428571428571
   }
  },
  {
   "pad": "zero",
   "templates": "t3",
   "method": "sound",
   "all": {
    "cat_map": 0.7479251873265214,
    "cat_p5": 0.85,
    "cat_p10": 0.8374999999999999,
    "class_map": 0.6260961543743475,
    "class_top3": 0.8571428571428571,
    "tag_acc": 0.589041095890411,
    "tag_mrr": 0.7051300820155875,
    "pooled_auc": 0.9592016715510692,
    "pooled_auc_strict": 0.9445763780709368,
    "spec_top1": 0.875,
    "spec_top3": 0.9166666666666666
   },
   "dev": {
    "cat_map": 0.7428199736405021,
    "cat_p5": 0.875,
    "cat_p10": 0.7,
    "class_map": 0.6625634652022035,
    "class_top3": 0.8870967741935484,
    "tag_acc": 0.6209677419354839,
    "tag_mrr": 0.7232177412169984,
    "pooled_auc": 0.9641454252750201,
    "pooled_auc_strict": 0.9495302491103202,
    "spec_top1": 0.8,
    "spec_top3": 1.0
   },
   "test": {
    "cat_map": 0.7732243482406984,
    "cat_p5": 0.8,
    "cat_p10": 0.7000000000000001,
    "class_map": 0.619360899639458,
    "class_top3": 0.8103448275862069,
    "tag_acc": 0.5473684210526316,
    "tag_mrr": 0.68152092684743,
    "pooled_auc": 0.9527509104228822,
    "pooled_auc_strict": 0.9381084007204997,
    "spec_top1": 0.8571428571428571,
    "spec_top3": 0.9047619047619048
   }
  },
  {
   "pad": "zero",
   "templates": "t3",
   "method": "center",
   "all": {
    "cat_map": 0.7113266964725198,
    "cat_p5": 0.85,
    "cat_p10": 0.7749999999999999,
    "class_map": 0.6242070802503793,
    "class_top3": 0.8428571428571429,
    "tag_acc": 0.5730593607305936,
    "tag_mrr": 0.7024972971901691,
    "pooled_auc": 0.9569955497666341,
    "pooled_auc_strict": 0.9427986027139594,
    "spec_top1": 0.75,
    "spec_top3": 0.9166666666666666
   },
   "dev": {
    "cat_map": 0.7082306928343325,
    "cat_p5": 0.825,
    "cat_p10": 0.7124999999999999,
    "class_map": 0.6412667770398142,
    "class_top3": 0.8548387096774194,
    "tag_acc": 0.6008064516129032,
    "tag_mrr": 0.7192411307330662,
    "pooled_auc": 0.9652535551381809,
    "pooled_auc_strict": 0.9514296559905101,
    "spec_top1": 0.6,
    "spec_top3": 0.9
   },
   "test": {
    "cat_map": 0.7129527355473899,
    "cat_p5": 0.75,
    "cat_p10": 0.6625,
    "class_map": 0.6568296279529588,
    "class_top3": 0.7931034482758621,
    "tag_acc": 0.5368421052631579,
    "tag_mrr": 0.6806421881447032,
    "pooled_auc": 0.9465230516596417,
    "pooled_auc_strict": 0.9319054280202864,
    "spec_top1": 0.8571428571428571,
    "spec_top3": 0.8571428571428571
   }
  },
  {
   "pad": "zero",
   "templates": "t3",
   "method": "both",
   "all": {
    "cat_map": 0.7545544072897863,
    "cat_p5": 0.85,
    "cat_p10": 0.8374999999999999,
    "class_map": 0.6295778568154784,
    "class_top3": 0.8428571428571429,
    "tag_acc": 0.5730593607305936,
    "tag_mrr": 0.7024972971901691,
    "pooled_auc": 0.9623862002062303,
    "pooled_auc_strict": 0.9479070969397451,
    "spec_top1": 0.9166666666666666,
    "spec_top3": 0.9166666666666666
   },
   "dev": {
    "cat_map": 0.7569497491591481,
    "cat_p5": 0.85,
    "cat_p10": 0.7374999999999999,
    "class_map": 0.6593895422627897,
    "class_top3": 0.8387096774193549,
    "tag_acc": 0.6008064516129032,
    "tag_mrr": 0.7192411307330662,
    "pooled_auc": 0.9682763616850013,
    "pooled_auc_strict": 0.9543800711743772,
    "spec_top1": 0.85,
    "spec_top3": 0.95
   },
   "test": {
    "cat_map": 0.7768694680931955,
    "cat_p5": 0.8,
    "cat_p10": 0.725,
    "class_map": 0.6320349511109097,
    "class_top3": 0.8275862068965517,
    "tag_acc": 0.5368421052631579,
    "tag_mrr": 0.6806421881447032,
    "pooled_auc": 0.9545637933048086,
    "pooled_auc_strict": 0.9393882778267479,
    "spec_top1": 0.8571428571428571,
    "spec_top3": 0.9047619047619048
   }
  },
  {
   "pad": "zero",
   "templates": "t3",
   "method": "term_sd",
   "all": {
    "cat_map": 0.7113266964725198,
    "cat_p5": 0.85,
    "cat_p10": 0.7749999999999999,
    "class_map": 0.6242070802503793,
    "class_top3": 0.8428571428571429,
    "tag_acc": 0.5707762557077626,
    "tag_mrr": 0.7004832234156526,
    "pooled_auc": 0.9547480462390101,
    "pooled_auc_strict": 0.9412084393946546,
    "spec_top1": 0.75,
    "spec_top3": 0.9166666666666666
   },
   "dev": {
    "cat_map": 0.7082306928343325,
    "cat_p5": 0.825,
    "cat_p10": 0.7124999999999999,
    "class_map": 0.6412667770398142,
    "class_top3": 0.8548387096774194,
    "tag_acc": 0.5806451612903226,
    "tag_mrr": 0.7099978762277149,
    "pooled_auc": 0.9625784813522941,
    "pooled_auc_strict": 0.9494111506524318,
    "spec_top1": 0.6,
    "spec_top3": 0.9
   },
   "test": {
    "cat_map": 0.7129527355473899,
    "cat_p5": 0.75,
    "cat_p10": 0.6625,
    "class_map": 0.6568296279529588,
    "class_top3": 0.7931034482758621,
    "tag_acc": 0.5578947368421052,
    "tag_mrr": 0.6880640976399084,
    "pooled_auc": 0.9446561981306691,
    "pooled_auc_strict": 0.9306151713741873,
    "spec_top1": 0.8571428571428571,
    "spec_top3": 0.8571428571428571
   }
  },
  {
   "pad": "zero",
   "templates": "t3",
   "method": "both_sd",
   "all": {
    "cat_map": 0.7419979603370317,
    "cat_p5": 0.825,
    "cat_p10": 0.8125,
    "class_map": 0.6137491601963941,
    "class_top3": 0.8142857142857143,
    "tag_acc": 0.5707762557077626,
    "tag_mrr": 0.7004832234156526,
    "pooled_auc": 0.9591180600781505,
    "pooled_auc_strict": 0.9443836218322312,
    "spec_top1": 0.9166666666666666,
    "spec_top3": 0.9166666666666666
   },
   "dev": {
    "cat_map": 0.7383885442934307,
    "cat_p5": 0.8,
    "cat_p10": 0.7,
    "class_map": 0.6636712966205031,
    "class_top3": 0.8387096774193549,
    "tag_acc": 0.5806451612903226,
    "tag_mrr": 0.7099978762277149,
    "pooled_auc": 0.9639930238797961,
    "pooled_auc_strict": 0.9498642941874259,
    "spec_top1": 0.8,
    "spec_top3": 0.95
   },
   "test": {
    "cat_map": 0.7736227636847813,
    "cat_p5": 0.8,
    "cat_p10": 0.6875,
    "class_map": 0.6297619157865565,
    "class_top3": 0.8448275862068966,
    "tag_acc": 0.5578947368421052,
    "tag_mrr": 0.6880640976399084,
    "pooled_auc": 0.9527447170699425,
    "pooled_auc_strict": 0.9371798311169024,
    "spec_top1": 0.8571428571428571,
    "spec_top3": 0.9047619047619048
   }
  },
  {
   "pad": "zero",
   "templates": "t5",
   "method": "raw",
   "all": {
    "cat_map": 0.7195432854073197,
    "cat_p5": 0.875,
    "cat_p10": 0.7875,
    "class_map": 0.6207924807074807,
    "class_top3": 0.8285714285714286,
    "tag_acc": 0.6004566210045662,
    "tag_mrr": 0.7122834126130668,
    "pooled_auc": 0.9552127428633452,
    "pooled_auc_strict": 0.9398409897200707,
    "spec_top1": 0.8333333333333334,
    "spec_top3": 0.9583333333333334
   },
   "dev": {
    "cat_map": 0.7192035638255845,
    "cat_p5": 0.825,
    "cat_p10": 0.7124999999999999,
    "class_map": 0.6365957342756754,
    "class_top3": 0.8548387096774194,
    "tag_acc": 0.6209677419354839,
    "tag_mrr": 0.7251502314078777,
    "pooled_auc": 0.96239602897773,
    "pooled_auc_strict": 0.9472868327402135,
    "spec_top1": 0.7,
    "spec_top3": 0.9
   },
   "test": {
    "cat_map": 0.7103068995186603,
    "cat_p5": 0.75,
    "cat_p10": 0.6375,
    "class_map": 0.6498509450129192,
    "class_top3": 0.7931034482758621,
    "tag_acc": 0.5736842105263158,
    "tag_mrr": 0.6954888280808925,
    "pooled_auc": 0.9461948039538365,
    "pooled_auc_strict": 0.9304483003104281,
    "spec_top1": 0.8095238095238095,
    "spec_top3": 0.8571428571428571
   }
  },
  {
   "pad": "zero",
   "templates": "t5",
   "method": "sound",
   "all": {
    "cat_map": 0.7528268584243417,
    "cat_p5": 0.85,
    "cat_p10": 0.8500000000000001,
    "class_map": 0.631513081220779,
    "class_top3": 0.8714285714285714,
    "tag_acc": 0.6004566210045662,
    "tag_mrr": 0.7122834126130668,
    "pooled_auc": 0.9614140684359058,
    "pooled_auc_strict": 0.946796857316809,
    "spec_top1": 0.875,
    "spec_top3": 0.9583333333333334
   },
   "dev": {
    "cat_map": 0.7561693052788976,
    "cat_p5": 0.8999999999999999,
    "cat_p10": 0.725,
    "class_map": 0.6608365270149242,
    "class_top3": 0.8870967741935484,
    "tag_acc": 0.6209677419354839,
    "tag_mrr": 0.7251502314078777,
    "pooled_auc": 0.9661851354977193,
    "pooled_auc_strict": 0.9514424673784104,
    "spec_top1": 0.8,
    "spec_top3": 1.0
   },
   "test": {
    "cat_map": 0.7558766882096197,
    "cat_p5": 0.8,
    "cat_p10": 0.6875,
    "class_map": 0.6321180749414527,
    "class_top3": 0.8275862068965517,
    "tag_acc": 0.5736842105263158,
    "tag_mrr": 0.6954888280808925,
    "pooled_auc": 0.9552176344223018,
    "pooled_auc_strict": 0.9407823298713576,
    "spec_top1": 0.8095238095238095,
    "spec_top3": 0.9047619047619048
   }
  },
  {
   "pad": "zero",
   "templates": "t5",
   "method": "center",
   "all": {
    "cat_map": 0.7195432854073197,
    "cat_p5": 0.875,
    "cat_p10": 0.7875,
    "class_map": 0.6207924807074807,
    "class_top3": 0.8285714285714286,
    "tag_acc": 0.591324200913242,
    "tag_mrr": 0.713834516703891,
    "pooled_auc": 0.9582488738738739,
    "pooled_auc_strict": 0.9438378826226468,
    "spec_top1": 0.8333333333333334,
    "spec_top3": 0.9583333333333334
   },
   "dev": {
    "cat_map": 0.7192035638255845,
    "cat_p5": 0.825,
    "cat_p10": 0.7124999999999999,
    "class_map": 0.6365957342756754,
    "class_top3": 0.8548387096774194,
    "tag_acc": 0.5967741935483871,
    "tag_mrr": 0.7193428063719185,
    "pooled_auc": 0.9658282801180574,
    "pooled_auc_strict": 0.951911743772242,
    "spec_top1": 0.7,
    "spec_top3": 0.9
   },
   "test": {
    "cat_map": 0.7103068995186603,
    "cat_p5": 0.75,
    "cat_p10": 0.6375,
    "class_map": 0.6498509450129192,
    "class_top3": 0.7931034482758621,
    "tag_acc": 0.5842105263157895,
    "tag_mrr": 0.7066447491372023,
    "pooled_auc": 0.9485571257179866,
    "pooled_auc_strict": 0.9336044788512884,
    "spec_top1": 0.8095238095238095,
    "spec_top3": 0.8571428571428571
   }
  },
  {
   "pad": "zero",
   "templates": "t5",
   "method": "both",
   "all": {
    "cat_map": 0.7583908419074668,
    "cat_p5": 0.85,
    "cat_p10": 0.8374999999999999,
    "class_map": 0.631183476723012,
    "class_top3": 0.8428571428571429,
    "tag_acc": 0.591324200913242,
    "tag_mrr": 0.713834516703891,
    "pooled_auc": 0.9637078720286552,
    "pooled_auc_strict": 0.9492772700148758,
    "spec_top1": 0.875,
    "spec_top3": 0.9166666666666666
   },
   "dev": {
    "cat_map": 0.7592935004031254,
    "cat_p5": 0.85,
    "cat_p10": 0.7374999999999999,
    "class_map": 0.6625594786345831,
    "class_top3": 0.8548387096774194,
    "tag_acc": 0.5967741935483871,
    "tag_mrr": 0.7193428063719185,
    "pooled_auc": 0.9691033002414811,
    "pooled_auc_strict": 0.9551772241992883,
    "spec_top1": 0.85,
    "spec_top3": 0.95
   },
   "test": {
    "cat_map": 0.7741438076784091,
    "cat_p5": 0.8,
    "cat_p10": 0.725,
    "class_map": 0.6288217086231476,
    "class_top3": 0.8793103448275862,
    "tag_acc": 0.5842105263157895,
    "tag_mrr": 0.7066447491372023,
    "pooled_auc": 0.9565430119514017,
    "pooled_auc_strict": 0.9414977196949373,
    "spec_top1": 0.8095238095238095,
    "spec_top3": 0.9047619047619048
   }
  },
  {
   "pad": "zero",
   "templates": "t5",
   "method": "term_sd",
   "all": {
    "cat_map": 0.7195432854073197,
    "cat_p5": 0.875,
    "cat_p10": 0.7875,
    "class_map": 0.6207924807074807,
    "class_top3": 0.8285714285714286,
    "tag_acc": 0.5753424657534246,
    "tag_mrr": 0.7039218843087036,
    "pooled_auc": 0.9559677900792358,
    "pooled_auc_strict": 0.9423465182231808,
    "spec_top1": 0.8333333333333334,
    "spec_top3": 0.9583333333333334
   },
   "dev": {
    "cat_map": 0.7192035638255845,
    "cat_p5": 0.825,
    "cat_p10": 0.7124999999999999,
    "class_map": 0.6365957342756754,
    "class_top3": 0.8548387096774194,
    "tag_acc": 0.5887096774193549,
    "tag_mrr": 0.7139980675215183,
    "pooled_auc": 0.9630893480010733,
    "pooled_auc_strict": 0.9499814946619217,
    "spec_top1": 0.7,
    "spec_top3": 0.9
   },
   "test": {
    "cat_map": 0.7103068995186603,
    "cat_p5": 0.75,
    "cat_p10": 0.6375,
    "class_map": 0.6498509450129192,
    "class_top3": 0.7931034482758621,
    "tag_acc": 0.5578947368421052,
    "tag_mrr": 0.6907698135888196,
    "pooled_auc": 0.9468663403654433,
    "pooled_auc_strict": 0.9326407784974259,
    "spec_top1": 0.8095238095238095,
    "spec_top3": 0.8571428571428571
   }
  },
  {
   "pad": "zero",
   "templates": "t5",
   "method": "both_sd",
   "all": {
    "cat_map": 0.7440666122282521,
    "cat_p5": 0.85,
    "cat_p10": 0.8,
    "class_map": 0.6171414447933016,
    "class_top3": 0.8428571428571429,
    "tag_acc": 0.5753424657534246,
    "tag_mrr": 0.7039218843087036,
    "pooled_auc": 0.9602011084880061,
    "pooled_auc_strict": 0.9455557431675933,
    "spec_top1": 0.875,
    "spec_top3": 0.9166666666666666
   },
   "dev": {
    "cat_map": 0.7428221662902166,
    "cat_p5": 0.8,
    "cat_p10": 0.7124999999999999,
    "class_map": 0.6513844220634469,
    "class_top3": 0.8548387096774194,
    "tag_acc": 0.5887096774193549,
    "tag_mrr": 0.7139980675215183,
    "pooled_auc": 0.9645468204990609,
    "pooled_auc_strict": 0.9504583629893238,
    "spec_top1": 0.8,
    "spec_top3": 0.95
   },
   "test": {
    "cat_map": 0.7663582183951299,
    "cat_p5": 0.8,
    "cat_p10": 0.7000000000000001,
    "class_map": 0.6256447411677083,
    "class_top3": 0.8448275862068966,
    "tag_acc": 0.5578947368421052,
    "tag_mrr": 0.6907698135888196,
    "pooled_auc": 0.9544602758342446,
    "pooled_auc_strict": 0.9390521404208025,
    "spec_top1": 0.8095238095238095,
    "spec_top3": 0.9047619047619048
   }
  },
  {
   "pad": "spaced",
   "templates": "t1",
   "method": "raw",
   "all": {
    "cat_map": 0.706922872057108,
    "cat_p5": 0.875,
    "cat_p10": 0.7875000000000001,
    "class_map": 0.6107295823937894,
    "class_top3": 0.8714285714285714,
    "tag_acc": 0.5616438356164384,
    "tag_mrr": 0.6909010612328351,
    "pooled_auc": 0.9446412338543363,
    "pooled_auc_strict": 0.9287270946884006,
    "spec_top1": 0.75,
    "spec_top3": 0.9583333333333334
   },
   "dev": {
    "cat_map": 0.6884970537189887,
    "cat_p5": 0.75,
    "cat_p10": 0.6499999999999999,
    "class_map": 0.6423744153944477,
    "class_top3": 0.8548387096774194,
    "tag_acc": 0.592741935483871,
    "tag_mrr": 0.7138462062406248,
    "pooled_auc": 0.9530222699221894,
    "pooled_auc_strict": 0.9374448398576513,
    "spec_top1": 0.7,
    "spec_top3": 0.9
   },
   "test": {
    "cat_map": 0.7079575497135653,
    "cat_p5": 0.75,
    "cat_p10": 0.6625000000000001,
    "class_map": 0.6028376226217312,
    "class_top3": 0.7758620689655172,
    "tag_acc": 0.5210526315789473,
    "tag_mrr": 0.660951608801615,
    "pooled_auc": 0.93430268154487,
    "pooled_auc_strict": 0.917817198737848,
    "spec_top1": 0.6190476190476191,
    "spec_top3": 0.9047619047619048
   }
  },
  {
   "pad": "spaced",
   "templates": "t1",
   "method": "sound",
   "all": {
    "cat_map": 0.7490033176025689,
    "cat_p5": 0.8500000000000001,
    "cat_p10": 0.825,
    "class_map": 0.6199766764100778,
    "class_top3": 0.8714285714285714,
    "tag_acc": 0.5616438356164384,
    "tag_mrr": 0.6909010612328351,
    "pooled_auc": 0.956998772115489,
    "pooled_auc_strict": 0.940935040239756,
    "spec_top1": 0.875,
    "spec_top3": 0.9583333333333334
   },
   "dev": {
    "cat_map": 0.7477036476412733,
    "cat_p5": 0.825,
    "cat_p10": 0.7375,
    "class_map": 0.6691017688292265,
    "class_top3": 0.8870967741935484,
    "tag_acc": 0.592741935483871,
    "tag_mrr": 0.7138462062406248,
    "pooled_auc": 0.9636592433592702,
    "pooled_auc_strict": 0.9478372479240806,
    "spec_top1": 0.9,
    "spec_top3": 0.95
   },
   "test": {
    "cat_map": 0.7696312428996632,
    "cat_p5": 0.8,
    "cat_p10": 0.7,
    "class_map": 0.6155754649787434,
    "class_top3": 0.7931034482758621,
    "tag_acc": 0.5210526315789473,
    "tag_mrr": 0.660951608801615,
    "pooled_auc": 0.948381942306263,
    "pooled_auc_strict": 0.9320555321350554,
    "spec_top1": 0.8571428571428571,
    "spec_top3": 0.9047619047619048
   }
  },
  {
   "pad": "spaced",
   "templates": "t1",
   "method": "center",
   "all": {
    "cat_map": 0.706922872057108,
    "cat_p5": 0.875,
    "cat_p10": 0.7875000000000001,
    "class_map": 0.6107295823937894,
    "class_top3": 0.8714285714285714,
    "tag_acc": 0.5776255707762558,
    "tag_mrr": 0.7087323497675215,
    "pooled_auc": 0.949996608053837,
    "pooled_auc_strict": 0.9345255170527238,
    "spec_top1": 0.75,
    "spec_top3": 0.9583333333333334
   },
   "dev": {
    "cat_map": 0.6884970537189887,
    "cat_p5": 0.75,
    "cat_p10": 0.6499999999999999,
    "class_map": 0.6423744153944477,
    "class_top3": 0.8548387096774194,
    "tag_acc": 0.6129032258064516,
    "tag_mrr": 0.7366968087148714,
    "pooled_auc": 0.9584314462033807,
    "pooled_auc_strict": 0.9437812574139977,
    "spec_top1": 0.7,
    "spec_top3": 0.9
   },
   "test": {
    "cat_map": 0.7079575497135653,
    "cat_p5": 0.75,
    "cat_p10": 0.6625000000000001,
    "class_map": 0.6028376226217312,
    "class_top3": 0.7758620689655172,
    "tag_acc": 0.531578947368421,
    "tag_mrr": 0.6722313717730859,
    "pooled_auc": 0.9392547096025283,
    "pooled_auc_strict": 0.922591787069329,
    "spec_top1": 0.6190476190476191,
    "spec_top3": 0.9047619047619048
   }
  },
  {
   "pad": "spaced",
   "templates": "t1",
   "method": "both",
   "all": {
    "cat_map": 0.7673928880987917,
    "cat_p5": 0.85,
    "cat_p10": 0.825,
    "class_map": 0.6187623766919265,
    "class_top3": 0.8571428571428571,
    "tag_acc": 0.5776255707762558,
    "tag_mrr": 0.7087323497675215,
    "pooled_auc": 0.9614535846087051,
    "pooled_auc_strict": 0.9457649911096102,
    "spec_top1": 0.8333333333333334,
    "spec_top3": 0.9583333333333334
   },
   "dev": {
    "cat_map": 0.7531848837739687,
    "cat_p5": 0.8,
    "cat_p10": 0.7374999999999999,
    "class_map": 0.6936505103141615,
    "class_top3": 0.9032258064516129,
    "tag_acc": 0.6129032258064516,
    "tag_mrr": 0.7366968087148714,
    "pooled_auc": 0.9673871746713174,
    "pooled_auc_strict": 0.9529214709371293,
    "spec_top1": 0.85,
    "spec_top3": 0.95
   },
   "test": {
    "cat_map": 0.7791705700145516,
    "cat_p5": 0.8,
    "cat_p10": 0.7250000000000001,
    "class_map": 0.6199613689407614,
    "class_top3": 0.8275862068965517,
    "tag_acc": 0.531578947368421,
    "tag_mrr": 0.6722313717730859,
    "pooled_auc": 0.9536958391285422,
    "pooled_auc_strict": 0.9363742191392328,
    "spec_top1": 0.8571428571428571,
    "spec_top3": 0.9047619047619048
   }
  },
  {
   "pad": "spaced",
   "templates": "t1",
   "method": "term_sd",
   "all": {
    "cat_map": 0.706922872057108,
    "cat_p5": 0.875,
    "cat_p10": 0.7875000000000001,
    "class_map": 0.6107295823937894,
    "class_top3": 0.8714285714285714,
    "tag_acc": 0.54337899543379,
    "tag_mrr": 0.6884737470690645,
    "pooled_auc": 0.947702126071855,
    "pooled_auc_strict": 0.9326695195800395,
    "spec_top1": 0.75,
    "spec_top3": 0.9583333333333334
   },
   "dev": {
    "cat_map": 0.6884970537189887,
    "cat_p5": 0.75,
    "cat_p10": 0.6499999999999999,
    "class_map": 0.6423744153944477,
    "class_top3": 0.8548387096774194,
    "tag_acc": 0.592741935483871,
    "tag_mrr": 0.7203287804617625,
    "pooled_auc": 0.9560660048296217,
    "pooled_auc_strict": 0.941944009489917,
    "spec_top1": 0.7,
    "spec_top3": 0.9
   },
   "test": {
    "cat_map": 0.7079575497135653,
    "cat_p5": 0.75,
    "cat_p10": 0.6625000000000001,
    "class_map": 0.6028376226217312,
    "class_top3": 0.7758620689655172,
    "tag_acc": 0.4789473684210526,
    "tag_mrr": 0.6468945455880694,
    "pooled_auc": 0.9369729014265946,
    "pooled_auc_strict": 0.920643627281902,
    "spec_top1": 0.6190476190476191,
    "spec_top3": 0.9047619047619048
   }
  },
  {
   "pad": "spaced",
   "templates": "t1",
   "method": "both_sd",
   "all": {
    "cat_map": 0.7663527622016136,
    "cat_p5": 0.85,
    "cat_p10": 0.8125,
    "class_map": 0.6053154405539275,
    "class_top3": 0.8428571428571429,
    "tag_acc": 0.54337899543379,
    "tag_mrr": 0.6884737470690645,
    "pooled_auc": 0.9586613345272984,
    "pooled_auc_strict": 0.9423814685301989,
    "spec_top1": 0.8333333333333334,
    "spec_top3": 0.9166666666666666
   },
   "dev": {
    "cat_map": 0.7489604716512064,
    "cat_p5": 0.8,
    "cat_p10": 0.75,
    "class_map": 0.6948401730457527,
    "class_top3": 0.9032258064516129,
    "tag_acc": 0.592741935483871,
    "tag_mrr": 0.7203287804617625,
    "pooled_auc": 0.9639989267507378,
    "pooled_auc_strict": 0.9490481613285884,
    "spec_top1": 0.85,
    "spec_top3": 0.95
   },
   "test": {
    "cat_map": 0.7970968265684556,
    "cat_p5": 0.8,
    "cat_p10": 0.725,
    "class_map": 0.6219079826197252,
    "class_top3": 0.8275862068965517,
    "tag_acc": 0.4789473684210526,
    "tag_mrr": 0.6468945455880694,
    "pooled_auc": 0.9516599955407858,
    "pooled_auc_strict": 0.9335222409586224,
    "spec_top1": 0.7619047619047619,
    "spec_top3": 0.9047619047619048
   }
  },
  {
   "pad": "spaced",
   "templates": "t3",
   "method": "raw",
   "all": {
    "cat_map": 0.733627480353698,
    "cat_p5": 0.875,
    "cat_p10": 0.8125,
    "class_map": 0.631684550946142,
    "class_top3": 0.8857142857142857,
    "tag_acc": 0.591324200913242,
    "tag_mrr": 0.7090230796224233,
    "pooled_auc": 0.9471253256268316,
    "pooled_auc_strict": 0.9317075539868769,
    "spec_top1": 0.875,
    "spec_top3": 0.9583333333333334
   },
   "dev": {
    "cat_map": 0.721332154539629,
    "cat_p5": 0.825,
    "cat_p10": 0.7,
    "class_map": 0.6685541340636711,
    "class_top3": 0.8548387096774194,
    "tag_acc": 0.625,
    "tag_mrr": 0.7321579254982588,
    "pooled_auc": 0.955209551918433,
    "pooled_auc_strict": 0.9403696322657177,
    "spec_top1": 0.8,
    "spec_top3": 0.9
   },
   "test": {
    "cat_map": 0.7408307618711064,
    "cat_p5": 0.775,
    "cat_p10": 0.675,
    "class_map": 0.6341914042688647,
    "class_top3": 0.8275862068965517,
    "tag_acc": 0.5473684210526316,
    "tag_mrr": 0.6788260176371224,
    "pooled_auc": 0.9368516886619173,
    "pooled_auc_strict": 0.9206635879354617,
    "spec_top1": 0.7142857142857143,
    "spec_top3": 0.9047619047619048
   }
  },
  {
   "pad": "spaced",
   "templates": "t3",
   "method": "sound",
   "all": {
    "cat_map": 0.7712992046064168,
    "cat_p5": 0.85,
    "cat_p10": 0.8375,
    "class_map": 0.6432533280537299,
    "class_top3": 0.9,
    "tag_acc": 0.591324200913242,
    "tag_mrr": 0.7090230796224233,
    "pooled_auc": 0.959616845761424,
    "pooled_auc_strict": 0.9441547048862647,
    "spec_top1": 0.8333333333333334,
    "spec_top3": 1.0
   },
   "dev": {
    "cat_map": 0.7656542451859354,
    "cat_p5": 0.875,
    "cat_p10": 0.7375,
    "class_map": 0.6948406693679899,
    "class_top3": 0.9193548387096774,
    "tag_acc": 0.625,
    "tag_mrr": 0.7321579254982588,
    "pooled_auc": 0.9671735980681513,
    "pooled_auc_strict": 0.9523753262158956,
    "spec_top1": 0.85,
    "spec_top3": 1.0
   },
   "test": {
    "cat_map": 0.7943363173833515,
    "cat_p5": 0.775,
    "cat_p10": 0.7375,
    "class_map": 0.6240989688025537,
    "class_top3": 0.8275862068965517,
    "tag_acc": 0.5473684210526316,
    "tag_mrr": 0.6788260176371224,
    "pooled_auc": 0.9499099309529624,
    "pooled_auc_strict": 0.9336116646865699,
    "spec_top1": 0.8095238095238095,
    "spec_top3": 0.9047619047619048
   }
  },
  {
   "pad": "spaced",
   "templates": "t3",
   "method": "center",
   "all": {
    "cat_map": 0.733627480353698,
    "cat_p5": 0.875,
    "cat_p10": 0.8125,
    "class_map": 0.631684550946142,
    "class_top3": 0.8857142857142857,
    "tag_acc": 0.5821917808219178,
    "tag_mrr": 0.7082388382353201,
    "pooled_auc": 0.9511218861934223,
    "pooled_auc_strict": 0.93656489016223,
    "spec_top1": 0.875,
    "spec_top3": 0.9583333333333334
   },
   "dev": {
    "cat_map": 0.721332154539629,
    "cat_p5": 0.825,
    "cat_p10": 0.7,
    "class_map": 0.6685541340636711,
    "class_top3": 0.8548387096774194,
    "tag_acc": 0.6129032258064516,
    "tag_mrr": 0.7332454862192767,
    "pooled_auc": 0.9593807351757445,
    "pooled_auc_strict": 0.9456540925266904,
    "spec_top1": 0.8,
    "spec_top3": 0.9
   },
   "test": {
    "cat_map": 0.7408307618711064,
    "cat_p5": 0.775,
    "cat_p10": 0.675,
    "class_map": 0.6341914042688647,
    "class_top3": 0.8275862068965517,
    "tag_acc": 0.5421052631578948,
    "tag_mrr": 0.6755985819194191,
    "pooled_auc": 0.9407968544845184,
    "pooled_auc_strict": 0.9251156121054178,
    "spec_top1": 0.7142857142857143,
    "spec_top3": 0.9047619047619048
   }
  },
  {
   "pad": "spaced",
   "templates": "t3",
   "method": "both",
   "all": {
    "cat_map": 0.775188178344212,
    "cat_p5": 0.85,
    "cat_p10": 0.825,
    "class_map": 0.6431672182428108,
    "class_top3": 0.9,
    "tag_acc": 0.5821917808219178,
    "tag_mrr": 0.7082388382353201,
    "pooled_auc": 0.9633947953978075,
    "pooled_auc_strict": 0.9482859522158192,
    "spec_top1": 0.8333333333333334,
    "spec_top3": 0.9583333333333334
   },
   "dev": {
    "cat_map": 0.7703344479824905,
    "cat_p5": 0.85,
    "cat_p10": 0.75,
    "class_map": 0.6863326451282192,
    "class_top3": 0.9193548387096774,
    "tag_acc": 0.6129032258064516,
    "tag_mrr": 0.7332454862192767,
    "pooled_auc": 0.9702516769519721,
    "pooled_auc_strict": 0.9564493475682088,
    "spec_top1": 0.9,
    "spec_top3": 0.95
   },
   "test": {
    "cat_map": 0.7965686178706268,
    "cat_p5": 0.775,
    "cat_p10": 0.7375,
    "class_map": 0.6331310086379538,
    "class_top3": 0.8275862068965517,
    "tag_acc": 0.5421052631578948,
    "tag_mrr": 0.6755985819194191,
    "pooled_auc": 0.9545230941283475,
    "pooled_auc_strict": 0.9377227608937263,
    "spec_top1": 0.8095238095238095,
    "spec_top3": 0.9047619047619048
   }
  },
  {
   "pad": "spaced",
   "templates": "t3",
   "method": "term_sd",
   "all": {
    "cat_map": 0.733627480353698,
    "cat_p5": 0.875,
    "cat_p10": 0.8125,
    "class_map": 0.631684550946142,
    "class_top3": 0.8857142857142857,
    "tag_acc": 0.5639269406392694,
    "tag_mrr": 0.699348237848909,
    "pooled_auc": 0.9491552358080972,
    "pooled_auc_strict": 0.9350142161508546,
    "spec_top1": 0.875,
    "spec_top3": 0.9583333333333334
   },
   "dev": {
    "cat_map": 0.721332154539629,
    "cat_p5": 0.825,
    "cat_p10": 0.7,
    "class_map": 0.6685541340636711,
    "class_top3": 0.8548387096774194,
    "tag_acc": 0.5967741935483871,
    "tag_mrr": 0.7250881387877356,
    "pooled_auc": 0.9572465790179769,
    "pooled_auc_strict": 0.9439345195729537,
    "spec_top1": 0.8,
    "spec_top3": 0.9
   },
   "test": {
    "cat_map": 0.7408307618711064,
    "cat_p5": 0.775,
    "cat_p10": 0.675,
    "class_map": 0.6341914042688647,
    "class_top3": 0.8275862068965517,
    "tag_acc": 0.5210526315789473,
    "tag_mrr": 0.6657508934655983,
    "pooled_auc": 0.9390768718966878,
    "pooled_auc_strict": 0.9237942168397655,
    "spec_top1": 0.7142857142857143,
    "spec_top3": 0.9047619047619048
   }
  },
  {
   "pad": "spaced",
   "templates": "t3",
   "method": "both_sd",
   "all": {
    "cat_map": 0.7682368356425465,
    "cat_p5": 0.85,
    "cat_p10": 0.8125,
    "class_map": 0.6256953056101257,
    "class_top3": 0.8714285714285714,
    "tag_acc": 0.5639269406392694,
    "tag_mrr": 0.699348237848909,
    "pooled_auc": 0.961727653858678,
    "pooled_auc_strict": 0.9461088960786663,
    "spec_top1": 0.875,
    "spec_top3": 0.9583333333333334
   },
   "dev": {
    "cat_map": 0.7569966515589786,
    "cat_p5": 0.825,
    "cat_p10": 0.7375,
    "class_map": 0.6843928205030713,
    "class_top3": 0.9193548387096774,
    "tag_acc": 0.5967741935483871,
    "tag_mrr": 0.7250881387877356,
    "pooled_auc": 0.9678078883820768,
    "pooled_auc_strict": 0.9535316725978648,
    "spec_top1": 0.9,
    "spec_top3": 0.95
   },
   "test": {
    "cat_map": 0.7923047240435662,
    "cat_p5": 0.8,
    "cat_p10": 0.725,
    "class_map": 0.6436344180966361,
    "class_top3": 0.8448275862068966,
    "tag_acc": 0.5210526315789473,
    "tag_mrr": 0.6657508934655983,
    "pooled_auc": 0.9538444795990955,
    "pooled_auc_strict": 0.9364492711966173,
    "spec_top1": 0.9047619047619048,
    "spec_top3": 0.9047619047619048
   }
  },
  {
   "pad": "spaced",
   "templates": "t5",
   "method": "raw",
   "all": {
    "cat_map": 0.7403152944695368,
    "cat_p5": 0.875,
    "cat_p10": 0.8375,
    "class_map": 0.6306648972281436,
    "class_top3": 0.8857142857142857,
    "tag_acc": 0.6004566210045662,
    "tag_mrr": 0.7167575604039297,
    "pooled_auc": 0.9505566183653533,
    "pooled_auc_strict": 0.935182764384699,
    "spec_top1": 0.8333333333333334,
    "spec_top3": 0.9583333333333334
   },
   "dev": {
    "cat_map": 0.7288654463711456,
    "cat_p5": 0.85,
    "cat_p10": 0.7,
    "class_map": 0.6623461312956833,
    "class_top3": 0.8387096774193549,
    "tag_acc": 0.625,
    "tag_mrr": 0.7350574492825074,
    "pooled_auc": 0.9579908773812718,
    "pooled_auc_strict": 0.9432763938315539,
    "spec_top1": 0.8,
    "spec_top3": 0.9
   },
   "test": {
    "cat_map": 0.7339504300278596,
    "cat_p5": 0.775,
    "cat_p10": 0.675,
    "class_map": 0.644370763116193,
    "class_top3": 0.8103448275862069,
    "tag_acc": 0.5684210526315789,
    "tag_mrr": 0.6928713896571546,
    "pooled_auc": 0.9411640318373732,
    "pooled_auc_strict": 0.9250150104114769,
    "spec_top1": 0.7142857142857143,
    "spec_top3": 0.9523809523809523
   }
  },
  {
   "pad": "spaced",
   "templates": "t5",
   "method": "sound",
   "all": {
    "cat_map": 0.7754300541392372,
    "cat_p5": 0.85,
    "cat_p10": 0.8625,
    "class_map": 0.6522621153856848,
    "class_top3": 0.9142857142857143,
    "tag_acc": 0.6004566210045662,
    "tag_mrr": 0.7167575604039297,
    "pooled_auc": 0.9627605014653208,
    "pooled_auc_strict": 0.947461972250364,
    "spec_top1": 0.8333333333333334,
    "spec_top3": 1.0
   },
   "dev": {
    "cat_map": 0.7717425709437424,
    "cat_p5": 0.8999999999999999,
    "cat_p10": 0.75,
    "class_map": 0.6858513149747846,
    "class_top3": 0.9193548387096774,
    "tag_acc": 0.625,
    "tag_mrr": 0.7350574492825074,
    "pooled_auc": 0.969741346927824,
    "pooled_auc_strict": 0.9549988137603797,
    "spec_top1": 0.85,
    "spec_top3": 1.0
   },
   "test": {
    "cat_map": 0.7797688455311863,
    "cat_p5": 0.8,
    "cat_p10": 0.7375,
    "class_map": 0.6508153250257953,
    "class_top3": 0.8620689655172413,
    "tag_acc": 0.5684210526315789,
    "tag_mrr": 0.6928713896571546,
    "pooled_auc": 0.9538878330696734,
    "pooled_auc_strict": 0.9379255611338929,
    "spec_top1": 0.8095238095238095,
    "spec_top3": 0.9047619047619048
   }
  },
  {
   "pad": "spaced",
   "templates": "t5",
   "method": "center",
   "all": {
    "cat_map": 0.7403152944695368,
    "cat_p5": 0.875,
    "cat_p10": 0.8375,
    "class_map": 0.6306648972281436,
    "class_top3": 0.8857142857142857,
    "tag_acc": 0.589041095890411,
    "tag_mrr": 0.7166557925025382,
    "pooled_auc": 0.9532756024096386,
    "pooled_auc_strict": 0.9387681212046387,
    "spec_top1": 0.8333333333333334,
    "spec_top3": 0.9583333333333334
   },
   "dev": {
    "cat_map": 0.7288654463711456,
    "cat_p5": 0.85,
    "cat_p10": 0.7,
    "class_map": 0.6623461312956833,
    "class_top3": 0.8387096774193549,
    "tag_acc": 0.6048387096774194,
    "tag_mrr": 0.7341978621034598,
    "pooled_auc": 0.9607158572578481,
    "pooled_auc_strict": 0.9471805456702254,
    "spec_top1": 0.8,
    "spec_top3": 0.9
   },
   "test": {
    "cat_map": 0.7339504300278596,
    "cat_p5": 0.775,
    "cat_p10": 0.675,
    "class_map": 0.644370763116193,
    "class_top3": 0.8103448275862069,
    "tag_acc": 0.5684210526315789,
    "tag_mrr": 0.69375877533923,
    "pooled_auc": 0.9439333453661334,
    "pooled_auc_strict": 0.9281464377419231,
    "spec_top1": 0.7142857142857143,
    "spec_top3": 0.9523809523809523
   }
  },
  {
   "pad": "spaced",
   "templates": "t5",
   "method": "both",
   "all": {
    "cat_map": 0.7842722653956207,
    "cat_p5": 0.85,
    "cat_p10": 0.8500000000000001,
    "class_map": 0.6437703791946205,
    "class_top3": 0.8714285714285714,
    "tag_acc": 0.589041095890411,
    "tag_mrr": 0.7166557925025382,
    "pooled_auc": 0.965359749810051,
    "pooled_auc_strict": 0.9504587719521214,
    "spec_top1": 0.7916666666666666,
    "spec_top3": 0.9583333333333334
   },
   "dev": {
    "cat_map": 0.7786263627759495,
    "cat_p5": 0.85,
    "cat_p10": 0.75,
    "class_map": 0.6880803109400951,
    "class_top3": 0.9193548387096774,
    "tag_acc": 0.6048387096774194,
    "tag_mrr": 0.7341978621034598,
    "pooled_auc": 0.9716114837671049,
    "pooled_auc_strict": 0.9579487544483986,
    "spec_top1": 0.9,
    "spec_top3": 1.0
   },
   "test": {
    "cat_map": 0.8002964532472743,
    "cat_p5": 0.8,
    "cat_p10": 0.7375,
    "class_map": 0.6358470829434432,
    "class_top3": 0.8448275862068966,
    "tag_acc": 0.5684210526315789,
    "tag_mrr": 0.69375877533923,
    "pooled_auc": 0.9572817904806395,
    "pooled_auc_strict": 0.9407392148596686,
    "spec_top1": 0.7619047619047619,
    "spec_top3": 0.9047619047619048
   }
  },
  {
   "pad": "spaced",
   "templates": "t5",
   "method": "term_sd",
   "all": {
    "cat_map": 0.7403152944695368,
    "cat_p5": 0.875,
    "cat_p10": 0.8375,
    "class_map": 0.6306648972281436,
    "class_top3": 0.8857142857142857,
    "tag_acc": 0.5799086757990868,
    "tag_mrr": 0.7094754522618477,
    "pooled_auc": 0.9510448890155215,
    "pooled_auc_strict": 0.9370711400638849,
    "spec_top1": 0.8333333333333334,
    "spec_top3": 0.9583333333333334
   },
   "dev": {
    "cat_map": 0.7288654463711456,
    "cat_p5": 0.85,
    "cat_p10": 0.7,
    "class_map": 0.6623461312956833,
    "class_top3": 0.8387096774193549,
    "tag_acc": 0.5967741935483871,
    "tag_mrr": 0.7250974680232269,
    "pooled_auc": 0.9583783203649048,
    "pooled_auc_strict": 0.9453580071174377,
    "spec_top1": 0.8,
    "spec_top3": 0.9
   },
   "test": {
    "cat_map": 0.7339504300278596,
    "cat_p5": 0.775,
    "cat_p10": 0.675,
    "class_map": 0.644370763116193,
    "class_top3": 0.8103448275862069,
    "tag_acc": 0.5578947368421052,
    "tag_mrr": 0.6890846106364683,
    "pooled_auc": 0.9418727283666182,
    "pooled_auc_strict": 0.9265998863041173,
    "spec_top1": 0.7142857142857143,
    "spec_top3": 0.9523809523809523
   }
  },
  {
   "pad": "spaced",
   "templates": "t5",
   "method": "both_sd",
   "all": {
    "cat_map": 0.773238958326166,
    "cat_p5": 0.85,
    "cat_p10": 0.8,
    "class_map": 0.634303067677147,
    "class_top3": 0.8857142857142857,
    "tag_acc": 0.5799086757990868,
    "tag_mrr": 0.7094754522618477,
    "pooled_auc": 0.9635615095517204,
    "pooled_auc_strict": 0.9482228601031503,
    "spec_top1": 0.875,
    "spec_top3": 0.9583333333333334
   },
   "dev": {
    "cat_map": 0.7612631638485738,
    "cat_p5": 0.8,
    "cat_p10": 0.7375,
    "class_map": 0.6871950955434535,
    "class_top3": 0.9354838709677419,
    "tag_acc": 0.5967741935483871,
    "tag_mrr": 0.7250974680232269,
    "pooled_auc": 0.9690845183793936,
    "pooled_auc_strict": 0.9550320284697509,
    "spec_top1": 0.9,
    "spec_top3": 1.0
   },
   "test": {
    "cat_map": 0.7968612828098313,
    "cat_p5": 0.775,
    "cat_p10": 0.7374999999999999,
    "class_map": 0.6471411183926385,
    "class_top3": 0.8275862068965517,
    "tag_acc": 0.5578947368421052,
    "tag_mrr": 0.6890846106364683,
    "pooled_auc": 0.9563881781279087,
    "pooled_auc_strict": 0.9393547439287676,
    "spec_top1": 0.8571428571428571,
    "spec_top3": 0.9047619047619048
   }
  }
 ],
 "thresholds": {
  "rule": "on the 219 tuning files (lenient labels), 0.5 z grid, top-down: strong = lowest t with LR(z >= t) >= 100; plausible = lowest t below it with band LR >= 20; weak = lowest t below that with band LR >= 5",
  "grid_lenient": [
   {
    "z": 0.0,
    "tp": 438,
    "fp": 5459,
    "recall": 0.9865,
    "fpr": 0.41107,
    "lr": 2.4,
    "precision": 0.0743
   },
   {
    "z": 0.25,
    "tp": 431,
    "fp": 4270,
    "recall": 0.9707,
    "fpr": 0.32154,
    "lr": 3.0,
    "precision": 0.0917
   },
   {
    "z": 0.5,
    "tp": 424,
    "fp": 3314,
    "recall": 0.955,
    "fpr": 0.24955,
    "lr": 3.8,
    "precision": 0.1134
   },
   {
    "z": 0.75,
    "tp": 419,
    "fp": 2477,
    "recall": 0.9437,
    "fpr": 0.18652,
    "lr": 5.1,
    "precision": 0.1447
   },
   {
    "z": 1.0,
    "tp": 411,
    "fp": 1765,
    "recall": 0.9257,
    "fpr": 0.13291,
    "lr": 7.0,
    "precision": 0.1889
   },
   {
    "z": 1.25,
    "tp": 400,
    "fp": 1214,
    "recall": 0.9009,
    "fpr": 0.09142,
    "lr": 9.9,
    "precision": 0.2478
   },
   {
    "z": 1.5,
    "tp": 388,
    "fp": 853,
    "recall": 0.8739,
    "fpr": 0.06423,
    "lr": 13.6,
    "precision": 0.3127
   },
   {
    "z": 1.75,
    "tp": 380,
    "fp": 567,
    "recall": 0.8559,
    "fpr": 0.0427,
    "lr": 20.0,
    "precision": 0.4013
   },
   {
    "z": 2.0,
    "tp": 365,
    "fp": 359,
    "recall": 0.8221,
    "fpr": 0.02703,
    "lr": 30.4,
    "precision": 0.5041
   },
   {
    "z": 2.25,
    "tp": 345,
    "fp": 235,
    "recall": 0.777,
    "fpr": 0.0177,
    "lr": 43.9,
    "precision": 0.5948
   },
   {
    "z": 2.5,
    "tp": 309,
    "fp": 123,
    "recall": 0.6959,
    "fpr": 0.00926,
    "lr": 75.1,
    "precision": 0.7153
   },
   {
    "z": 2.75,
    "tp": 279,
    "fp": 76,
    "recall": 0.6284,
    "fpr": 0.00572,
    "lr": 109.8,
    "precision": 0.7859
   },
   {
    "z": 3.0,
    "tp": 228,
    "fp": 40,
    "recall": 0.5135,
    "fpr": 0.00301,
    "lr": 170.5,
    "precision": 0.8507
   },
   {
    "z": 3.25,
    "tp": 193,
    "fp": 24,
    "recall": 0.4347,
    "fpr": 0.00181,
    "lr": 240.5,
    "precision": 0.8894
   },
   {
    "z": 3.5,
    "tp": 157,
    "fp": 11,
    "recall": 0.3536,
    "fpr": 0.00083,
    "lr": 426.9,
    "precision": 0.9345
   },
   {
    "z": 3.75,
    "tp": 132,
    "fp": 9,
    "recall": 0.2973,
    "fpr": 0.00068,
    "lr": 438.7,
    "precision": 0.9362
   },
   {
    "z": 4.0,
    "tp": 101,
    "fp": 5,
    "recall": 0.2275,
    "fpr": 0.00038,
    "lr": 604.2,
    "precision": 0.9528
   },
   {
    "z": 4.25,
    "tp": 72,
    "fp": 4,
    "recall": 0.1622,
    "fpr": 0.0003,
    "lr": 538.4,
    "precision": 0.9474
   },
   {
    "z": 4.5,
    "tp": 46,
    "fp": 2,
    "recall": 0.1036,
    "fpr": 0.00015,
    "lr": 687.9,
    "precision": 0.9583
   },
   {
    "z": 4.75,
    "tp": 31,
    "fp": 2,
    "recall": 0.0698,
    "fpr": 0.00015,
    "lr": 463.6,
    "precision": 0.9394
   },
   {
    "z": 5.0,
    "tp": 19,
    "fp": 0,
    "recall": 0.0428,
    "fpr": 0.0,
    "lr": null,
    "precision": 1.0
   },
   {
    "z": 5.25,
    "tp": 9,
    "fp": 0,
    "recall": 0.0203,
    "fpr": 0.0,
    "lr": null,
    "precision": 1.0
   },
   {
    "z": 5.5,
    "tp": 5,
    "fp": 0,
    "recall": 0.0113,
    "fpr": 0.0,
    "lr": null,
    "precision": 1.0
   },
   {
    "z": 5.75,
    "tp": 1,
    "fp": 0,
    "recall": 0.0023,
    "fpr": 0.0,
    "lr": null,
    "precision": 1.0
   },
   {
    "z": 6.0,
    "tp": 0,
    "fp": 0,
    "recall": 0.0,
    "fpr": 0.0,
    "lr": null,
    "precision": 0.0
   }
  ],
  "grid_strict": [
   {
    "z": 0.0,
    "tp": 438,
    "fp": 6812,
    "recall": 0.9865,
    "fpr": 0.45761,
    "lr": 2.2,
    "precision": 0.0604
   },
   {
    "z": 0.25,
    "tp": 431,
    "fp": 5548,
    "recall": 0.9707,
    "fpr": 0.3727,
    "lr": 2.6,
    "precision": 0.0721
   },
   {
    "z": 0.5,
    "tp": 424,
    "fp": 4504,
    "recall": 0.955,
    "fpr": 0.30257,
    "lr": 3.2,
    "precision": 0.086
   },
   {
    "z": 0.75,
    "tp": 419,
    "fp": 3548,
    "recall": 0.9437,
    "fpr": 0.23834,
    "lr": 4.0,
    "precision": 0.1056
   },
   {
    "z": 1.0,
    "tp": 411,
    "fp": 2718,
    "recall": 0.9257,
    "fpr": 0.18259,
    "lr": 5.1,
    "precision": 0.1314
   },
   {
    "z": 1.25,
    "tp": 400,
    "fp": 2056,
    "recall": 0.9009,
    "fpr": 0.13812,
    "lr": 6.5,
    "precision": 0.1629
   },
   {
    "z": 1.5,
    "tp": 388,
    "fp": 1564,
    "recall": 0.8739,
    "fpr": 0.10507,
    "lr": 8.3,
    "precision": 0.1988
   },
   {
    "z": 1.75,
    "tp": 380,
    "fp": 1174,
    "recall": 0.8559,
    "fpr": 0.07887,
    "lr": 10.9,
    "precision": 0.2445
   },
   {
    "z": 2.0,
    "tp": 365,
    "fp": 870,
    "recall": 0.8221,
    "fpr": 0.05844,
    "lr": 14.1,
    "precision": 0.2955
   },
   {
    "z": 2.25,
    "tp": 345,
    "fp": 645,
    "recall": 0.777,
    "fpr": 0.04333,
    "lr": 17.9,
    "precision": 0.3485
   },
   {
    "z": 2.5,
    "tp": 309,
    "fp": 415,
    "recall": 0.6959,
    "fpr": 0.02788,
    "lr": 25.0,
    "precision": 0.4268
   },
   {
    "z": 2.75,
    "tp": 279,
    "fp": 281,
    "recall": 0.6284,
    "fpr": 0.01888,
    "lr": 33.3,
    "precision": 0.4982
   },
   {
    "z": 3.0,
    "tp": 228,
    "fp": 190,
    "recall": 0.5135,
    "fpr": 0.01276,
    "lr": 40.2,
    "precision": 0.5455
   },
   {
    "z": 3.25,
    "tp": 193,
    "fp": 119,
    "recall": 0.4347,
    "fpr": 0.00799,
    "lr": 54.4,
    "precision": 0.6186
   },
   {
    "z": 3.5,
    "tp": 157,
    "fp": 72,
    "recall": 0.3536,
    "fpr": 0.00484,
    "lr": 73.1,
    "precision": 0.6856
   },
   {
    "z": 3.75,
    "tp": 132,
    "fp": 54,
    "recall": 0.2973,
    "fpr": 0.00363,
    "lr": 82.0,
    "precision": 0.7097
   },
   {
    "z": 4.0,
    "tp": 101,
    "fp": 37,
    "recall": 0.2275,
    "fpr": 0.00249,
    "lr": 91.5,
    "precision": 0.7319
   },
   {
    "z": 4.25,
    "tp": 72,
    "fp": 26,
    "recall": 0.1622,
    "fpr": 0.00175,
    "lr": 92.8,
    "precision": 0.7347
   },
   {
    "z": 4.5,
    "tp": 46,
    "fp": 15,
    "recall": 0.1036,
    "fpr": 0.00101,
    "lr": 102.8,
    "precision": 0.7541
   },
   {
    "z": 4.75,
    "tp": 31,
    "fp": 10,
    "recall": 0.0698,
    "fpr": 0.00067,
    "lr": 103.9,
    "precision": 0.7561
   },
   {
    "z": 5.0,
    "tp": 19,
    "fp": 4,
    "recall": 0.0428,
    "fpr": 0.00027,
    "lr": 159.3,
    "precision": 0.8261
   },
   {
    "z": 5.25,
    "tp": 9,
    "fp": 3,
    "recall": 0.0203,
    "fpr": 0.0002,
    "lr": 100.6,
    "precision": 0.75
   },
   {
    "z": 5.5,
    "tp": 5,
    "fp": 1,
    "recall": 0.0113,
    "fpr": 7e-05,
    "lr": 167.6,
    "precision": 0.8333
   },
   {
    "z": 5.75,
    "tp": 1,
    "fp": 1,
    "recall": 0.0023,
    "fpr": 7e-05,
    "lr": 33.5,
    "precision": 0.5
   },
   {
    "z": 6.0,
    "tp": 0,
    "fp": 0,
    "recall": 0.0,
    "fpr": 0.0,
    "lr": null,
    "precision": 0.0
   }
  ],
  "picked": {
   "strong": 3.0,
   "plausible": 2.5,
   "weak": 2.0
  },
  "shipped": {
   "strong": 3.0,
   "plausible": 2.5,
   "weak": 2.0
  },
  "bands_tuning": {
   "lenient": {
    "n_pos": 444,
    "n_neg": 13280,
    "base_rate": 0.0324,
    "strong": {
     "z_range": [
      3.0,
      null
     ],
     "tp": 228,
     "fp": 40,
     "lr": 170.49,
     "precision": 0.851,
     "precision_even_odds": 0.994,
     "recall_at_or_above": 0.514
    },
    "plausible": {
     "z_range": [
      2.5,
      3.0
     ],
     "tp": 81,
     "fp": 83,
     "lr": 29.19,
     "precision": 0.494,
     "precision_even_odds": 0.967,
     "recall_at_or_above": 0.696
    },
    "weak": {
     "z_range": [
      2.0,
      2.5
     ],
     "tp": 56,
     "fp": 236,
     "lr": 7.1,
     "precision": 0.192,
     "precision_even_odds": 0.877,
     "recall_at_or_above": 0.822
    },
    "no match": {
     "z_range": [
      null,
      2.0
     ],
     "tp": 79,
     "fp": 12921,
     "lr": 0.18,
     "precision": 0.006,
     "precision_even_odds": 0.155,
     "recall_at_or_above": 1.0
    }
   },
   "strict": {
    "n_pos": 444,
    "n_neg": 14886,
    "base_rate": 0.029,
    "strong": {
     "z_range": [
      3.0,
      null
     ],
     "tp": 228,
     "fp": 190,
     "lr": 40.23,
     "precision": 0.545,
     "precision_even_odds": 0.976,
     "recall_at_or_above": 0.514
    },
    "plausible": {
     "z_range": [
      2.5,
      3.0
     ],
     "tp": 81,
     "fp": 225,
     "lr": 12.07,
     "precision": 0.265,
     "precision_even_odds": 0.923,
     "recall_at_or_above": 0.696
    },
    "weak": {
     "z_range": [
      2.0,
      2.5
     ],
     "tp": 56,
     "fp": 455,
     "lr": 4.13,
     "precision": 0.11,
     "precision_even_odds": 0.805,
     "recall_at_or_above": 0.822
    },
    "no match": {
     "z_range": [
      null,
      2.0
     ],
     "tp": 79,
     "fp": 14016,
     "lr": 0.19,
     "precision": 0.006,
     "precision_even_odds": 0.159,
     "recall_at_or_above": 1.0
    }
   }
  }
 },
 "reference_sensitivity": {
  "folder-balanced (used)": {
   "dev": {
    "cat_map": 0.777586606292167,
    "cat_p5": 0.85,
    "cat_p10": 0.75,
    "class_map": 0.6779892152094823,
    "class_top3": 0.8870967741935484,
    "tag_acc": 0.6008064516129032,
    "tag_mrr": 0.7313790616536111,
    "pooled_auc": 0.9731773544405689,
    "pooled_auc_strict": 0.9595833926453143,
    "spec_top1": 0.9,
    "spec_top3": 1.0
   },
   "test": {
    "cat_map": 0.7932570388183038,
    "cat_p5": 0.825,
    "cat_p10": 0.7125,
    "class_map": 0.6390066424489291,
    "class_top3": 0.8793103448275862,
    "tag_acc": 0.5526315789473685,
    "tag_mrr": 0.6876431706014001,
    "pooled_auc": 0.9558856317750858,
    "pooled_auc_strict": 0.9393954636620294,
    "spec_top1": 0.7619047619047619,
    "spec_top3": 0.9047619047619048
   }
  },
  "uniform over files": {
   "dev": {
    "cat_map": 0.7718379693024295,
    "cat_p5": 0.85,
    "cat_p10": 0.75,
    "class_map": 0.6733486372216337,
    "class_top3": 0.8870967741935484,
    "tag_acc": 0.6048387096774194,
    "tag_mrr": 0.7347743311927373,
    "pooled_auc": 0.9716329487523477,
    "pooled_auc_strict": 0.9580948991696323,
    "spec_top1": 0.9,
    "spec_top3": 1.0
   },
   "test": {
    "cat_map": 0.7943198773256326,
    "cat_p5": 0.825,
    "cat_p10": 0.7125,
    "class_map": 0.6435487807470288,
    "class_top3": 0.896551724137931,
    "tag_acc": 0.5421052631578948,
    "tag_mrr": 0.6877697542124378,
    "pooled_auc": 0.9566429903631428,
    "pooled_auc_strict": 0.9400701337523474,
    "spec_top1": 0.7619047619047619,
    "spec_top3": 0.9047619047619048
   }
  },
  "without music-like refs": {
   "dev": {
    "cat_map": 0.7799268851074638,
    "cat_p5": 0.875,
    "cat_p10": 0.7749999999999999,
    "class_map": 0.6769617486987801,
    "class_top3": 0.8870967741935484,
    "tag_acc": 0.6169354838709677,
    "tag_mrr": 0.7386258398859205,
    "pooled_auc": 0.9738985779447277,
    "pooled_auc_strict": 0.9605219454329774,
    "spec_top1": 0.8,
    "spec_top3": 1.0
   },
   "test": {
    "cat_map": 0.7712286243312736,
    "cat_p5": 0.8,
    "cat_p10": 0.7125,
    "class_map": 0.6346012267444308,
    "class_top3": 0.8448275862068966,
    "tag_acc": 0.5631578947368421,
    "tag_mrr": 0.6975672928451159,
    "pooled_auc": 0.9592043841860696,
    "pooled_auc_strict": 0.9430243104791834,
    "spec_top1": 0.8095238095238095,
    "spec_top3": 0.9523809523809523
   }
  },
  "transductive (tuning set itself)": {
   "dev": {
    "cat_map": 0.7728621385713701,
    "cat_p5": 0.875,
    "cat_p10": 0.7625,
    "class_map": 0.6770566888086154,
    "class_top3": 0.9193548387096774,
    "tag_acc": 0.6290322580645161,
    "tag_mrr": 0.7471394689539851,
    "pooled_auc": 0.9737123691977462,
    "pooled_auc_strict": 0.9601024911032029,
    "spec_top1": 0.85,
    "spec_top3": 1.0
   },
   "test": {
    "cat_map": 0.7940628661543965,
    "cat_p5": 0.8,
    "cat_p10": 0.7125,
    "class_map": 0.6287646366614319,
    "class_top3": 0.8448275862068966,
    "tag_acc": 0.5736842105263158,
    "tag_mrr": 0.7073285970723643,
    "pooled_auc": 0.9612526144797052,
    "pooled_auc_strict": 0.945261500530155,
    "spec_top1": 0.8095238095238095,
    "spec_top3": 0.9047619047619048
   }
  },
  "no term bias": {
   "dev": {
    "cat_map": 0.7706841705015869,
    "cat_p5": 0.8999999999999999,
    "cat_p10": 0.7375,
    "class_map": 0.6757453275548259,
    "class_top3": 0.9032258064516129,
    "tag_acc": 0.6209677419354839,
    "tag_mrr": 0.7338462142037805,
    "pooled_auc": 0.9712261872819963,
    "pooled_auc_strict": 0.9565115065243179,
    "spec_top1": 0.85,
    "spec_top3": 1.0
   },
   "test": {
    "cat_map": 0.7699222515537358,
    "cat_p5": 0.8,
    "cat_p10": 0.7125,
    "class_map": 0.6454645266686625,
    "class_top3": 0.8620689655172413,
    "tag_acc": 0.5578947368421052,
    "tag_mrr": 0.6902199626875211,
    "pooled_auc": 0.9532366462462972,
    "pooled_auc_strict": 0.9373035871689726,
    "spec_top1": 0.8095238095238095,
    "spec_top3": 0.9047619047619048
   }
  }
 },
 "term_bias": {
  "highest": [
   [
    "a professionally mixed and mastered music track",
    0.182
   ],
   [
    "a high quality studio recording",
    0.172
   ],
   [
    "a glockenspiel",
    0.124
   ],
   [
    "romantic music",
    0.114
   ],
   [
    "upbeat corporate pop music",
    0.113
   ],
   [
    "a dry close sound",
    0.112
   ],
   [
    "a robotic synthetic sound",
    0.11
   ],
   [
    "an enter key press",
    0.107
   ],
   [
    "warm and cozy music",
    0.106
   ],
   [
    "tech house music",
    0.104
   ]
  ],
  "lowest": [
   [
    "a man speaking",
    -0.119
   ],
   [
    "people talking in the background",
    -0.12
   ],
   [
    "a man shouting",
    -0.124
   ],
   [
    "a crowd talking",
    -0.125
   ],
   [
    "people laughing",
    -0.13
   ]
  ]
 },
 "seconds": 66.7
}