Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

8.5 KB
Raw Blame
{
  "bins": 15,
  "models": {
    "jev": {
      "accuracy": 0.9948,
      "answer_confidence": {
        "acc_at_50_coverage": 1.0,
        "auroc": 1.0,
        "ece": 0.0274,
        "gate_at_0.85": {
          "accuracy_of_automated": 1.0,
          "automated": 181,
          "automated_share": 0.9427,
          "threshold": 0.85
        },
        "mean_confidence": 0.9674,
        "reliability": [
          {
            "accuracy": 0.6667,
            "bin": "0.47-0.53",
            "gap": -0.1533,
            "mean_confidence": 0.5133,
            "n": 3
          },
          {
            "accuracy": 1.0,
            "bin": "0.73-0.80",
            "gap": -0.234,
            "mean_confidence": 0.766,
            "n": 5
          },
          {
            "accuracy": 1.0,
            "bin": "0.80-0.87",
            "gap": -0.1633,
            "mean_confidence": 0.8367,
            "n": 6
          },
          {
            "accuracy": 1.0,
            "bin": "0.87-0.93",
            "gap": -0.0918,
            "mean_confidence": 0.9082,
            "n": 11
          },
          {
            "accuracy": 1.0,
            "bin": "0.93-1.00",
            "gap": -0.0098,
            "mean_confidence": 0.9902,
            "n": 167
          }
        ]
      },
      "entropy_confidence": {
        "acc_at_50_coverage": 1.0,
        "auroc": 1.0,
        "ece": 0.0678,
        "gate_at_0.85": {
          "accuracy_of_automated": 1.0,
          "automated": 160,
          "automated_share": 0.8333,
          "threshold": 0.85
        },
        "mean_confidence": 0.9269,
        "reliability": [
          {
            "accuracy": 0.5,
            "bin": "0.33-0.40",
            "gap": -0.1168,
            "mean_confidence": 0.3832,
            "n": 2
          },
          {
            "accuracy": 1.0,
            "bin": "0.40-0.47",
            "gap": -0.5792,
            "mean_confidence": 0.4208,
            "n": 1
          },
          {
            "accuracy": 1.0,
            "bin": "0.53-0.60",
            "gap": -0.4056,
            "mean_confidence": 0.5944,
            "n": 2
          },
          {
            "accuracy": 1.0,
            "bin": "0.60-0.67",
            "gap": -0.368,
            "mean_confidence": 0.632,
            "n": 5
          },
          {
            "accuracy": 1.0,
            "bin": "0.67-0.73",
            "gap": -0.3004,
            "mean_confidence": 0.6996,
            "n": 6
          },
          {
            "accuracy": 1.0,
            "bin": "0.73-0.80",
            "gap": -0.2503,
            "mean_confidence": 0.7497,
            "n": 7
          },
          {
            "accuracy": 1.0,
            "bin": "0.80-0.87",
            "gap": -0.1727,
            "mean_confidence": 0.8273,
            "n": 12
          },
          {
            "accuracy": 1.0,
            "bin": "0.87-0.93",
            "gap": -0.0901,
            "mean_confidence": 0.9099,
            "n": 28
          },
          {
            "accuracy": 1.0,
            "bin": "0.93-1.00",
            "gap": -0.011,
            "mean_confidence": 0.989,
            "n": 129
          }
        ]
      },
      "invariants_checked": false,
      "n": 192
    },
    "laya": {
      "accuracy": 0.3125,
      "answer_confidence": {
        "acc_at_50_coverage": 0.375,
        "auroc": 0.6193,
        "ece": 0.2614,
        "gate_at_0.85": {
          "accuracy_of_automated": 0.4444,
          "automated": 27,
          "automated_share": 0.1406,
          "threshold": 0.85
        },
        "mean_confidence": 0.5739,
        "reliability": [
          {
            "accuracy": 0.0,
            "bin": "0.27-0.33",
            "gap": 0.3117,
            "mean_confidence": 0.3117,
            "n": 21
          },
          {
            "accuracy": 0.2727,
            "bin": "0.33-0.40",
            "gap": 0.096,
            "mean_confidence": 0.3687,
            "n": 33
          },
          {
            "accuracy": 0.2857,
            "bin": "0.40-0.47",
            "gap": 0.1513,
            "mean_confidence": 0.437,
            "n": 21
          },
          {
            "accuracy": 0.4545,
            "bin": "0.47-0.53",
            "gap": 0.0436,
            "mean_confidence": 0.4981,
            "n": 33
          },
          {
            "accuracy": 0.3333,
            "bin": "0.53-0.60",
            "gap": 0.2467,
            "mean_confidence": 0.58,
            "n": 9
          },
          {
            "accuracy": 0.3333,
            "bin": "0.60-0.67",
            "gap": 0.2868,
            "mean_confidence": 0.6202,
            "n": 9
          },
          {
            "accuracy": 0.0,
            "bin": "0.67-0.73",
            "gap": 0.7171,
            "mean_confidence": 0.7171,
            "n": 6
          },
          {
            "accuracy": 0.375,
            "bin": "0.73-0.80",
            "gap": 0.3899,
            "mean_confidence": 0.7649,
            "n": 24
          },
          {
            "accuracy": 0.3333,
            "bin": "0.80-0.87",
            "gap": 0.4807,
            "mean_confidence": 0.8141,
            "n": 9
          },
          {
            "accuracy": 0.6,
            "bin": "0.87-0.93",
            "gap": 0.3008,
            "mean_confidence": 0.9008,
            "n": 15
          },
          {
            "accuracy": 0.25,
            "bin": "0.93-1.00",
            "gap": 0.7135,
            "mean_confidence": 0.9635,
            "n": 12
          }
        ]
      },
      "choice_confidence_max_drift": 0.000223,
      "entropy_confidence": {
        "acc_at_50_coverage": 0.3438,
        "auroc": 0.5966,
        "ece": 0.2202,
        "gate_at_0.85": {
          "accuracy_of_automated": 0.5,
          "automated": 6,
          "automated_share": 0.0312,
          "threshold": 0.85
        },
        "mean_confidence": 0.2889,
        "reliability": [
          {
            "accuracy": 0.125,
            "bin": "0.00-0.07",
            "gap": -0.0895,
            "mean_confidence": 0.0355,
            "n": 24
          },
          {
            "accuracy": 0.25,
            "bin": "0.07-0.13",
            "gap": -0.1545,
            "mean_confidence": 0.0955,
            "n": 48
          },
          {
            "accuracy": 0.4545,
            "bin": "0.13-0.20",
            "gap": -0.2872,
            "mean_confidence": 0.1674,
            "n": 33
          },
          {
            "accuracy": 0.0,
            "bin": "0.20-0.27",
            "gap": 0.246,
            "mean_confidence": 0.246,
            "n": 6
          },
          {
            "accuracy": 0.5,
            "bin": "0.27-0.33",
            "gap": -0.2094,
            "mean_confidence": 0.2906,
            "n": 12
          },
          {
            "accuracy": 0.25,
            "bin": "0.33-0.40",
            "gap": 0.1227,
            "mean_confidence": 0.3727,
            "n": 12
          },
          {
            "accuracy": 0.25,
            "bin": "0.40-0.47",
            "gap": 0.179,
            "mean_confidence": 0.429,
            "n": 12
          },
          {
            "accuracy": 0.25,
            "bin": "0.47-0.53",
            "gap": 0.2287,
            "mean_confidence": 0.4787,
            "n": 12
          },
          {
            "accuracy": 0.5,
            "bin": "0.53-0.60",
            "gap": 0.0461,
            "mean_confidence": 0.5461,
            "n": 6
          },
          {
            "accuracy": 1.0,
            "bin": "0.60-0.67",
            "gap": -0.3658,
            "mean_confidence": 0.6342,
            "n": 3
          },
          {
            "accuracy": 0.3333,
            "bin": "0.67-0.73",
            "gap": 0.3761,
            "mean_confidence": 0.7094,
            "n": 9
          },
          {
            "accuracy": 1.0,
            "bin": "0.73-0.80",
            "gap": -0.2512,
            "mean_confidence": 0.7488,
            "n": 3
          },
          {
            "accuracy": 0.0,
            "bin": "0.80-0.87",
            "gap": 0.8253,
            "mean_confidence": 0.8253,
            "n": 6
          },
          {
            "accuracy": 0.5,
            "bin": "0.87-0.93",
            "gap": 0.4028,
            "mean_confidence": 0.9028,
            "n": 6
          }
        ]
      },
      "invariants_checked": true,
      "n": 192,
      "noul_answers": 768,
      "noul_confidence_max_drift": 0.0,
      "shipped_choice_confidence_is_entropy": true,
      "shipped_noul_confidence_is_max_p": true
    }
  },
  "source": "research/benchmarks/feishu_zh/results/v1"
}