Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

4.7 KB
Raw Blame
{
  "bench": [
    {
      "checkpoint": "english",
      "states": 1,
      "questions_per_state": 4,
      "looped_ms": 18.6,
      "batched_ms": 18.6,
      "speedup": 1.0,
      "looped_qps": 215.6,
      "batched_qps": 215.6
    },
    {
      "checkpoint": "english",
      "states": 2,
      "questions_per_state": 4,
      "looped_ms": 33.4,
      "batched_ms": 27.0,
      "speedup": 1.2,
      "looped_qps": 239.6,
      "batched_qps": 296.2
    },
    {
      "checkpoint": "english",
      "states": 4,
      "questions_per_state": 4,
      "looped_ms": 70.1,
      "batched_ms": 45.2,
      "speedup": 1.6,
      "looped_qps": 228.2,
      "batched_qps": 354.3
    },
    {
      "checkpoint": "english",
      "states": 8,
      "questions_per_state": 4,
      "looped_ms": 138.4,
      "batched_ms": 85.8,
      "speedup": 1.6,
      "looped_qps": 231.2,
      "batched_qps": 372.8
    },
    {
      "checkpoint": "english",
      "states": 16,
      "questions_per_state": 4,
      "looped_ms": 279.3,
      "batched_ms": 181.4,
      "speedup": 1.5,
      "looped_qps": 229.1,
      "batched_qps": 352.8
    },
    {
      "checkpoint": "english",
      "states": 32,
      "questions_per_state": 4,
      "looped_ms": 556.2,
      "batched_ms": 361.3,
      "speedup": 1.5,
      "looped_qps": 230.1,
      "batched_qps": 354.3
    },
    {
      "checkpoint": "english",
      "states": 64,
      "questions_per_state": 4,
      "looped_ms": 1114.1,
      "batched_ms": 705.7,
      "speedup": 1.6,
      "looped_qps": 229.8,
      "batched_qps": 362.7
    },
    {
      "checkpoint": "multilingual",
      "states": 1,
      "questions_per_state": 4,
      "looped_ms": 9.3,
      "batched_ms": 9.3,
      "speedup": 1.0,
      "looped_qps": 432.0,
      "batched_qps": 428.0
    },
    {
      "checkpoint": "multilingual",
      "states": 2,
      "questions_per_state": 4,
      "looped_ms": 17.1,
      "batched_ms": 13.1,
      "speedup": 1.3,
      "looped_qps": 469.0,
      "batched_qps": 610.8
    },
    {
      "checkpoint": "multilingual",
      "states": 4,
      "questions_per_state": 4,
      "looped_ms": 33.0,
      "batched_ms": 20.3,
      "speedup": 1.6,
      "looped_qps": 484.6,
      "batched_qps": 786.8
    },
    {
      "checkpoint": "multilingual",
      "states": 8,
      "questions_per_state": 4,
      "looped_ms": 66.4,
      "batched_ms": 36.0,
      "speedup": 1.8,
      "looped_qps": 482.0,
      "batched_qps": 888.3
    },
    {
      "checkpoint": "multilingual",
      "states": 16,
      "questions_per_state": 4,
      "looped_ms": 132.1,
      "batched_ms": 74.3,
      "speedup": 1.8,
      "looped_qps": 484.4,
      "batched_qps": 861.8
    },
    {
      "checkpoint": "multilingual",
      "states": 32,
      "questions_per_state": 4,
      "looped_ms": 265.8,
      "batched_ms": 161.7,
      "speedup": 1.6,
      "looped_qps": 481.5,
      "batched_qps": 791.6
    },
    {
      "checkpoint": "multilingual",
      "states": 64,
      "questions_per_state": 4,
      "looped_ms": 528.8,
      "batched_ms": 319.6,
      "speedup": 1.7,
      "looped_qps": 484.1,
      "batched_qps": 801.1
    },
    {
      "checkpoint": "typed-decisions",
      "states": 1,
      "questions_per_state": 4,
      "looped_ms": 18.9,
      "batched_ms": 18.7,
      "speedup": 1.0,
      "looped_qps": 211.6,
      "batched_qps": 213.6
    },
    {
      "checkpoint": "typed-decisions",
      "states": 2,
      "questions_per_state": 4,
      "looped_ms": 33.7,
      "batched_ms": 27.2,
      "speedup": 1.2,
      "looped_qps": 237.5,
      "batched_qps": 293.6
    },
    {
      "checkpoint": "typed-decisions",
      "states": 4,
      "questions_per_state": 4,
      "looped_ms": 70.4,
      "batched_ms": 45.4,
      "speedup": 1.6,
      "looped_qps": 227.4,
      "batched_qps": 352.7
    },
    {
      "checkpoint": "typed-decisions",
      "states": 8,
      "questions_per_state": 4,
      "looped_ms": 139.7,
      "batched_ms": 86.3,
      "speedup": 1.6,
      "looped_qps": 229.1,
      "batched_qps": 370.9
    },
    {
      "checkpoint": "typed-decisions",
      "states": 16,
      "questions_per_state": 4,
      "looped_ms": 281.4,
      "batched_ms": 182.4,
      "speedup": 1.5,
      "looped_qps": 227.4,
      "batched_qps": 350.9
    },
    {
      "checkpoint": "typed-decisions",
      "states": 32,
      "questions_per_state": 4,
      "looped_ms": 561.4,
      "batched_ms": 363.6,
      "speedup": 1.5,
      "looped_qps": 228.0,
      "batched_qps": 352.1
    },
    {
      "checkpoint": "typed-decisions",
      "states": 64,
      "questions_per_state": 4,
      "looped_ms": 1126.3,
      "batched_ms": 709.6,
      "speedup": 1.6,
      "looped_qps": 227.3,
      "batched_qps": 360.8
    }
  ],
  "consistency": [],
  "router": []
}