Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
{
"bench": [
{
"checkpoint": "english",
"states": 1,
"questions_per_state": 4,
"looped_ms": 18.6,
"batched_ms": 18.6,
"speedup": 1.0,
"looped_qps": 215.6,
"batched_qps": 215.6
},
{
"checkpoint": "english",
"states": 2,
"questions_per_state": 4,
"looped_ms": 33.4,
"batched_ms": 27.0,
"speedup": 1.2,
"looped_qps": 239.6,
"batched_qps": 296.2
},
{
"checkpoint": "english",
"states": 4,
"questions_per_state": 4,
"looped_ms": 70.1,
"batched_ms": 45.2,
"speedup": 1.6,
"looped_qps": 228.2,
"batched_qps": 354.3
},
{
"checkpoint": "english",
"states": 8,
"questions_per_state": 4,
"looped_ms": 138.4,
"batched_ms": 85.8,
"speedup": 1.6,
"looped_qps": 231.2,
"batched_qps": 372.8
},
{
"checkpoint": "english",
"states": 16,
"questions_per_state": 4,
"looped_ms": 279.3,
"batched_ms": 181.4,
"speedup": 1.5,
"looped_qps": 229.1,
"batched_qps": 352.8
},
{
"checkpoint": "english",
"states": 32,
"questions_per_state": 4,
"looped_ms": 556.2,
"batched_ms": 361.3,
"speedup": 1.5,
"looped_qps": 230.1,
"batched_qps": 354.3
},
{
"checkpoint": "english",
"states": 64,
"questions_per_state": 4,
"looped_ms": 1114.1,
"batched_ms": 705.7,
"speedup": 1.6,
"looped_qps": 229.8,
"batched_qps": 362.7
},
{
"checkpoint": "multilingual",
"states": 1,
"questions_per_state": 4,
"looped_ms": 9.3,
"batched_ms": 9.3,
"speedup": 1.0,
"looped_qps": 432.0,
"batched_qps": 428.0
},
{
"checkpoint": "multilingual",
"states": 2,
"questions_per_state": 4,
"looped_ms": 17.1,
"batched_ms": 13.1,
"speedup": 1.3,
"looped_qps": 469.0,
"batched_qps": 610.8
},
{
"checkpoint": "multilingual",
"states": 4,
"questions_per_state": 4,
"looped_ms": 33.0,
"batched_ms": 20.3,
"speedup": 1.6,
"looped_qps": 484.6,
"batched_qps": 786.8
},
{
"checkpoint": "multilingual",
"states": 8,
"questions_per_state": 4,
"looped_ms": 66.4,
"batched_ms": 36.0,
"speedup": 1.8,
"looped_qps": 482.0,
"batched_qps": 888.3
},
{
"checkpoint": "multilingual",
"states": 16,
"questions_per_state": 4,
"looped_ms": 132.1,
"batched_ms": 74.3,
"speedup": 1.8,
"looped_qps": 484.4,
"batched_qps": 861.8
},
{
"checkpoint": "multilingual",
"states": 32,
"questions_per_state": 4,
"looped_ms": 265.8,
"batched_ms": 161.7,
"speedup": 1.6,
"looped_qps": 481.5,
"batched_qps": 791.6
},
{
"checkpoint": "multilingual",
"states": 64,
"questions_per_state": 4,
"looped_ms": 528.8,
"batched_ms": 319.6,
"speedup": 1.7,
"looped_qps": 484.1,
"batched_qps": 801.1
},
{
"checkpoint": "typed-decisions",
"states": 1,
"questions_per_state": 4,
"looped_ms": 18.9,
"batched_ms": 18.7,
"speedup": 1.0,
"looped_qps": 211.6,
"batched_qps": 213.6
},
{
"checkpoint": "typed-decisions",
"states": 2,
"questions_per_state": 4,
"looped_ms": 33.7,
"batched_ms": 27.2,
"speedup": 1.2,
"looped_qps": 237.5,
"batched_qps": 293.6
},
{
"checkpoint": "typed-decisions",
"states": 4,
"questions_per_state": 4,
"looped_ms": 70.4,
"batched_ms": 45.4,
"speedup": 1.6,
"looped_qps": 227.4,
"batched_qps": 352.7
},
{
"checkpoint": "typed-decisions",
"states": 8,
"questions_per_state": 4,
"looped_ms": 139.7,
"batched_ms": 86.3,
"speedup": 1.6,
"looped_qps": 229.1,
"batched_qps": 370.9
},
{
"checkpoint": "typed-decisions",
"states": 16,
"questions_per_state": 4,
"looped_ms": 281.4,
"batched_ms": 182.4,
"speedup": 1.5,
"looped_qps": 227.4,
"batched_qps": 350.9
},
{
"checkpoint": "typed-decisions",
"states": 32,
"questions_per_state": 4,
"looped_ms": 561.4,
"batched_ms": 363.6,
"speedup": 1.5,
"looped_qps": 228.0,
"batched_qps": 352.1
},
{
"checkpoint": "typed-decisions",
"states": 64,
"questions_per_state": 4,
"looped_ms": 1126.3,
"batched_ms": 709.6,
"speedup": 1.6,
"looped_qps": 227.3,
"batched_qps": 360.8
}
],
"consistency": [],
"router": []
}