Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
using System.Runtime.CompilerServices;
namespace Laya;
/// <summary>
/// The routing outcome: which checkpoint, why, and what was detected. Ports Python's
/// <c>RouteDecision</c> (a <c>dict</c> subclass there; a plain record here, since the .NET SDK
/// has no need to serialise it as a raw dictionary).
/// </summary>
/// <param name="Model">The checkpoint selected.</param>
/// <param name="Reason">
/// Human-readable explanation, byte-identical to Python's (including <c>%r</c>-style quoting and
/// half-even <c>%.0f%%</c> percentage formatting).
/// </param>
/// <param name="Detection">
/// The language/script detection behind the decision, or <see langword="null"/> when the model
/// was picked by an explicit <c>model</c>/<c>task</c>/<c>lang</c> argument instead.
/// </param>
/// <param name="Workflow">
/// The typed-decisions workflow the question ids matched, if any — reported even when
/// <see cref="LayaRouterOptions.AutoTaskDetection"/> is off and so played no part in the decision.
/// </param>
public sealed record RouteDecision(LayaCheckpoint Model, string Reason, LanguageAnalysis? Detection, string? Workflow);
/// <summary>Options for a <see cref="LayaRouter"/>.</summary>
public sealed class LayaRouterOptions
{
/// <summary>
/// Template used to build each checkpoint's <see cref="LayaEngine"/>: execution provider,
/// thread counts, <see cref="LayaOptions.AllowDownload"/>, cache directory, HuggingFace
/// token/repo, <see cref="LayaOptions.LangTemperatures"/> and <see cref="LayaOptions.Calibration"/> are all taken from this template. <see cref="LayaOptions.Checkpoint"/> and
/// <see cref="LayaOptions.ModelDirectory"/> on the template are ignored — the router always
/// resolves each checkpoint through its own <see cref="LayaOptions.Checkpoint"/>-aware path
/// (<c>LAYA_ONNX_ROOT</c>, cache, or download), the same way <see cref="LayaEngine.Create"/> does.
/// </summary>
public LayaOptions EngineOptions { get; set; } = new();
/// <summary>
/// How many checkpoints stay resident at once (least-recently-used is evicted). Default 2,
/// matching Python's <c>Router(max_loaded=2)</c>, so alternating two languages does not reload
/// a checkpoint on every request. Raised automatically by <see cref="LayaRouter.Attach"/> and
/// <see cref="LayaRouter.Preload"/> to fit what they load.
/// </summary>
public int MaxLoaded { get; set; } = 2;
/// <summary>
/// Checkpoint used when a state has no letters or its Latin language is undetermined without
/// non-English evidence. Default
/// <see cref="LayaCheckpoint.Multilingual"/>, matching Python's default.
/// </summary>
// Follows the reference, which moved this to multilingual: undecided Latin text is no
// evidence of English, and multilingual leads on 50 of the 51 languages swept.
public LayaCheckpoint Default { get; set; } = LayaCheckpoint.Multilingual;
/// <summary>
/// Whether <see cref="LayaRouter.Route(object?, QuestionSet?, string?, string?, string?)"/> may pick <see cref="LayaCheckpoint.TypedDecisions"/>
/// on its own when the question ids match one of the four typed-decisions workflow
/// signatures. Default <see langword="false"/>: that checkpoint is fine-tuned on specific
/// synthetic workflows and must never be a silent default.
/// </summary>
public bool AutoTaskDetection { get; set; }
/// <summary>Whether every checkpoint should be built immediately when the router is constructed.</summary>
public bool Preload { get; set; }
/// <summary>
/// A language hint applied to every request, Python's <c>Router(lang_guess=...)</c>: a language
/// code or a callback taking the state. Checked after an explicit <c>lang</c> and a per-call hint,
/// and before the built-in detection; a hint that names no language falls through to detection.
/// See <see cref="Laya.LangGuess"/>.
/// </summary>
public LangGuess? LangGuess { get; init; }
}
/// <summary>
/// Lazily loads Laya checkpoints and sends each request to the right one. Ports Python's
/// <c>Router</c>.
/// </summary>
/// <remarks>
/// <para>
/// <code>
/// using var router = new LayaRouter();
/// router.Predict(new { message = "Mein Konto wurde zweimal belastet" }, questions); // -> multilingual
/// router.Predict(new { message = "I was charged twice" }, questions); // -> english
/// router.Predict(state, questions, model: "typed-decisions"); // explicit
/// </code>
/// </para>
/// <para>
/// Checkpoints are downloaded/built on first use. <see cref="LayaRouterOptions.MaxLoaded"/> caps
/// how many stay resident (least-recently-used is evicted). For a server or a demo, prefer
/// <see cref="LayaRouterOptions.Preload"/> or <see cref="Preload"/>: a cold load costs seconds,
/// while detection costs microseconds, so anything that alternates languages at
/// <c>MaxLoaded=1</c> reloads on every request.
/// </para>
/// <para>
/// <b>Lifetime.</b> Unlike Python, which relies on the garbage collector to keep an evicted
/// checkpoint alive until nothing references it, .NET must dispose the underlying
/// <see cref="LayaEngine"/> explicitly. The router protects itself: a checkpoint evicted or
/// unloaded while a router-driven <see cref="Predict(object, QuestionSet)"/> call is still
/// running on it is disposed only once that call returns, never mid-inference. <see cref="Load"/>,
/// however, hands back a bare reference with no such protection — if you keep using it after the
/// router has evicted it (e.g. by loading more checkpoints than <c>MaxLoaded</c> allows), it may
/// already be disposed. Route requests through <see cref="Predict(object, QuestionSet)"/>, or use
/// <see cref="Attach"/> to hand the router an instance you own and will dispose yourself.
/// </para>
/// </remarks>
public sealed class LayaRouter : ILayaLongPredictor, IDisposable
{
// Checkpoint keys people are likely to type, mapped to their canonical name. Matches
// Python's _ALIASES exactly.
private static readonly Dictionary<string, string> Aliases = new(StringComparer.Ordinal)
{
["en"] = "english", ["laya"] = "english", ["default"] = "english",
["multi"] = "multilingual", ["ml"] = "multilingual", ["laya-multilingual"] = "multilingual",
["typed"] = "typed-decisions", ["typed_decisions"] = "typed-decisions",
["laya-typed-decisions"] = "typed-decisions", ["decisions"] = "typed-decisions",
};
// Question-id signatures of the four typed-decisions workflows, used only when
// AutoTaskDetection is enabled. Order matches Python's _TYPED_DECISION_WORKFLOWS.
private static readonly (string Workflow, string[] Ids)[] TypedDecisionWorkflows =
[
("agent_trace_observability", ["action", "needs_review", "outcome", "risk", "urgency"]),
("customer_service", ["action", "category", "churn_risk", "needs_human", "urgency"]),
("invoice_processing", ["discrepancy_severity", "disposition", "duplicate", "matches_order", "urgency"]),
("security_incidents", ["credential_compromise", "disposition", "severity", "true_positive", "urgency"]),
];
private static readonly LayaCheckpoint[] AllCheckpoints =
[LayaCheckpoint.English, LayaCheckpoint.Multilingual, LayaCheckpoint.TypedDecisions];
private sealed class Slot
{
public required ILayaPredictor Predictor { get; init; }
public int Leases;
public bool Retired;
// False for an instance handed in via Attach: the router does not own its lifetime and
// must never dispose it.
public required bool OwnsDisposal { get; init; }
}
private readonly Func<LayaCheckpoint, ILayaPredictor> _engineFactory;
private readonly Lock _lock = new();
private readonly Dictionary<LayaCheckpoint, Slot> _slots = new();
private readonly List<LayaCheckpoint> _order = []; // least-recently-used first
private int _maxLoaded;
private bool _disposed;
/// <summary>Checkpoint used when a state has no letters at all.</summary>
public LayaCheckpoint Default { get; }
/// <summary>
/// Whether <see cref="Route(object?, QuestionSet?, string?, string?, string?)"/> may pick <see cref="LayaCheckpoint.TypedDecisions"/> on its
/// own when the question ids match a typed-decisions workflow signature.
/// </summary>
public bool AutoTaskDetection { get; }
/// <summary>
/// The language hint installed with <see cref="LayaRouterOptions.LangGuess"/>, applied to every
/// request after a per-call hint, or <see langword="null"/>.
/// </summary>
public LangGuess? LangGuess { get; }
/// <summary>Create a router that builds real <see cref="LayaEngine"/> instances on demand.</summary>
public LayaRouter(LayaRouterOptions? options = null)
: this(options, BuildDefaultFactory((options ?? new LayaRouterOptions()).EngineOptions))
{
}
/// <summary>
/// Test-only seam: builds checkpoints via <paramref name="engineFactory"/> instead of
/// <see cref="LayaEngine.Create"/>, so LRU/attach/preload/unload behaviour can be exercised
/// with fake predictors and no ONNX weights.
/// </summary>
internal LayaRouter(LayaRouterOptions? options, Func<LayaCheckpoint, ILayaPredictor> engineFactory)
{
ArgumentNullException.ThrowIfNull(engineFactory);
options ??= new LayaRouterOptions();
_engineFactory = engineFactory;
_maxLoaded = Math.Max(1, options.MaxLoaded);
Default = options.Default;
AutoTaskDetection = options.AutoTaskDetection;
LangGuess = options.LangGuess;
if (options.Preload) Preload();
}
private static Func<LayaCheckpoint, ILayaPredictor> BuildDefaultFactory(LayaOptions template) => checkpoint =>
LayaEngine.Create(EngineOptions(template, checkpoint));
/// <summary>The options of the engine the router builds for <paramref name="checkpoint"/> from its <paramref name="template"/>.</summary>
internal static LayaOptions EngineOptions(LayaOptions template, LayaCheckpoint checkpoint) =>
new()
{
Checkpoint = checkpoint,
ExecutionProvider = template.ExecutionProvider,
IntraOpThreads = template.IntraOpThreads,
InterOpThreads = template.InterOpThreads,
AllowDownload = template.AllowDownload,
HuggingFaceRepo = template.HuggingFaceRepo,
HuggingFaceToken = template.HuggingFaceToken,
CacheDirectory = template.CacheDirectory,
DownloadProgress = template.DownloadProgress,
// agent_kwargs reaches every checkpoint the router builds (router.py:994); each engine resolves it itself.
LangTemperatures = template.LangTemperatures,
Calibration = template.Calibration,
};
// ── name resolution ──────────────────────────────────────────────────────
/// <summary>
/// Resolve a checkpoint name or alias (case-insensitive) to its <see cref="LayaCheckpoint"/>.
/// </summary>
/// <exception cref="ArgumentException">If <paramref name="name"/> matches no checkpoint or alias.</exception>
public static LayaCheckpoint NormaliseName(string name)
{
ArgumentNullException.ThrowIfNull(name);
var key = name.Trim().ToLowerInvariant();
if (Aliases.TryGetValue(key, out var mapped)) key = mapped;
return key switch
{
"english" => LayaCheckpoint.English,
"multilingual" => LayaCheckpoint.Multilingual,
"typed-decisions" => LayaCheckpoint.TypedDecisions,
// Python's text, list reprs included (router.py:187): sorted(DEFAULT_MODELS), sorted(_ALIASES).
_ => throw new ArgumentException(
$"unknown model {PythonStr.Repr(name)}; choose one of {PyList(["english", "multilingual", "typed-decisions"])} "
+ $"(or an alias: {PyList(Aliases.Keys)})"),
};
}
private static string PyList(IEnumerable<string> items) =>
"[" + string.Join(", ", items.OrderBy(k => k, StringComparer.Ordinal).Select(PythonStr.Repr)) + "]";
/// <summary>
/// Name of the typed-decisions workflow whose question ids these are, else
/// <see langword="null"/>. Requires an exact id-set match, so an unrelated schema that
/// happens to contain <c>"urgency"</c> is never captured.
/// </summary>
public static string? MatchTypedDecisionsWorkflow(QuestionSet? questions)
{
var ids = new HashSet<string>(questions?.Ids ?? [], StringComparer.Ordinal);
foreach (var (workflow, signature) in TypedDecisionWorkflows)
if (ids.SetEquals(signature)) return workflow;
return null;
}
// ── routing ───────────────────────────────────────────────────────────────
/// <summary>
/// Decide which checkpoint to use, without loading or running anything.
/// </summary>
/// <remarks>
/// Precedence: explicit <paramref name="model"/> > explicit <paramref name="task"/> >
/// detected workflow (opt-in via <see cref="AutoTaskDetection"/>) > explicit
/// <paramref name="lang"/> > the hint installed with <see cref="LayaRouterOptions.LangGuess"/>
/// > detected script/language > <see cref="Default"/>. An explicit <paramref name="lang"/>
/// decides only when its code names a language (blank, <c>C</c>, <c>POSIX</c>, <c>und</c> ... fall through).
/// </remarks>
public RouteDecision Route(
object? state, QuestionSet? questions = null,
string? model = null, string? task = null, string? lang = null)
=> Route(state, questions, model, task, lang, null);
/// <summary>
/// <see cref="Route(object?, QuestionSet?, string?, string?, string?)"/> with Python's per-call
/// <c>lang_guess</c> hint: checked after an explicit <paramref name="lang"/> and before the
/// built-in detection, ahead of the hint installed with <see cref="LayaRouterOptions.LangGuess"/>.
/// </summary>
public RouteDecision Route(
object? state, QuestionSet? questions,
string? model, string? task, string? lang, LangGuess? langGuess)
{
if (model is not null)
return new RouteDecision(NormaliseName(model), $"explicit model={PythonStr.Repr(model)}", null, null);
if (task is not null)
{
var normalisedTaskSpelling = task.ToLowerInvariant().Replace("-", "_");
var key = NormaliseName(normalisedTaskSpelling == "typed_decisions" ? "typed-decisions" : task);
return new RouteDecision(key, $"explicit task={PythonStr.Repr(task)}", null, null);
}
var workflow = MatchTypedDecisionsWorkflow(questions);
if (workflow is not null && AutoTaskDetection)
return new RouteDecision(LayaCheckpoint.TypedDecisions,
$"question ids match the {PythonStr.Repr(workflow)} typed-decisions workflow", null, workflow);
if (lang is not null)
{
// An explicit `lang` is decisive only when the code names a language. Blank or whitespace
// resolves to no usable hint, so it falls through to lang_guess/detection (router.py:1242).
var resolved = LangGuess.EnglishFromCode(lang);
if (resolved is { } isEnglish)
return new RouteDecision(isEnglish ? LayaCheckpoint.English : LayaCheckpoint.Multilingual,
$"explicit lang={PythonStr.Repr(lang)}", null, workflow);
}
// Caller-supplied hint, per-call first then the one installed on the router (router.py:1254).
foreach (var (source, hint) in new[] { ("lang_guess", langGuess), ("Router(lang_guess=...)", LangGuess) })
{
if (hint?.Resolve(state) is { } guessEnglish)
return new RouteDecision(guessEnglish ? LayaCheckpoint.English : LayaCheckpoint.Multilingual,
$"{source}: the caller identified this as {(guessEnglish ? "English" : "non-English")} text",
null, workflow);
}
var det = LanguageDetection.Analyse(state);
LayaCheckpoint model2;
string reason;
if (det.Script == "unknown")
{
model2 = Default;
reason = $"no letters detected in state; using default ({CheckpointName(model2)})";
}
else if (det.Script != "latin")
{
model2 = LayaCheckpoint.Multilingual;
reason = $"non-Latin script ({det.Script}, {FormatPercent0(det.NonLatinFraction)}% of letters); "
+ "the English checkpoint cannot read it";
}
else if (!det.IsEnglish)
{
model2 = LayaCheckpoint.Multilingual;
reason = det.MixedSegment is { Length: > 0 } mixed
? $"Latin script, mostly English, but a line or field reads as {PythonStr.Repr(det.Language!)} "
+ $"({PythonStr.Repr(LanguageDetection.SliceCodePoints(mixed, 60))}); the English checkpoint cannot read it"
: det.Language is { } identified
? $"Latin script but language looks like {PythonStr.Repr(identified)}, not English"
// Unidentified Latin-script language: routed on the non-English letters alone,
// because no stopword list here covers it.
: $"Latin script, language not identified but {FormatPercent0(det.DiacriticRate)}% "
+ "non-English letters; not safe for the English checkpoint";
}
else if (det.LanguageUndecided)
{
model2 = Default;
reason = $"Latin script, language not identified and no non-English letters; using default ({CheckpointName(model2)})";
}
else
{
model2 = LayaCheckpoint.English;
reason = "English Latin text";
}
return new RouteDecision(model2, reason, det, workflow);
}
private static string CheckpointName(LayaCheckpoint c) => LayaOptions.CheckpointSubfolder(c);
/// <summary>Python's <c>%.0f%%</c>: half-even rounding to the nearest integer percent.</summary>
private static string FormatPercent0(double fraction) =>
Math.Round(100.0 * fraction, MidpointRounding.ToEven).ToString("0", System.Globalization.CultureInfo.InvariantCulture);
// ── loading ───────────────────────────────────────────────────────────────
/// <summary>
/// Return the predictor for <paramref name="name"/>, downloading and building it on first
/// use. Concurrent callers share a single build instead of racing duplicates.
/// </summary>
/// <remarks>See the lifetime note on <see cref="LayaRouter"/>: the returned reference is not leased.</remarks>
public ILayaPredictor Load(string name)
{
var key = NormaliseName(name);
lock (_lock)
{
ThrowIfDisposed();
var slot = FindOrBuild(key);
Evict();
return slot.Predictor;
}
}
// Must be called with _lock held.
private Slot FindOrBuild(LayaCheckpoint key)
{
if (_slots.TryGetValue(key, out var existing))
{
Touch(key);
return existing;
}
var predictor = _engineFactory(key);
var slot = new Slot { Predictor = predictor, OwnsDisposal = true };
_slots[key] = slot;
_order.Add(key);
return slot;
}
// Must be called with _lock held.
private void Touch(LayaCheckpoint key)
{
_order.Remove(key);
_order.Add(key);
}
// Must be called with _lock held. Evicts down to _maxLoaded; a victim still leased (an
// in-flight router-driven Predict) is marked retired instead of disposed immediately.
private void Evict()
{
while (_order.Count > _maxLoaded)
{
var victim = _order[0];
_order.RemoveAt(0);
if (_slots.Remove(victim, out var slot)) RetireOrDispose(slot);
}
}
// Must be called with _lock held.
private static void RetireOrDispose(Slot slot)
{
slot.Retired = true;
if (slot.Leases == 0 && slot.OwnsDisposal && slot.Predictor is IDisposable disposable) disposable.Dispose();
}
/// <summary>
/// Register an already-built predictor under <paramref name="name"/> instead of loading a
/// second copy. Useful when the process has a checkpoint loaded for other reasons.
/// </summary>
/// <remarks>
/// The router never disposes an attached predictor — the caller keeps ownership. Raises
/// <see cref="LayaRouterOptions.MaxLoaded"/> to fit whatever is now resident, so it is not
/// immediately evicted.
/// </remarks>
public ILayaPredictor Attach(string name, ILayaPredictor predictor)
{
ArgumentNullException.ThrowIfNull(predictor);
var key = NormaliseName(name);
lock (_lock)
{
ThrowIfDisposed();
if (_slots.TryGetValue(key, out var old)) RetireOrDispose(old);
_slots[key] = new Slot { Predictor = predictor, OwnsDisposal = false };
Touch(key);
_maxLoaded = Math.Max(_maxLoaded, _slots.Count);
}
return predictor;
}
/// <summary>
/// Download and build checkpoints up front so no request ever pays a model load.
/// <see cref="LayaRouterOptions.MaxLoaded"/> is raised to fit whatever is preloaded.
/// Omit <paramref name="names"/> to preload every checkpoint.
/// </summary>
public LayaRouter Preload(IEnumerable<string>? names = null)
{
var keys = names is null
? AllCheckpoints
: names.Select(NormaliseName).ToArray();
lock (_lock)
{
ThrowIfDisposed();
_maxLoaded = Math.Max(_maxLoaded, Math.Max(keys.Length, _slots.Count));
foreach (var key in keys)
{
if (_slots.ContainsKey(key)) continue; // an attached predictor is already built
FindOrBuild(key);
Evict();
}
}
return this;
}
/// <summary>Free one checkpoint, or all of them when <paramref name="name"/> is omitted.</summary>
public void Unload(string? name = null)
{
lock (_lock)
{
ThrowIfDisposed();
if (name is null)
{
foreach (var slot in _slots.Values) RetireOrDispose(slot);
_slots.Clear();
_order.Clear();
}
else
{
var key = NormaliseName(name);
if (_slots.Remove(key, out var slot)) RetireOrDispose(slot);
_order.Remove(key);
}
}
}
/// <summary>Checkpoints currently resident, least-recently-used first.</summary>
public IReadOnlyList<LayaCheckpoint> Loaded
{
get { lock (_lock) return [.. _order]; }
}
// ── running ───────────────────────────────────────────────────────────────
/// <summary>Route with no explicit override, then answer every question in one forward pass.</summary>
public LayaResult Predict(object? state, QuestionSet questions) => Predict(state, questions, null, null, null);
/// <summary>
/// Route, then answer every question in one forward pass on the chosen checkpoint. The
/// result is the checkpoint's usual answer set, plus <see cref="LayaResult.Routing"/>
/// recording the decision.
/// </summary>
public LayaResult Predict(
object? state, QuestionSet questions,
string? model = null, string? task = null, string? lang = null)
=> Predict(state, questions, model, task, lang, null);
/// <summary>
/// <see cref="Predict(object?, QuestionSet, string?, string?, string?)"/> with Python's per-call
/// <c>lang_guess</c> hint; see <see cref="Route(object?, QuestionSet?, string?, string?, string?, LangGuess?)"/>.
/// </summary>
public LayaResult Predict(
object? state, QuestionSet questions,
string? model, string? task, string? lang, LangGuess? langGuess)
=> PredictRouted(state, questions, model, task, lang, langGuess,
(predictor, decision) => CallPredict(predictor, state, questions, EffectiveLang(lang, decision), null, null));
// router.py:1367-1369: the language the caller gave, else the one routing detected. A hint (lang_guess) or an explicit
// code leaves the detection empty, so only a state that was actually analysed offers one.
private static string? EffectiveLang(string? lang, RouteDecision decision) =>
lang is null && decision.Detection?.Language is { Length: > 0 } detected ? detected : lang;
// router.py:1383-1396: the language and any token budget go to the agent; an agent that takes none of them is called
// plainly, and one that cannot take a budget is refused (Python's TypeError for the keyword).
private static LayaResult CallPredict(
ILayaPredictor predictor, object? state, QuestionSet questions, string? lang, int? maxLen, int? headMaxLen)
{
if (predictor is ILayaCallPredictor engine)
return engine.Predict(state, questions, new LayaPredictOptions { Lang = lang, MaxLen = maxLen, HeadMaxLen = headMaxLen });
if (maxLen is not null || headMaxLen is not null)
throw new NotSupportedException(
$"{predictor.GetType().Name}.Predict() got an unexpected keyword argument '{(maxLen is not null ? "max_len" : "head_max_len")}'");
return predictor.Predict(state, questions);
}
/// <summary>
/// Routes as <see cref="Predict(object?, QuestionSet)"/> does (no model, task, language or hint overrides), with
/// per-call options (a <see langword="null"/> <paramref name="options"/> gates nothing, so a literal <c>Predict(s, q, null)</c> still means what it meant); see <see cref="Predict(object?, QuestionSet, string?, string?, string?, LangGuess?, LayaPredictOptions?)"/>.
/// </summary>
public LayaResult Predict(object? state, QuestionSet questions, LayaPredictOptions? options)
=> Predict(state, questions, null, null, null, null, options);
/// <summary>
/// <see cref="Predict(object?, QuestionSet, string?, string?, string?, LangGuess?)"/> with per-call options.
/// Ports the <c>min_confidence</c>, <c>max_len</c> and <c>head_max_len</c> arguments of Python's <c>Router.predict</c>:
/// the threshold is validated after the state and questions are checked and before anything is routed or loaded, the
/// gate is applied to the answers whichever checkpoint produced them, and the token budgets reach the checkpoint. The
/// language is the <paramref name="lang"/> argument, else the one routing detected, and reaches the checkpoint too.
/// <c>PredictLong</c> takes none of these, as in Python.
/// </summary>
/// <exception cref="ArgumentException">The threshold is invalid, with Python's <c>ValueError</c> text; or
/// <see cref="LayaPredictOptions.Lang"/> is set (use <paramref name="lang"/>).</exception>
public LayaResult Predict(
object? state, QuestionSet questions, string? model, string? task, string? lang, LangGuess? langGuess,
LayaPredictOptions? options)
{
// Python's `predict` takes `lang` once, as the routing hint that is also the agent's language; a second spelling
// would be an ambiguous one.
if (options?.Lang is not null)
throw new ArgumentException(
"LayaPredictOptions.Lang is not accepted by a router; pass the lang argument of Predict, which routes and "
+ "selects the language's temperatures");
return PredictRouted(state, questions, model, task, lang, langGuess,
(predictor, decision) => CallPredict(
predictor, state, questions, EffectiveLang(lang, decision), options?.MaxLen, options?.HeadMaxLen),
options?.MinConfidence);
}
[WarnSite, MethodImpl(MethodImplOptions.NoInlining)]
LayaResult ILayaLongPredictor.PredictLong(
object state, QuestionSet questions, int? window, int? stride, string aggregate, int? batchSize) =>
PredictLong(state, questions, window, stride, aggregate, batchSize);
/// <summary>
/// Route using the usual model/task/lang hints, then scan the entire state. An attached predictor
/// must implement ILayaLongPredictor; unsupported predictors are never silently truncated.
/// </summary>
[WarnSite, MethodImpl(MethodImplOptions.NoInlining)]
public LayaResult PredictLong(
object state, QuestionSet questions, int? window = null, int? stride = null,
string aggregate = "auto", int? batchSize = null,
string? model = null, string? task = null, string? lang = null)
=> PredictLong(state, questions, window, stride, aggregate, batchSize, model, task, lang, null);
/// <summary>
/// <see cref="PredictLong(object, QuestionSet, int?, int?, string, int?, string?, string?, string?)"/>
/// with Python's per-call <c>lang_guess</c> hint; see
/// <see cref="Route(object?, QuestionSet?, string?, string?, string?, LangGuess?)"/>.
/// </summary>
[WarnSite, MethodImpl(MethodImplOptions.NoInlining)]
public LayaResult PredictLong(
object state, QuestionSet questions, int? window, int? stride,
string aggregate, int? batchSize,
string? model, string? task, string? lang, LangGuess? langGuess)
{
StateGuard.NotNull(state);
return PredictRouted(state, questions, model, task, lang, langGuess, (predictor, decision) =>
{
if (predictor is not ILayaLongPredictor scanner)
throw new NotSupportedException(
$"{predictor.GetType().Name} has no PredictLong, so a state longer than its window cannot be scanned");
// router.py:331-333: the caller's language, else the detected one (no truthiness test here, unlike predict).
var effective = lang ?? decision.Detection?.Language;
if (scanner is ILayaLongLangPredictor takesLang)
return takesLang.PredictLong(state, questions, new LayaLongOptions
{
Window = window, Stride = stride, Aggregate = aggregate, BatchSize = batchSize, Lang = effective,
});
if (effective is not null)
WarnOnce.Warn(
$"laya: Router.predict_long: {predictor.GetType().Name}.predict_long has no `lang` parameter, so the "
+ $"{(lang is not null ? "requested" : "detected")} language {ShownLang(effective)} is not reaching it "
+ "and the scan runs without it");
return scanner.PredictLong(state, questions, window, stride, aggregate, batchSize);
});
}
// router.py:350-353: bounded to 32 code points before it is rendered, `...` after a longer one.
private static string ShownLang(string lang)
{
var runes = lang.EnumerateRunes().Count();
return PythonStr.Repr(LanguageDetection.SliceCodePoints(lang, 32)) + (runes > 32 ? "..." : "");
}
private LayaResult PredictRouted(
object? state, QuestionSet questions, string? model, string? task, string? lang, LangGuess? langGuess,
Func<ILayaPredictor, RouteDecision, LayaResult> predict, MinConfidence? minConfidence = null)
{
StateGuard.NotNull(state);
ArgumentNullException.ThrowIfNull(questions);
// router.py:1338: validated after the state and questions, before routing or loading anything.
var gate = minConfidence?.Check();
var decision = Route(state, questions, model, task, lang, langGuess);
// Inference runs outside the lock, exactly as Python's predict() calls
// agent.system_one after load() has already returned and released the lock — so
// concurrent predictions on a shared checkpoint are not serialised by the router.
var result = WithLease(decision.Model, predictor => predict(predictor, decision));
return (gate is { } g ? ConfidenceGate.Apply(result, g) : result).WithRouting(decision);
}
// Loads (or finds) the checkpoint, holds a lease so eviction cannot dispose it mid-call, and
// runs `run` on it outside the lock.
private T WithLease<T>(LayaCheckpoint checkpoint, Func<ILayaPredictor, T> run)
{
Slot slot;
lock (_lock)
{
ThrowIfDisposed();
slot = FindOrBuild(checkpoint);
slot.Leases++;
Evict();
}
try
{
return run(slot.Predictor);
}
finally
{
lock (_lock)
{
slot.Leases--;
if (slot.Retired && slot.Leases == 0 && slot.OwnsDisposal && slot.Predictor is IDisposable disposable)
disposable.Dispose();
}
}
}
// ── batching ──────────────────────────────────────────────────────────────
/// <summary>
/// Route a heterogeneous request batch without loading any checkpoint. Ports Python
/// <c>Router.route_batch</c> (router.py:1538): each request is validated and routed exactly as
/// <see cref="Route(object?, QuestionSet?, string?, string?, string?, LangGuess?)"/> does with its own
/// <c>Model</c>, <c>Task</c>, <c>Lang</c> and <c>LangGuess</c>, and the decisions keep input order.
/// </summary>
/// <exception cref="ArgumentException">A request is null, has a null state or null questions, or names an unknown model or task.</exception>
public IReadOnlyList<RouteDecision> RouteBatch(IReadOnlyList<LayaBatchRequest> requests)
{
ArgumentNullException.ThrowIfNull(requests);
var decisions = new List<RouteDecision>(requests.Count);
for (var i = 0; i < requests.Count; i++)
{
var request = requests[i]
?? throw new ArgumentException($"request {i} must be a dict, got NoneType");
if (request.State is null)
throw new ArgumentNullException(null, $"request {i} 'state' must not be None; pass a string, dict, or list");
if (request.Questions is null)
throw new ArgumentException($"request {i} 'questions' must be a dict, got NoneType");
decisions.Add(Route(request.State, request.Questions, request.Model, request.Task,
request.Lang, request.LangGuess));
}
return decisions;
}
/// <summary>
/// Route and answer a heterogeneous request batch with minimal checkpoint churn. Ports Python
/// <c>Router.predict_batch</c> (router.py:1603): requests are routed first
/// (<see cref="RouteBatch"/>), grouped by routed checkpoint in first-seen order, and within a
/// checkpoint split again by question schema (order-sensitive, router.py:221) so states sharing
/// a schema share forward passes via <see cref="ILayaBatchPredictor.PredictBatch"/>. Results are
/// returned in input order, each carrying its <see cref="LayaResult.Routing"/>.
/// </summary>
/// <param name="requests">The requests; an empty list returns an empty list and loads nothing.</param>
/// <param name="options">Passed to every checkpoint batch; <see cref="LayaBatchOptions.SortByLength"/>
/// acts per question group, as in Python. <see cref="LayaBatchOptions.MinConfidence"/> is batch-wide: it is
/// applied by the router to every result, is not passed to the checkpoints, and is not part of the grouping.</param>
/// <exception cref="NotSupportedException">A routed checkpoint (an attached predictor) has no batch support.</exception>
/// <exception cref="ArgumentException">
/// <paramref name="options"/> sets <see cref="LayaBatchOptions.Lang"/>, <see cref="LayaBatchOptions.MaxLen"/> or
/// <see cref="LayaBatchOptions.HeadMaxLen"/>, which Python's <c>Router.predict_batch</c> does not take (it is a
/// <c>TypeError</c> there): they are per request, on <see cref="LayaBatchRequest"/>.
/// </exception>
public IReadOnlyList<LayaResult> PredictBatch(
IReadOnlyList<LayaBatchRequest> requests, LayaBatchOptions? options = null)
{
foreach (var (name, set) in new[] { ("lang", options?.Lang is not null), ("max_len", options?.MaxLen is not null),
("head_max_len", options?.HeadMaxLen is not null) })
if (set)
throw new ArgumentException(
$"Router.predict_batch() got an unexpected keyword argument '{name}'; set it on each LayaBatchRequest");
// router.py:1669: validated before any request is looked at.
var gate = options?.MinConfidence?.Check();
// router.py:1789 calls the agent without min_confidence and gates the results itself.
var inner = options is { MinConfidence: not null }
? new LayaBatchOptions { BatchSize = options.BatchSize, SortByLength = options.SortByLength }
: options;
var decisions = RouteBatch(requests);
if (decisions.Count == 0) return [];
// First-seen order, so cache effects stay deterministic and an interleaved workload costs at
// most one load per routed checkpoint (router.py:1674).
var checkpoints = new List<LayaCheckpoint>();
var byCheckpoint = new Dictionary<LayaCheckpoint, List<int>>();
for (var i = 0; i < decisions.Count; i++)
{
if (!byCheckpoint.TryGetValue(decisions[i].Model, out var indices))
{
byCheckpoint[decisions[i].Model] = indices = [];
checkpoints.Add(decisions[i].Model);
}
indices.Add(i);
}
var results = new LayaResult[requests.Count];
foreach (var checkpoint in checkpoints)
{
var indices = byCheckpoint[checkpoint];
WithLease(checkpoint, predictor =>
{
if (predictor is not ILayaBatchPredictor batcher)
throw new NotSupportedException(
$"{predictor.GetType().Name} has no PredictBatch, so requests cannot share forward passes");
// router.py:1751: a language only matters to a checkpoint that has per-language temperatures.
var langAware = predictor is ILayaLangTemperatures { HasLangTemperatures: true };
foreach (var (questions, group, key) in GroupBySchema(requests, indices, decisions, langAware))
{
// Passed only when set, as Python passes its keyword arguments (router.py:1769-1773).
var groupOptions = key.Lang is null && key.MaxLen is null && key.HeadMaxLen is null
? inner
: new LayaBatchOptions
{
BatchSize = inner?.BatchSize, SortByLength = inner?.SortByLength ?? false,
Lang = key.Lang, MaxLen = key.MaxLen, HeadMaxLen = key.HeadMaxLen,
};
var batch = batcher.PredictBatch([.. group.Select(i => requests[i].State)], questions, groupOptions);
if (batch.Count != group.Count)
throw new InvalidOperationException(
$"internal error: Agent.predict_batch returned {batch.Count} results for {group.Count} states");
for (var k = 0; k < batch.Count; k++)
results[group[k]] = (gate is { } g ? ConfidenceGate.Apply(batch[k], g) : batch[k])
.WithRouting(decisions[group[k]]);
}
return 0;
});
}
return results;
}
// What a forward pass shares besides the question schema: the token budgets and, for a checkpoint with per-language
// temperatures, the language (raw, not normalised: "de" and "de-AT" are separate passes).
private readonly record struct GroupKey(string? Lang, int? MaxLen, int? HeadMaxLen);
// Splits one checkpoint's requests by question schema, token budgets and language, first-seen order
// (router.py:1726-1776).
private static List<(QuestionSet Questions, List<int> Indices, GroupKey Key)> GroupBySchema(
IReadOnlyList<LayaBatchRequest> requests, List<int> indices, IReadOnlyList<RouteDecision> decisions, bool langAware)
{
var groups = new List<(string Schema, QuestionSet Questions, List<int> Indices, GroupKey Key)>();
QuestionSet? lastQuestions = null;
string? lastSchema = null;
foreach (var i in indices)
{
var questions = requests[i].Questions;
string schema;
if (ReferenceEquals(questions, lastQuestions)) schema = lastSchema!;
else
{
schema = QuestionSchema(questions);
lastQuestions = questions;
lastSchema = schema;
}
// The request's own language, else the detection's (router.py:1751-1756); none for a checkpoint without any.
var lang = langAware ? requests[i].Lang ?? decisions[i].Detection?.Language : null;
var key = new GroupKey(lang, requests[i].MaxLen, requests[i].HeadMaxLen);
var at = groups.FindIndex(g => g.Schema == schema && g.Key == key);
if (at < 0) groups.Add((schema, questions, [i], key));
else groups[at].Indices.Add(i);
}
return [.. groups.Select(g => (g.Questions, g.Indices, g.Key))];
}
/// <summary>
/// Order-sensitive signature of a question set, for sharing forward passes (router.py:221): ids, types,
/// instructions, labels and rendered options all count, in declaration order.
/// </summary>
internal static string QuestionSchema(QuestionSet questions)
{
// json.dumps(questions, sort_keys=False, ensure_ascii=False, default=str): insertion order at
// every level, ", " / ": " separators, Python float repr (so 1 and 1.0 differ). A question
// carries no memory of how its dict was spelled, so each is written in canonical key order
// (type, instructions, criteria) and a noul only lists the wordings it was given.
var all = new List<KeyValuePair<string, object?>>();
foreach (var (id, question) in questions)
{
var entry = new List<KeyValuePair<string, object?>>
{
new("type", SequenceBuilder.TypeName(question.Type)),
new("instructions", question.Instructions),
};
switch (question)
{
case ChoiceQuestion choice:
entry.Add(new("criteria", choice.Options.ToList()));
break;
case ScoreQuestion score:
entry.Add(new("criteria", score.Levels));
break;
case NoulQuestion { IfFalse: null, IfTrue: null }:
break;
case NoulQuestion noul:
var wording = new List<KeyValuePair<string, object?>>();
if (noul.IfFalse is not null) wording.Add(new("false", noul.IfFalse));
if (noul.IfTrue is not null) wording.Add(new("true", noul.IfTrue));
entry.Add(new("criteria", wording));
break;
}
all.Add(new(id, entry));
}
return PythonJson.Dumps(all, escapeNonAscii: false, fallbackToStr: true);
}
private void ThrowIfDisposed() => ObjectDisposedException.ThrowIf(_disposed, this);
/// <inheritdoc/>
public void Dispose()
{
lock (_lock)
{
if (_disposed) return;
_disposed = true;
foreach (var slot in _slots.Values)
{
if (slot.Leases == 0 && slot.OwnsDisposal && slot.Predictor is IDisposable disposable)
disposable.Dispose();
else
slot.Retired = true; // an in-flight Predict disposes it on release
}
_slots.Clear();
_order.Clear();
}
}
}