Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

feat(dotnet): add calibration and typed answers

kevin-gatimuedef7fd30be3
diff --git a/laya-dotnet/src/Laya/Answers.cs b/laya-dotnet/src/Laya/Answers.cs
new file mode 100644
index 00000000..45cb8251
--- /dev/null
+++ b/laya-dotnet/src/Laya/Answers.cs
@@ -0,0 +1,225 @@
+namespace Laya;
+
+/// <summary>Token accounting for one <c>Predict</c> call. Laya emits no tokens, so output is 0.</summary>
+/// <param name="InputTokens">Non-padding tokens across every question in the batch.</param>
+/// <param name="OutputTokens">Always 0: a System 1 pass generates nothing.</param>
+public readonly record struct Usage(int InputTokens, int OutputTokens);
+
+/// <summary>
+/// The action head's read on whether the answer can be acted on directly.
+/// </summary>
+/// <param name="ActProbability">
+/// Probability of acting rather than escalating. Corresponds to Python's
+/// <c>action.act_probability</c>: softmax over the action logits, first component.
+/// </param>
+public readonly record struct ActionInfo(double ActProbability);
+
+/// <summary>One answer. Cast with <see cref="AsChoice"/>, <see cref="AsScore"/> or <see cref="AsNoul"/>.</summary>
+public abstract class Answer
+{
+ internal Answer(QuestionType type, double confidence, double answerConfidence, ActionInfo action)
+ {
+ Type = type;
+ Confidence = confidence;
+ AnswerConfidence = answerConfidence;
+ Action = action;
+ }
+
+ /// <summary>Which question type produced this answer.</summary>
+ public QuestionType Type { get; }
+
+ /// <summary>
+ /// Calibrated confidence in [0, 1]. For choice and score this is normalized Shannon entropy
+ /// (<c>1 - H(p)/log k</c>); for noul it is <c>max(p, 1 - p)</c>. Rounded to 4 decimals, as Python does.
+ /// </summary>
+ public double Confidence { get; }
+
+ /// <summary>
+ /// The calibrated <c>max(p)</c> confidence, in [0, 1], added alongside <see cref="Confidence"/>
+ /// in Python 0.3.21 (<c>answer_confidence</c> in <c>agent.py</c>'s <c>_decode_answers</c>,
+ /// computed by the new <c>laya/confidence.py</c>-adjacent <c>common.answer_confidence</c>).
+ /// It means the same thing on every question type — the probability mass on the reported
+ /// answer — unlike <see cref="Confidence"/>, whose formula differs between noul and the other
+ /// two types, so it is the field to gate abstention on across mixed question types. Rounded to
+ /// 4 decimals, as Python does. <see cref="Confidence"/> itself is unchanged.
+ /// </summary>
+ public double AnswerConfidence { get; }
+
+ /// <summary>The action head's output.</summary>
+ public ActionInfo Action { get; }
+
+ /// <summary>This answer as a choice answer.</summary>
+ /// <exception cref="InvalidOperationException">If the question was not a choice question.</exception>
+ public ChoiceAnswer AsChoice() => this as ChoiceAnswer
+ ?? throw new InvalidOperationException($"answer is {Type}, not Choice");
+
+ /// <summary>This answer as a score answer.</summary>
+ /// <exception cref="InvalidOperationException">If the question was not a score question.</exception>
+ public ScoreAnswer AsScore() => this as ScoreAnswer
+ ?? throw new InvalidOperationException($"answer is {Type}, not Score");
+
+ /// <summary>This answer as a noul answer.</summary>
+ /// <exception cref="InvalidOperationException">If the question was not a noul question.</exception>
+ public NoulAnswer AsNoul() => this as NoulAnswer
+ ?? throw new InvalidOperationException($"answer is {Type}, not Noul");
+}
+
+/// <summary>The answer to a <see cref="QuestionType.Choice"/> question.</summary>
+public sealed class ChoiceAnswer : Answer
+{
+ internal ChoiceAnswer(string choice, IReadOnlyList<KeyValuePair<string, double>> probabilities,
+ double confidence, double answerConfidence, ActionInfo action)
+ : base(QuestionType.Choice, confidence, answerConfidence, action)
+ {
+ Choice = choice;
+ Probabilities = probabilities;
+ }
+
+ /// <summary>The highest-probability label.</summary>
+ public string Choice { get; }
+
+ /// <summary>Probability per label, in the order the options were declared. Rounded to 4 decimals.</summary>
+ public IReadOnlyList<KeyValuePair<string, double>> Probabilities { get; }
+
+ /// <summary>The probability assigned to one label.</summary>
+ /// <exception cref="KeyNotFoundException">If no option carries that label.</exception>
+ public double this[string label]
+ {
+ get
+ {
+ foreach (var kv in Probabilities)
+ if (string.Equals(kv.Key, label, StringComparison.Ordinal)) return kv.Value;
+ throw new KeyNotFoundException($"no option labelled '{label}'");
+ }
+ }
+
+ /// <inheritdoc/>
+ public override string ToString() => $"{Choice} ({Confidence:0.###} confidence)";
+}
+
+/// <summary>The answer to a <see cref="QuestionType.Score"/> question.</summary>
+public sealed class ScoreAnswer : Answer
+{
+ internal ScoreAnswer(double score, IReadOnlyList<object?> legend, IReadOnlyList<double> probabilities,
+ double confidence, double answerConfidence, ActionInfo action)
+ : base(QuestionType.Score, confidence, answerConfidence, action)
+ {
+ Score = score;
+ Legend = legend;
+ Probabilities = probabilities;
+ }
+
+ /// <summary>
+ /// Expected level: the probability-weighted mean of the level indices, so a fractional value
+ /// between 0 and <c>Legend.Count - 1</c>. Rounded to 4 decimals.
+ /// </summary>
+ public double Score { get; }
+
+ /// <summary>What each level index means, as supplied on the question.</summary>
+ public IReadOnlyList<object?> Legend { get; }
+
+ /// <summary>Probability per level index. Rounded to 4 decimals.</summary>
+ public IReadOnlyList<double> Probabilities { get; }
+
+ /// <summary>The single most likely level index, as opposed to the expected value.</summary>
+ public int MostLikelyLevel
+ {
+ get
+ {
+ var best = 0;
+ for (var i = 1; i < Probabilities.Count; i++)
+ if (Probabilities[i] > Probabilities[best]) best = i;
+ return best;
+ }
+ }
+
+ /// <inheritdoc/>
+ public override string ToString() => $"{Score:0.####} of 0..{Legend.Count - 1}";
+}
+
+/// <summary>The answer to a <see cref="QuestionType.Noul"/> question.</summary>
+public sealed class NoulAnswer : Answer
+{
+ internal NoulAnswer(double probability, double confidence, double answerConfidence, ActionInfo action)
+ : base(QuestionType.Noul, confidence, answerConfidence, action) => Probability = probability;
+
+ /// <summary>Probability that the statement holds, in [0, 1]. Rounded to 4 decimals.</summary>
+ public double Probability { get; }
+
+ /// <summary>Whether the statement more likely holds than not.</summary>
+ public bool Value => Probability >= 0.5;
+
+ /// <inheritdoc/>
+ public override string ToString() => $"{Value} (p={Probability:0.####})";
+}
+
+/// <summary>The result of one <c>Predict</c> call: one answer per question, plus token usage.</summary>
+public sealed class LayaResult
+{
+ private readonly Dictionary<string, Answer> _byId;
+ private readonly List<string> _order;
+
+ internal LayaResult(string model, List<string> order, Dictionary<string, Answer> byId, Usage usage,
+ RouteDecision? routing = null, IReadOnlyDictionary<string, ShortlistInfo>? shortlist = null)
+ {
+ Model = model;
+ _order = order;
+ _byId = byId;
+ Usage = usage;
+ Routing = routing;
+ Shortlist = shortlist;
+ }
+
+ /// <summary>The model identifier, matching Python's <c>"laya-rl-agent"</c>.</summary>
+ public string Model { get; }
+
+ /// <summary>Token accounting for the call.</summary>
+ public Usage Usage { get; }
+
+ /// <summary>
+ /// The routing decision that picked <see cref="Model"/>, when this result came from
+ /// <see cref="LayaRouter.Predict(object, QuestionSet)"/>. <see langword="null"/> for a
+ /// result answered directly by a <see cref="LayaEngine"/>. Mirrors Python's <c>result["routing"]</c>.
+ /// </summary>
+ public RouteDecision? Routing { get; }
+
+ /// <summary>
+ /// Per-question shortlist metadata, when this result came from
+ /// <see cref="LayaShortlist.Predict(ILayaPredictor, object, QuestionSet, LayaEmbedFunction, int)"/>.
+ /// <see langword="null"/> otherwise. Mirrors Python's <c>result["shortlist"]</c>. A result can
+ /// carry both <see cref="Routing"/> and <see cref="Shortlist"/> when a shortlist ran over a
+ /// <see cref="LayaRouter"/>.
+ /// </summary>
+ public IReadOnlyDictionary<string, ShortlistInfo>? Shortlist { get; }
+
+ /// <summary>A copy of this result with <see cref="Routing"/> set. Used internally by <see cref="LayaRouter"/>.</summary>
+ internal LayaResult WithRouting(RouteDecision routing) => new(Model, _order, _byId, Usage, routing, Shortlist);
+
+ /// <summary>A copy of this result with <see cref="Shortlist"/> set. Used internally by <see cref="LayaShortlist"/>.</summary>
+ internal LayaResult WithShortlist(IReadOnlyDictionary<string, ShortlistInfo> shortlist) =>
+ new(Model, _order, _byId, Usage, Routing, shortlist);
+
+ /// <summary>The question ids, in the order they were asked.</summary>
+ public IReadOnlyList<string> Ids => _order;
+
+ /// <summary>How many answers there are.</summary>
+ public int Count => _order.Count;
+
+ /// <summary>The answer to one question.</summary>
+ /// <exception cref="KeyNotFoundException">If no question carried that id.</exception>
+ public Answer this[string id] => _byId.TryGetValue(id, out var a)
+ ? a
+ : throw new KeyNotFoundException($"no answer for question '{id}'");
+
+ /// <summary>Try to get the answer to one question.</summary>
+ public bool TryGetAnswer(string id, out Answer? answer) => _byId.TryGetValue(id, out answer);
+
+ /// <summary>Every answer, in the order the questions were asked.</summary>
+ public IEnumerable<KeyValuePair<string, Answer>> Answers
+ {
+ get
+ {
+ foreach (var id in _order) yield return new(id, _byId[id]);
+ }
+ }
+}
diff --git a/laya-dotnet/src/Laya/Calibration.cs b/laya-dotnet/src/Laya/Calibration.cs
new file mode 100644
index 00000000..e0974de2
--- /dev/null
+++ b/laya-dotnet/src/Laya/Calibration.cs
@@ -0,0 +1,158 @@
+namespace Laya;
+
+/// <summary>
+/// Turns raw marker logits into calibrated probabilities and confidence. Ports
+/// <c>temp_bucket</c>, <c>clamp_temperature</c> and <c>confidence_from_probs</c> from
+/// <c>laya/common.py</c>, plus the readout arithmetic in <c>Agent.system_one</c>.
+/// </summary>
+public static class Calibration
+{
+ /// <summary>
+ /// Lowest temperature that will be applied. A fitted temperature below 1 sharpens logits
+ /// instead of softening them; the shipped <c>choice:11+</c> bucket of 0.1006 multiplies them
+ /// roughly tenfold, publishing a 0.24 top probability as 0.99. A caller gating on confidence
+ /// would be told a coin flip is a certainty, so such a value is refused rather than applied.
+ /// </summary>
+ public const double TempMin = 0.5;
+
+ /// <summary>Highest temperature that will be applied.</summary>
+ public const double TempMax = 5.0;
+
+ /// <summary>A usable temperature: confined to <see cref="TempMin"/>..<see cref="TempMax"/>, with
+ /// anything that is not a finite number falling back to 1.0.</summary>
+ public static double ClampTemperature(double t) =>
+ double.IsNaN(t) || double.IsInfinity(t) ? 1.0 : Math.Min(TempMax, Math.Max(TempMin, t));
+
+ /// <summary>
+ /// The calibration bucket key for a question type and option count, e.g. <c>"choice:3-5"</c>.
+ /// </summary>
+ public static string TempBucket(QuestionType type, int optionCount)
+ {
+ var size = optionCount <= 2 ? "2"
+ : optionCount <= 5 ? "3-5"
+ : optionCount <= 10 ? "6-10"
+ : "11+";
+ return $"{SequenceBuilder.TypeName(type)}:{size}";
+ }
+
+ /// <summary>The temperature to divide logits by for one question.</summary>
+ public static double ResolveTemperature(LayaConfig config, QuestionType type, int optionCount)
+ {
+ ArgumentNullException.ThrowIfNull(config);
+ return config.TemperatureByOptions.TryGetValue(TempBucket(type, optionCount), out var t)
+ ? t
+ : config.Temperature[(int)type];
+ }
+
+ /// <summary>
+ /// Tempered softmax over the first <paramref name="optionCount"/> logits of one row.
+ /// </summary>
+ /// <remarks>
+ /// Only the real markers are read. Padding columns carry the <c>-1e4</c> the model's
+ /// <c>masked_fill</c> wrote, which would underflow to zero anyway, but slicing first keeps the
+ /// normalisation identical to Python's <c>logits[r, :k]</c>.
+ /// </remarks>
+ public static double[] Probabilities(ReadOnlySpan<float> logits, int optionCount, double temperature)
+ {
+ if (optionCount <= 0) throw new ArgumentOutOfRangeException(nameof(optionCount));
+
+ var p = new double[optionCount];
+ var max = double.NegativeInfinity;
+ for (var i = 0; i < optionCount; i++)
+ {
+ p[i] = logits[i] / temperature;
+ if (p[i] > max) max = p[i];
+ }
+
+ var sum = 0.0;
+ for (var i = 0; i < optionCount; i++)
+ {
+ p[i] = Math.Exp(p[i] - max);
+ sum += p[i];
+ }
+ for (var i = 0; i < optionCount; i++) p[i] /= sum;
+ return p;
+ }
+
+ /// <summary>Softmax over a whole row, used for the action head.</summary>
+ public static double[] Softmax(ReadOnlySpan<float> logits)
+ {
+ var p = new double[logits.Length];
+ var max = double.NegativeInfinity;
+ foreach (var z in logits) if (z > max) max = z;
+
+ var sum = 0.0;
+ for (var i = 0; i < logits.Length; i++)
+ {
+ p[i] = Math.Exp(logits[i] - max);
+ sum += p[i];
+ }
+ for (var i = 0; i < p.Length; i++) p[i] /= sum;
+ return p;
+ }
+
+ /// <summary>
+ /// Normalized Shannon entropy confidence, <c>1 - H(p) / log k</c>, clipped to [0, 1].
+ /// A single-option question is fully decided by construction, so it returns 1.0.
+ /// </summary>
+ public static double ConfidenceFromProbs(ReadOnlySpan<double> p, int optionCount)
+ {
+ if (optionCount < 2) return 1.0;
+
+ var entropy = 0.0;
+ for (var i = 0; i < optionCount; i++)
+ {
+ var clipped = Math.Clamp(p[i], 1e-12, 1.0);
+ entropy -= p[i] * Math.Log(clipped);
+ }
+ return Math.Clamp(1.0 - entropy / Math.Log(optionCount), 0.0, 1.0);
+ }
+
+ /// <summary>
+ /// The calibrated <c>answer_confidence</c>: <c>max(p[:k])</c>, clipped to [0, 1]. A
+ /// single-option question is fully decided by construction, so it returns 1.0.
+ /// </summary>
+ /// <remarks>
+ /// Ports <c>answer_confidence</c> in <c>laya/common.py</c> (0.3.21, surfaced via the new
+ /// <c>laya/confidence.py</c>). Unlike <see cref="ConfidenceFromProbs"/> — normalized Shannon
+ /// entropy for choice/score, <c>max(p, 1-p)</c> for noul — this is the one quantity
+ /// temperature scaling fits and every calibration figure is computed on, so it means the same
+ /// thing across all three question types and is safe to gate a single threshold on. Since
+ /// <paramref name="p"/> here already holds exactly <paramref name="optionCount"/> entries
+ /// (see <see cref="Probabilities"/>), this is just <c>max(p)</c>.
+ /// </remarks>
+ public static double AnswerConfidence(ReadOnlySpan<double> p, int optionCount)
+ {
+ if (optionCount < 1) return 1.0;
+
+ var max = double.NegativeInfinity;
+ for (var i = 0; i < optionCount; i++) if (p[i] > max) max = p[i];
+ return Math.Clamp(max, 0.0, 1.0);
+ }
+
+ /// <summary>
+ /// The expected level of a score answer: the probability-weighted mean of the level indices,
+ /// so a fractional value rather than the most likely level.
+ /// </summary>
+ public static double ExpectedScore(ReadOnlySpan<double> p)
+ {
+ var expected = 0.0;
+ for (var i = 0; i < p.Length; i++) expected += i * p[i];
+ return expected;
+ }
+
+ /// <summary>The index of the largest probability, ties going to the lowest index as NumPy's
+ /// <c>argmax</c> does.</summary>
+ public static int ArgMax(ReadOnlySpan<double> p)
+ {
+ var best = 0;
+ for (var i = 1; i < p.Length; i++) if (p[i] > p[best]) best = i;
+ return best;
+ }
+
+ /// <summary>
+ /// Python's <c>round(x, 4)</c>. Both runtimes round half to even on the binary value, so this
+ /// agrees with the reference implementation.
+ /// </summary>
+ public static double Round4(double value) => Math.Round(value, 4, MidpointRounding.ToEven);
+}
diff --git a/laya-dotnet/tests/Laya.Tests/CalibrationTests.cs b/laya-dotnet/tests/Laya.Tests/CalibrationTests.cs
new file mode 100644
index 00000000..65252744
--- /dev/null
+++ b/laya-dotnet/tests/Laya.Tests/CalibrationTests.cs
@@ -0,0 +1,182 @@
+namespace Laya.Tests;
+
+/// <summary>Tier 1: the calibration maths, which needs no artifacts.</summary>
+public sealed class CalibrationTests
+{
+ [Theory]
+ [InlineData(QuestionType.Choice, 1, "choice:2")]
+ [InlineData(QuestionType.Choice, 2, "choice:2")]
+ [InlineData(QuestionType.Choice, 3, "choice:3-5")]
+ [InlineData(QuestionType.Choice, 5, "choice:3-5")]
+ [InlineData(QuestionType.Choice, 6, "choice:6-10")]
+ [InlineData(QuestionType.Choice, 10, "choice:6-10")]
+ [InlineData(QuestionType.Choice, 11, "choice:11+")]
+ [InlineData(QuestionType.Score, 4, "score:3-5")]
+ [InlineData(QuestionType.Noul, 2, "noul:2")]
+ public void TempBucketMatchesPythonBoundaries(QuestionType type, int k, string expected) =>
+ Assert.Equal(expected, Calibration.TempBucket(type, k));
+
+ [Theory]
+ [InlineData(1.0, 1.0)]
+ [InlineData(2.5, 2.5)]
+ [InlineData(0.1006, Calibration.TempMin)] // the shipped choice:11+ value, refused as too sharp
+ [InlineData(0.0, Calibration.TempMin)]
+ [InlineData(-3.0, Calibration.TempMin)]
+ [InlineData(99.0, Calibration.TempMax)]
+ [InlineData(double.NaN, 1.0)]
+ [InlineData(double.PositiveInfinity, 1.0)]
+ [InlineData(double.NegativeInfinity, 1.0)]
+ public void ClampTemperatureMatchesPython(double input, double expected) =>
+ Assert.Equal(expected, Calibration.ClampTemperature(input));
+
+ [Fact]
+ public void ProbabilitiesReadOnlyTheRealMarkers()
+ {
+ // The padded marker column carries the -1e4 the model's masked_fill wrote. A single-option
+ // question must still come out as a certainty rather than leaking the pad column in.
+ double[] p = Calibration.Probabilities([-0.9127780f, -10000.0f], optionCount: 1, temperature: 1.0);
+ Assert.Single(p);
+ Assert.Equal(1.0, p[0], 12);
+ }
+
+ [Fact]
+ public void ProbabilitiesSumToOneAndRespectTemperature()
+ {
+ float[] logits = [2.0f, 1.0f, 0.0f];
+ var sharp = Calibration.Probabilities(logits, 3, 0.5);
+ var soft = Calibration.Probabilities(logits, 3, 5.0);
+
+ Assert.Equal(1.0, sharp.Sum(), 12);
+ Assert.Equal(1.0, soft.Sum(), 12);
+ // A lower temperature concentrates mass on the top option; a higher one flattens it.
+ Assert.True(sharp[0] > soft[0]);
+ Assert.True(soft[2] > sharp[2]);
+ }
+
+ [Fact]
+ public void ProbabilitiesSurviveLargeLogits()
+ {
+ // act_logits in the goldens reach ±1600; a naive exp() would overflow to NaN.
+ var p = Calibration.Softmax([1366.2135f, -1624.665f]);
+ Assert.Equal(1.0, p[0], 12);
+ Assert.Equal(0.0, p[1], 12);
+ }
+
+ [Fact]
+ public void ConfidenceIsOneForASingleOption() =>
+ Assert.Equal(1.0, Calibration.ConfidenceFromProbs([1.0], optionCount: 1));
+
+ [Fact]
+ public void ConfidenceIsZeroForAUniformDistribution()
+ {
+ Assert.Equal(0.0, Calibration.ConfidenceFromProbs([0.5, 0.5], 2), 12);
+ Assert.Equal(0.0, Calibration.ConfidenceFromProbs([0.25, 0.25, 0.25, 0.25], 4), 12);
+ }
+
+ [Fact]
+ public void ConfidenceIsOneForACertainty() =>
+ Assert.Equal(1.0, Calibration.ConfidenceFromProbs([1.0, 0.0], 2), 12);
+
+ [Fact]
+ public void ConfidenceMatchesTheRecordedQuickstartValues()
+ {
+ // From golden/multilingual/case_quickstart.json: urgency's four-level distribution and its confidence.
+ double[] urgency = [0.0179, 0.465, 0.3208, 0.1964];
+ Assert.Equal(0.1976, Calibration.Round4(Calibration.ConfidenceFromProbs(urgency, 4)), 3);
+ }
+
+ [Fact]
+ public void ConfidenceIgnoresPaddingBeyondTheOptionCount()
+ {
+ // Passing a longer buffer must not change the answer: only the first k entries count.
+ double[] padded = [0.5, 0.5, 0.0, 0.0];
+ Assert.Equal(Calibration.ConfidenceFromProbs([0.5, 0.5], 2),
+ Calibration.ConfidenceFromProbs(padded, 2));
+ }
+
+ [Fact]
+ public void AnswerConfidenceIsOneWhenThereIsOnlyOneOption() =>
+ Assert.Equal(1.0, Calibration.AnswerConfidence([1.0], optionCount: 1));
+
+ [Fact]
+ public void AnswerConfidenceIsTheMaxProbability()
+ {
+ // Ports common.answer_confidence: float(np.clip(np.max(p[:k]), 0.0, 1.0)).
+ double[] urgency = [0.0179, 0.465, 0.3208, 0.1964];
+ Assert.Equal(0.465, Calibration.AnswerConfidence(urgency, 4), 12);
+ }
+
+ [Fact]
+ public void AnswerConfidenceIgnoresPaddingBeyondTheOptionCount()
+ {
+ double[] padded = [0.2, 0.3, 0.9, 0.9];
+ Assert.Equal(0.3, Calibration.AnswerConfidence(padded, 2), 12);
+ }
+
+ [Fact]
+ public void AnswerConfidenceDiffersFromConfidenceFromProbsOnTheSameDistribution()
+ {
+ // Two genuinely distinct fields on a skewed distribution: ConfidenceFromProbs is a
+ // normalized-entropy measure, AnswerConfidence is a bare max(p) - they must not collapse
+ // to the same number here, or Answers.AnswerConfidence would be a redundant duplicate.
+ double[] urgency = [0.0179, 0.465, 0.3208, 0.1964];
+ var entropyConfidence = Calibration.ConfidenceFromProbs(urgency, 4);
+ var answerConfidence = Calibration.AnswerConfidence(urgency, 4);
+
+ Assert.Equal(0.1976, Calibration.Round4(entropyConfidence), 3);
+ Assert.Equal(0.465, answerConfidence, 12);
+ Assert.NotEqual(entropyConfidence, answerConfidence, 3);
+ }
+
+ [Fact]
+ public void ExpectedScoreIsAProbabilityWeightedMean()
+ {
+ double[] urgency = [0.0179, 0.465, 0.3208, 0.1964];
+ Assert.Equal(1.6956, Calibration.Round4(Calibration.ExpectedScore(urgency)), 3);
+ Assert.Equal(0.0, Calibration.ExpectedScore([1.0, 0.0, 0.0]), 12);
+ Assert.Equal(2.0, Calibration.ExpectedScore([0.0, 0.0, 1.0]), 12);
+ }
+
+ [Fact]
+ public void ArgMaxBreaksTiesTowardTheLowestIndexAsNumpyDoes() =>
+ Assert.Equal(0, Calibration.ArgMax([0.5, 0.5, 0.0]));
+
+ [Fact]
+ public void ResolveTemperaturePrefersTheBucketThenTheTypeDefault()
+ {
+ var config = LayaConfig.Parse("""
+ {"max_len": 1024, "head_max_len": 256,
+ "temperature": [1.5, 2.0, 2.5],
+ "temperature_by_options": {"choice:3-5": 3.0, "noul:2": 0.2}}
+ """);
+
+ Assert.Equal(3.0, Calibration.ResolveTemperature(config, QuestionType.Choice, 4));
+ Assert.Equal(1.5, Calibration.ResolveTemperature(config, QuestionType.Choice, 8));
+ Assert.Equal(2.0, Calibration.ResolveTemperature(config, QuestionType.Score, 4));
+ // 0.2 is below TempMin, so the bucket is clamped rather than applied as recorded.
+ Assert.Equal(Calibration.TempMin, Calibration.ResolveTemperature(config, QuestionType.Noul, 2));
+ }
+
+ [Fact]
+ public void ConfigFallsBackToPythonsInCodeDefaults()
+ {
+ var config = LayaConfig.Parse("{}");
+ Assert.Equal(512, config.MaxLen);
+ Assert.Equal(192, config.HeadMaxLen);
+ Assert.Equal([1.0, 1.0, 1.0], config.Temperature);
+ }
+
+ [Fact]
+ public void ShippedConfigParsesAsRecorded()
+ {
+ // meta.json records the config the goldens were produced under.
+ var meta = GoldenData.Meta;
+ var config = LayaConfig.Parse(meta.GetProperty("config").GetRawText());
+
+ Assert.Equal(meta.GetProperty("max_len").GetInt32(), config.MaxLen);
+ Assert.Equal(meta.GetProperty("head_max_len").GetInt32(), config.HeadMaxLen);
+ Assert.Equal([.. meta.GetProperty("temperature").EnumerateArray().Select(e => e.GetDouble())],
+ config.Temperature);
+ Assert.Empty(config.TemperatureByOptions);
+ }
+}