Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

feat(dotnet): add embedding shortlist for high-cardinality choice

kevin-gatimu99e0478a138f
diff --git a/laya-dotnet/src/Laya/LayaShortlist.cs b/laya-dotnet/src/Laya/LayaShortlist.cs
new file mode 100644
index 00000000..19fc6a9d
--- /dev/null
+++ b/laya-dotnet/src/Laya/LayaShortlist.cs
@@ -0,0 +1,304 @@
+namespace Laya;
+
+/// <summary>
+/// Embeds a batch of texts into vectors. Called once per shortlist, with the query text first,
+/// followed by one text per option in criteria order.
+/// </summary>
+/// <remarks>
+/// Mirrors Python's <c>embed_fn: Callable[[Sequence[str]], Any]</c>. Python's return value is
+/// duck-typed (a NumPy array, a list of lists, or a torch tensor via <c>.detach()</c>); this port
+/// asks for a plain <c>double[][]</c> instead, of shape <c>(texts.Count, dim)</c>, so a caller can
+/// implement <see cref="LayaEmbedFunction"/> without taking a dependency on this SDK's own tensor
+/// or NumPy-equivalent types — any bi-encoder wrapper that already returns row vectors can be
+/// adapted with a one-line lambda. Non-finite components are zeroed by
+/// <see cref="LayaShortlist"/> before use, matching Python's <c>np.nan_to_num</c>.
+/// </remarks>
+/// <param name="texts">The texts to embed, in order.</param>
+/// <returns>One row per input text, all rows the same length.</returns>
+public delegate double[][] LayaEmbedFunction(IReadOnlyList<string> texts);
+
+/// <summary>
+/// Shortlist metadata for one choice question: which labels were kept, their similarity scores
+/// (when ranking actually ran), and the <c>k</c>/<c>n</c> it ran under. Mirrors Python's
+/// <c>shortlist[qid]</c> entry.
+/// </summary>
+public sealed class ShortlistInfo
+{
+ internal ShortlistInfo(IReadOnlyList<string> labels, IReadOnlyList<double>? scores, int k, int n, bool passthrough)
+ {
+ Labels = labels;
+ Scores = scores;
+ K = k;
+ N = n;
+ Passthrough = passthrough;
+ }
+
+ /// <summary>The kept labels, in rank order (descending cosine similarity; ties keep the earlier label).</summary>
+ public IReadOnlyList<string> Labels { get; }
+
+ /// <summary>
+ /// Cosine similarity per kept label, aligned with <see cref="Labels"/>, or
+ /// <see langword="null"/> when nothing was dropped (<see cref="Passthrough"/>).
+ /// </summary>
+ public IReadOnlyList<double>? Scores { get; }
+
+ /// <summary>The shortlist budget this question was ranked under.</summary>
+ public int K { get; }
+
+ /// <summary>How many labels the original criteria had.</summary>
+ public int N { get; }
+
+ /// <summary>
+ /// Whether every label was kept without calling <see cref="LayaEmbedFunction"/> at all
+ /// (<c>K &gt;= N</c>).
+ /// </summary>
+ public bool Passthrough { get; }
+}
+
+/// <summary>
+/// Opt-in embedding shortlist for high-cardinality choice questions. Ports <c>laya/shortlist.py</c>.
+/// </summary>
+/// <remarks>
+/// <para>
+/// Choice options share one token budget, so a large label set leaves only a few tokens per
+/// label. <see cref="Predict"/> embeds the state and each option with a caller-supplied
+/// <see cref="LayaEmbedFunction"/>, keeps the top <c>k</c>, and runs a single
+/// <see cref="ILayaPredictor.Predict"/> on that reduced criteria set.
+/// </para>
+/// <para>
+/// This does not change the decision model's forward pass and does not add a second model call
+/// for ranking: ranking is pure cosine similarity on whatever vectors the caller's embedder
+/// returns. Porting Python's <c>embed_fn_from_agent</c> (mean-pooling the checkpoint's own
+/// encoder) is out of scope for this SDK; pass a real bi-encoder, or the deterministic
+/// hashing embedder used in this SDK's tests and samples for a dependency-free demo.
+/// </para>
+/// </remarks>
+public static class LayaShortlist
+{
+ /// <summary>Default shortlist size, matching Python's <c>DEFAULT_SHORTLIST_K</c>.</summary>
+ public const int DefaultShortlistK = 20;
+
+ /// <summary>
+ /// Return the top-<paramref name="k"/> choice labels for <paramref name="state"/>, ranked by
+ /// cosine similarity between <paramref name="state"/> (plus optional
+ /// <paramref name="instructions"/>) and each option's rendered text.
+ /// </summary>
+ /// <remarks>
+ /// <see cref="LayaEmbedFunction"/> is called once, with the query text first and then one
+ /// string per option in criteria order. When <paramref name="k"/> is at least the number of
+ /// labels, every label is returned in its original order and the embedder is not called.
+ /// Ties keep the earlier label; a zero vector scores 0 and does not outrank a label that came
+ /// before it.
+ /// </remarks>
+ public static IReadOnlyList<string> ShortlistChoice(
+ object? state,
+ IEnumerable<KeyValuePair<string, object?>> criteria,
+ LayaEmbedFunction embedFn,
+ int k = DefaultShortlistK,
+ object? instructions = null)
+ {
+ ArgumentNullException.ThrowIfNull(criteria);
+ ArgumentNullException.ThrowIfNull(embedFn);
+ var checkedK = CheckK(k);
+ // "" is a valid placeholder: RenderOptions never reads a choice question's Instructions.
+ var question = Question.Choice("", criteria);
+ var (labels, _, _, _) = Rank(state, question, embedFn, checkedK, instructions);
+ return labels;
+ }
+
+ /// <summary>
+ /// <see cref="ShortlistChoice(object, IEnumerable{KeyValuePair{string, object}}, LayaEmbedFunction, int, object)"/>
+ /// over bare labels, with no descriptions.
+ /// </summary>
+ public static IReadOnlyList<string> ShortlistChoice(
+ object? state,
+ IEnumerable<string> labels,
+ LayaEmbedFunction embedFn,
+ int k = DefaultShortlistK,
+ object? instructions = null)
+ {
+ ArgumentNullException.ThrowIfNull(labels);
+ return ShortlistChoice(
+ state, labels.Select(l => new KeyValuePair<string, object?>(l, null)), embedFn, k, instructions);
+ }
+
+ /// <summary>
+ /// Shortlist each choice question in <paramref name="questions"/>, then call
+ /// <paramref name="predictor"/> once over the reduced set.
+ /// </summary>
+ /// <remarks>
+ /// <para>
+ /// Non-choice questions are forwarded unchanged. A choice whose label count is <c>&lt;= k</c>
+ /// is forwarded unchanged and does not call <paramref name="embedFn"/>. <paramref name="questions"/>
+ /// itself is never mutated.
+ /// </para>
+ /// <para>
+ /// The returned <see cref="LayaResult"/> is whatever <paramref name="predictor"/> returned,
+ /// plus <see cref="LayaResult.Shortlist"/>. Probabilities on a shortlisted choice are over the
+ /// kept labels only. When <paramref name="predictor"/> is a <see cref="LayaRouter"/>, the
+ /// result carries both <see cref="LayaResult.Routing"/> and <see cref="LayaResult.Shortlist"/>.
+ /// </para>
+ /// </remarks>
+ /// <exception cref="ArgumentException">
+ /// If a choice question in <paramref name="questions"/> is missing <see cref="ChoiceQuestion"/>
+ /// options, or <paramref name="embedFn"/> returns a malformed matrix.
+ /// </exception>
+ public static LayaResult Predict(
+ ILayaPredictor predictor,
+ object? state,
+ QuestionSet questions,
+ LayaEmbedFunction embedFn,
+ int k = DefaultShortlistK)
+ {
+ ArgumentNullException.ThrowIfNull(predictor);
+ ArgumentNullException.ThrowIfNull(questions);
+ ArgumentNullException.ThrowIfNull(embedFn);
+ var checkedK = CheckK(k);
+
+ var reduced = new QuestionSet();
+ var meta = new Dictionary<string, ShortlistInfo>(StringComparer.Ordinal);
+
+ foreach (var (qid, question) in questions)
+ {
+ if (question is not ChoiceQuestion choice)
+ {
+ reduced[qid] = question;
+ continue;
+ }
+
+ var (labels, scores, passthrough, n) = Rank(state, choice, embedFn, checkedK, choice.Instructions);
+ meta[qid] = new ShortlistInfo(labels, scores, checkedK, n, passthrough);
+ if (passthrough)
+ {
+ reduced[qid] = question;
+ continue;
+ }
+ reduced[qid] = Question.Choice(choice.Instructions, SubsetOptions(choice.Options, labels));
+ }
+
+ var result = predictor.Predict(state, reduced);
+ return result.WithShortlist(meta);
+ }
+
+ // ── ranking ───────────────────────────────────────────────────────────────
+
+ private static (List<string> Labels, List<double>? Scores, bool Passthrough, int N) Rank(
+ object? state, ChoiceQuestion question, LayaEmbedFunction embedFn, int k, object? instructions)
+ {
+ var keys = question.Options.Select(o => o.Key).ToList();
+ var n = keys.Count;
+ if (k >= n) return (keys, null, true, n);
+
+ var query = QueryText(state, instructions);
+ var optionTexts = SequenceBuilder.RenderOptions(question);
+ var texts = new List<string>(n + 1) { query };
+ texts.AddRange(optionTexts);
+
+ var matrix = Embeddings(embedFn, texts);
+ var docs = new double[n][];
+ Array.Copy(matrix, 1, docs, 0, n);
+ var sims = Cosine(matrix[0], docs);
+
+ // Stable descending sort: ties keep the earlier label, matching NumPy's
+ // np.argsort(-sims, kind="mergesort") — LINQ's OrderBy family is documented stable.
+ var order = Enumerable.Range(0, n).OrderByDescending(i => sims[i]).Take(k).ToList();
+ var labels = order.Select(i => keys[i]).ToList();
+ var scores = order.Select(i => sims[i]).ToList();
+ return (labels, scores, false, n);
+ }
+
+ private static IEnumerable<KeyValuePair<string, object?>> SubsetOptions(
+ IReadOnlyList<KeyValuePair<string, object?>> options, IReadOnlyList<string> labels)
+ {
+ var byLabel = options.ToDictionary(o => o.Key, o => o.Value, StringComparer.Ordinal);
+ foreach (var label in labels) yield return new(label, byLabel[label]);
+ }
+
+ /// <summary>
+ /// Query text: <c>instructions + "\n" + state</c> when instructions is present and non-empty,
+ /// else just the serialized state. A non-string instructions value is dumped via
+ /// <see cref="PythonJson.State"/> (Python's <c>json.dumps(..., ensure_ascii=False)</c>, no
+ /// <c>default=str</c> fallback) — the same dialect <c>PythonJson.State</c> already uses for a
+ /// non-string state, so no separate dialect was needed for this.
+ /// </summary>
+ private static string QueryText(object? state, object? instructions)
+ {
+ var body = PythonJson.State(state);
+ if (instructions is null) return body;
+ if (instructions is string { Length: 0 }) return body;
+ return PythonJson.State(instructions) + "\n" + body;
+ }
+
+ private static int CheckK(int k)
+ {
+ if (k < 1) throw new ArgumentOutOfRangeException(nameof(k), k, "k must be a positive integer");
+ return k;
+ }
+
+ // ── embedding + cosine ───────────────────────────────────────────────────
+
+ private static double[][] Embeddings(LayaEmbedFunction embedFn, IReadOnlyList<string> texts)
+ {
+ var raw = embedFn(texts)
+ ?? throw new ArgumentException(
+ $"embed_fn must return an array of shape ({texts.Count}, dim), got null", nameof(embedFn));
+ if (raw.Length != texts.Count)
+ throw new ArgumentException(
+ $"embed_fn must return an array of shape ({texts.Count}, dim), got ({raw.Length}, ...)",
+ nameof(embedFn));
+
+ var dim = raw.Length > 0 ? raw[0]?.Length ?? -1 : -1;
+ if (dim < 1)
+ throw new ArgumentException(
+ $"embed_fn must return an array of shape ({texts.Count}, dim) with dim >= 1", nameof(embedFn));
+
+ var cleaned = new double[raw.Length][];
+ for (var i = 0; i < raw.Length; i++)
+ {
+ var row = raw[i]
+ ?? throw new ArgumentException($"embed_fn returned a null row at index {i}", nameof(embedFn));
+ if (row.Length != dim)
+ throw new ArgumentException(
+ $"embed_fn must return an array of shape ({texts.Count}, dim); row {i} has length "
+ + $"{row.Length}, expected {dim}", nameof(embedFn));
+
+ var outRow = new double[dim];
+ for (var j = 0; j < dim; j++)
+ {
+ var v = row[j];
+ // np.nan_to_num: only non-finite COMPONENTS are zeroed, not the whole vector.
+ outRow[j] = double.IsNaN(v) || double.IsInfinity(v) ? 0.0 : v;
+ }
+ cleaned[i] = outRow;
+ }
+ return cleaned;
+ }
+
+ private static double[] Cosine(double[] query, double[][] docs)
+ {
+ var sims = new double[docs.Length];
+ var qn = Norm(query);
+ if (qn == 0.0) return sims;
+
+ for (var i = 0; i < docs.Length; i++)
+ {
+ var denom = Norm(docs[i]) * qn;
+ if (denom > 0.0) sims[i] = Dot(docs[i], query) / denom;
+ }
+ return sims;
+ }
+
+ private static double Norm(double[] v)
+ {
+ var sum = 0.0;
+ foreach (var x in v) sum += x * x;
+ return Math.Sqrt(sum);
+ }
+
+ private static double Dot(double[] a, double[] b)
+ {
+ var sum = 0.0;
+ for (var i = 0; i < a.Length; i++) sum += a[i] * b[i];
+ return sum;
+ }
+}
diff --git a/laya-dotnet/tests/Laya.Tests/HashingEmbedder.cs b/laya-dotnet/tests/Laya.Tests/HashingEmbedder.cs
new file mode 100644
index 00000000..bf9a87ed
--- /dev/null
+++ b/laya-dotnet/tests/Laya.Tests/HashingEmbedder.cs
@@ -0,0 +1,82 @@
+using System.Text;
+
+namespace Laya.Tests;
+
+/// <summary>
+/// Deterministic, dependency-free demo embedder for shortlist tests. Ports
+/// <c>laya-dotnet/tools/hashing_embedder.py</c> bit-for-bit: a hashed character-trigram bag-of-features
+/// vector, chosen only because it is a pure function of the input text and is trivial to
+/// reimplement exactly with nothing but a string, UTF-8 encoding and 32-bit integer arithmetic.
+/// <b>This is not a real embedding model</b> — it exists solely so the golden vectors and this
+/// SDK's tests can pin an exact, reproducible <see cref="LayaEmbedFunction"/> without a real
+/// bi-encoder or a Hub download. See <c>hashing_embedder_probe.json</c> and
+/// <c>case_shortlist_many_options.json</c>, which were both recorded with this exact algorithm.
+/// </summary>
+/// <remarks>
+/// Algorithm (mirrors the Python docstring precisely):
+/// <list type="number">
+/// <item>Lowercase with full Unicode case mapping — <see cref="LanguageDetection.PythonLower"/>,
+/// not <see cref="string.ToLowerInvariant()"/> directly, because this project builds with
+/// <c>InvariantGlobalization</c> enabled, under which plain <c>ToLowerInvariant</c> leaves U+0130
+/// (Turkish capital I with dot above) untouched instead of expanding it to "i" + combining dot,
+/// as Python's <c>str.lower()</c> does.</item>
+/// <item>Pad with two ASCII spaces on each side.</item>
+/// <item>Slide a window of exactly 3 Unicode code points (via <see cref="System.Text.Rune"/>,
+/// never UTF-16 code units) across the padded text, one code point at a time.</item>
+/// <item>UTF-8-encode each window's own 3 code points (re-encoded fresh each time, not sliced
+/// from the whole padded text's byte buffer).</item>
+/// <item>Hash those bytes with 32-bit FNV-1a (unsigned, wrapping multiply).</item>
+/// <item><c>bucket = hash % 64</c>.</item>
+/// <item><c>sign = +1</c> when the hash's top bit is 0, else <c>-1</c>.</item>
+/// <item>Accumulate <c>sign</c> into <c>vector[bucket]</c> for every window (float64 throughout).</item>
+/// <item>No normalisation.</item>
+/// </list>
+/// </remarks>
+public static class HashingEmbedder
+{
+ /// <summary>Vector width, matching Python's <c>DIM</c>.</summary>
+ public const int Dimension = 64;
+
+ private const uint FnvOffsetBasis = 0x811c9dc5;
+ private const uint FnvPrime = 0x01000193;
+
+ /// <summary>Embed a batch of texts, one <see cref="Dimension"/>-length vector per input, in order.</summary>
+ public static double[][] Embed(IReadOnlyList<string> texts)
+ {
+ ArgumentNullException.ThrowIfNull(texts);
+ var result = new double[texts.Count][];
+ for (var i = 0; i < texts.Count; i++) result[i] = EmbedOne(texts[i]);
+ return result;
+ }
+
+ /// <summary>Embed a single text into a <see cref="Dimension"/>-length vector.</summary>
+ public static double[] EmbedOne(string? text)
+ {
+ var vector = new double[Dimension];
+ var padded = " " + LanguageDetection.PythonLower(text ?? string.Empty) + " ";
+ var runes = padded.EnumerateRunes().ToArray();
+ var windowCount = runes.Length - 2; // always >= 2: padding guarantees at least 4 code points
+
+ for (var i = 0; i < windowCount; i++)
+ {
+ var window = string.Concat(runes[i].ToString(), runes[i + 1].ToString(), runes[i + 2].ToString());
+ var hash = Fnv1a32(Encoding.UTF8.GetBytes(window));
+ var bucket = (int)(hash % Dimension);
+ var sign = (hash >> 31 & 1) == 0 ? 1.0 : -1.0;
+ vector[bucket] += sign;
+ }
+ return vector;
+ }
+
+ /// <summary>32-bit FNV-1a over raw bytes, wrapping like an unsigned 32-bit integer.</summary>
+ private static uint Fnv1a32(byte[] data)
+ {
+ var hash = FnvOffsetBasis;
+ foreach (var b in data)
+ {
+ hash ^= b;
+ hash *= FnvPrime; // uint * uint wraps silently (unchecked) by default in C#
+ }
+ return hash;
+ }
+}
diff --git a/laya-dotnet/tests/Laya.Tests/HashingEmbedderTests.cs b/laya-dotnet/tests/Laya.Tests/HashingEmbedderTests.cs
new file mode 100644
index 00000000..7784e6dd
--- /dev/null
+++ b/laya-dotnet/tests/Laya.Tests/HashingEmbedderTests.cs
@@ -0,0 +1,64 @@
+using System.Text.Json;
+
+namespace Laya.Tests;
+
+/// <summary>
+/// Tier 1: the hashing embedder against every entry of
+/// <c>golden/routing/hashing_embedder_probe.json</c>, which was generated by
+/// <c>laya-dotnet/tools/hashing_embedder.py</c>. Every language's port must agree bit-for-bit, because the
+/// shortlist goldens and every SDK's sample were recorded with it.
+/// </summary>
+public sealed class HashingEmbedderTests
+{
+ public static IEnumerable<object[]> ProbeCases()
+ {
+ using var doc = RoutingGoldenData.Load("hashing_embedder_probe.json");
+ // Index by position, not by text: several probe entries share the same text after
+ // trimming display, and whitespace-only text has no useful theory-case label anyway.
+ return
+ [
+ .. Enumerable.Range(0, doc.RootElement.GetArrayLength()).Select(i => new object[] { i }),
+ ];
+ }
+
+ [Theory]
+ [MemberData(nameof(ProbeCases))]
+ public void EveryProbeEntry(int index)
+ {
+ using var doc = RoutingGoldenData.Load("hashing_embedder_probe.json");
+ var entry = doc.RootElement[index];
+ var text = entry.GetProperty("text").GetString()!;
+ var expected = entry.GetProperty("vector").EnumerateArray().Select(e => e.GetDouble()).ToArray();
+
+ var actual = HashingEmbedder.EmbedOne(text);
+
+ Assert.Equal(HashingEmbedder.Dimension, actual.Length);
+ Assert.Equal(expected.Length, actual.Length);
+ for (var i = 0; i < expected.Length; i++)
+ Assert.True(expected[i] == actual[i],
+ $"text {JsonSerializer.Serialize(text)} bucket {i}: python {expected[i]}, dotnet {actual[i]}");
+ }
+
+ [Fact]
+ public void EmbedBatchesInOrderAndMatchesEmbedOne()
+ {
+ var texts = new[] { "alpha", "beta", "gamma" };
+ var batch = HashingEmbedder.Embed(texts);
+ Assert.Equal(texts.Length, batch.Length);
+ for (var i = 0; i < texts.Length; i++)
+ Assert.Equal(HashingEmbedder.EmbedOne(texts[i]), batch[i]);
+ }
+
+ [Fact]
+ public void EmptyBatchReturnsEmptyResult() => Assert.Empty(HashingEmbedder.Embed([]));
+
+ [Fact]
+ public void TurkishCapitalIWithDotUsesTheTwoCodePointLowercasing()
+ {
+ // Regression guard for the InvariantGlobalization pitfall documented on HashingEmbedder:
+ // plain ToLowerInvariant() would leave U+0130 untouched instead of expanding it.
+ var istanbul = HashingEmbedder.EmbedOne("İstanbul");
+ var alreadyLower = HashingEmbedder.EmbedOne("i̇stanbul"); // "i" + combining dot above
+ Assert.Equal(alreadyLower, istanbul);
+ }
+}
diff --git a/laya-dotnet/tests/Laya.Tests/LayaShortlistTests.cs b/laya-dotnet/tests/Laya.Tests/LayaShortlistTests.cs
new file mode 100644
index 00000000..f4b60078
--- /dev/null
+++ b/laya-dotnet/tests/Laya.Tests/LayaShortlistTests.cs
@@ -0,0 +1,439 @@
+using System.Text.Json;
+
+namespace Laya.Tests;
+
+/// <summary>
+/// Tier 1: the embedding shortlist, which needs no artifacts. A mock <see cref="ILayaPredictor"/>
+/// stands in for the decision model, so these tests never construct one. Named cases mirror
+/// <c>tests/test_shortlist.py</c> (excluding the torch mean-pool block, which ports Python's
+/// <c>embed_fn_from_agent</c> — out of scope per the porting plan); <see cref="EveryShortlistProbeEntry"/>
+/// consumes every entry of <c>golden/routing/shortlist_probe.json</c>.
+/// </summary>
+public sealed class LayaShortlistTests
+{
+ // Vectors: query [1, 0]. alpha and delta tie at cosine 1; gamma is 0.6; beta is 0.
+ // Stable order must keep alpha ahead of delta.
+ private static readonly Dictionary<string, object?> Criteria = new(StringComparer.Ordinal)
+ {
+ ["alpha"] = null, ["beta"] = "", ["gamma"] = "mid", ["delta"] = "same",
+ };
+
+ private static readonly Dictionary<string, double[]> OptionVectors = new(StringComparer.Ordinal)
+ {
+ ["alpha"] = [1.0, 0.0],
+ ["beta"] = [0.0, 1.0],
+ ["gamma: mid"] = [0.6, 0.8],
+ ["delta: same"] = [1.0, 0.0],
+ };
+
+ private sealed class TableEmbed(IReadOnlyDictionary<string, double[]> vectors)
+ {
+ public List<List<string>> Calls { get; } = [];
+
+ public double[][] Embed(IReadOnlyList<string> texts)
+ {
+ Calls.Add([.. texts]);
+ var missing = texts.Where(t => !vectors.ContainsKey(t)).ToList();
+ if (missing.Count > 0) throw new InvalidOperationException($"unexpected texts: {string.Join(", ", missing)}");
+ return [.. texts.Select(t => vectors[t])];
+ }
+ }
+
+ private static LayaEmbedFunction ForQuery(string queryText, IReadOnlyDictionary<string, double[]> optionVectors)
+ {
+ var full = new Dictionary<string, double[]>(optionVectors, StringComparer.Ordinal) { [queryText] = [1.0, 0.0] };
+ return new TableEmbed(full).Embed;
+ }
+
+ // ── deterministic top-k ──────────────────────────────────────────────────
+
+ [Fact]
+ public void TopKKeepsTheCosineTieInInputOrder()
+ {
+ var embed = new TableEmbed(new Dictionary<string, double[]>(OptionVectors, StringComparer.Ordinal) { ["pay me"] = [1.0, 0.0] });
+ var labels = LayaShortlist.ShortlistChoice("pay me", Criteria, embed.Embed, k: 2);
+ Assert.Equal(["alpha", "delta"], labels);
+ Assert.Single(embed.Calls);
+ Assert.Equal("pay me", embed.Calls[0][0]);
+ Assert.Equal(["alpha", "beta", "gamma: mid", "delta: same"], embed.Calls[0].Skip(1));
+ }
+
+ [Fact]
+ public void K1IsTheEarliestMax()
+ {
+ var embed = ForQuery("pay me", OptionVectors);
+ Assert.Equal(["alpha"], LayaShortlist.ShortlistChoice("pay me", Criteria, embed, k: 1));
+ }
+
+ [Fact]
+ public void K3AppendsTheNextCosine()
+ {
+ var embed = ForQuery("pay me", OptionVectors);
+ Assert.Equal(["alpha", "delta", "gamma"], LayaShortlist.ShortlistChoice("pay me", Criteria, embed, k: 3));
+ }
+
+ [Fact]
+ public void ZeroQueryKeepsOriginalOrder()
+ {
+ // Every cosine is 0 when the query vector is 0, so the earliest labels win.
+ var vectors = new Dictionary<string, double[]>(StringComparer.Ordinal)
+ {
+ ["pay me"] = [0.0, 0.0],
+ ["alpha"] = [1.0, 0.0],
+ ["beta"] = [0.0, 1.0],
+ ["gamma: mid"] = [0.6, 0.8],
+ ["delta: same"] = [3.0, 4.0],
+ };
+ var embed = new TableEmbed(vectors);
+ Assert.Equal(["alpha", "beta"], LayaShortlist.ShortlistChoice("pay me", Criteria, embed.Embed, k: 2));
+ }
+
+ [Fact]
+ public void NanVectorSortsBehindAFiniteMatch()
+ {
+ // A non-finite option vector is treated as 0 and loses to a real match.
+ var vectors = new Dictionary<string, double[]>(StringComparer.Ordinal)
+ {
+ ["pay me"] = [1.0, 0.0],
+ ["alpha"] = [double.NaN, double.NaN],
+ ["beta"] = [1.0, 0.0],
+ };
+ var embed = new TableEmbed(vectors);
+ var criteria = new Dictionary<string, object?> { ["alpha"] = null, ["beta"] = null };
+ Assert.Equal(["beta"], LayaShortlist.ShortlistChoice("pay me", criteria, embed.Embed, k: 1));
+ }
+
+ // ── list criteria and instructions ───────────────────────────────────────
+
+ [Fact]
+ public void InstructionsChangeTheQueryAndTheWinner()
+ {
+ var vectors = new Dictionary<string, double[]>(StringComparer.Ordinal)
+ {
+ ["Classify\npay me"] = [0.0, 1.0],
+ ["alpha"] = [1.0, 0.0],
+ ["beta"] = [0.0, 1.0],
+ ["gamma"] = [0.0, 0.2],
+ };
+ var embed = new TableEmbed(vectors);
+ var labels = LayaShortlist.ShortlistChoice("pay me", ["alpha", "beta", "gamma"], embed.Embed, k: 2, instructions: "Classify");
+ Assert.Equal(["beta", "gamma"], labels);
+ Assert.Equal("Classify\npay me", embed.Calls[0][0]);
+ Assert.Equal(["alpha", "beta", "gamma"], embed.Calls[0].Skip(1));
+ }
+
+ [Fact]
+ public void DictStateIsSerialized()
+ {
+ var state = new Dictionary<string, object?> { ["text"] = "hi" };
+ var vectors = new Dictionary<string, double[]>(StringComparer.Ordinal)
+ {
+ ["Classify\n{\"text\": \"hi\"}"] = [1.0, 0.0],
+ ["alpha"] = [1.0, 0.0],
+ ["beta"] = [0.0, 1.0],
+ };
+ var embed = new TableEmbed(vectors);
+ var labels = LayaShortlist.ShortlistChoice(state, ["alpha", "beta"], embed.Embed, k: 1, instructions: "Classify");
+ Assert.Equal(["alpha"], labels);
+ Assert.Equal("Classify\n{\"text\": \"hi\"}", embed.Calls[0][0]);
+ }
+
+ [Fact]
+ public void ZeroAndFalseCriterionValuesStayInTheOptionText()
+ {
+ // 0 and False are real criterion values, so they are part of the embedded text.
+ var rich = new Dictionary<string, object?>
+ {
+ ["zero"] = 0L, ["no"] = false, ["bare"] = null,
+ ["named"] = new Dictionary<string, object?> { ["desc"] = "payments" },
+ };
+ var richRendered = SequenceBuilder.RenderOptions(Question.Choice("", rich));
+ var vectors = new Dictionary<string, double[]>(StringComparer.Ordinal) { ["pay me"] = [1.0, 0.0] };
+ foreach (var t in richRendered) vectors[t] = [1.0, 0.0];
+ var embed = new TableEmbed(vectors);
+ LayaShortlist.ShortlistChoice("pay me", rich, embed.Embed, k: 1);
+ Assert.Equal(richRendered, embed.Calls[0].Skip(1));
+ }
+
+ // ── k >= n pass-through ───────────────────────────────────────────────────
+
+ [Fact]
+ public void KEqualsNReturnsEveryLabelInOrderWithoutCallingEmbedFn()
+ {
+ LayaEmbedFunction boom = _ => throw new InvalidOperationException("embed_fn should not run when k >= n");
+ Assert.Equal([.. Criteria.Keys], LayaShortlist.ShortlistChoice("pay me", Criteria, boom, k: 4));
+ }
+
+ [Fact]
+ public void KGreaterThanNReturnsEveryLabelInOrderWithoutCallingEmbedFn()
+ {
+ LayaEmbedFunction boom = _ => throw new InvalidOperationException("embed_fn should not run when k >= n");
+ Assert.Equal([.. Criteria.Keys], LayaShortlist.ShortlistChoice("pay me", Criteria, boom, k: 20));
+ }
+
+ // ── Predict: mock predictor sees only k criteria ─────────────────────────
+
+ private sealed class Recorder : ILayaPredictor
+ {
+ public List<(object? State, QuestionSet Questions)> Calls { get; } = [];
+
+ public LayaResult Predict(object? state, QuestionSet questions)
+ {
+ Calls.Add((state, questions));
+ var order = new List<string>();
+ var byId = new Dictionary<string, Answer>(StringComparer.Ordinal);
+ foreach (var (qid, q) in questions)
+ {
+ order.Add(qid);
+ if (q is ChoiceQuestion choice)
+ {
+ var labels = choice.Labels;
+ var probs = labels.Select((l, i) => new KeyValuePair<string, double>(l, i == 0 ? 1.0 : 0.0)).ToList();
+ byId[qid] = new ChoiceAnswer(labels[0], probs, 1.0, 1.0, new ActionInfo(1.0));
+ }
+ }
+ return new LayaResult("fake", order, byId, new Usage(0, 0));
+ }
+ }
+
+ [Fact]
+ public void PredictShortlistsChoiceCriteriaToTheTopK()
+ {
+ var sentinel = new Dictionary<string, object?> { ["desc"] = "payments" };
+ var full = new Dictionary<string, object?> { ["billing"] = sentinel, ["tech"] = "bugs", ["sales"] = null, ["other"] = "misc" };
+ var vectors = new Dictionary<string, double[]>(StringComparer.Ordinal)
+ {
+ ["Which desk?\nI was charged twice"] = [1.0, 0.0],
+ ["billing: {\"desc\": \"payments\"}"] = [0.0, 1.0],
+ ["tech: bugs"] = [1.0, 0.0],
+ ["sales"] = [0.2, 0.2],
+ ["other: misc"] = [0.0, 1.0],
+ };
+ var embed = new TableEmbed(vectors);
+ var scoreQ = Question.Score("How urgent?", "low", "mid", "high", "now");
+ var noulQ = Question.Noul("Is a refund requested?");
+ var questions = new QuestionSet
+ {
+ ["intent"] = Question.Choice("Which desk?", full),
+ ["urgency"] = scoreQ,
+ ["refund"] = noulQ,
+ };
+ var recorder = new Recorder();
+
+ var result = LayaShortlist.Predict(recorder, "I was charged twice", questions, embed.Embed, k: 2);
+
+ Assert.Single(recorder.Calls);
+ var gotQuestions = recorder.Calls[0].Questions;
+ Assert.Same("I was charged twice", recorder.Calls[0].State);
+ var reducedIntent = (ChoiceQuestion)gotQuestions["intent"];
+ Assert.Equal(["tech", "sales"], reducedIntent.Labels);
+ Assert.Equal("bugs", reducedIntent.Options.First(o => o.Key == "tech").Value);
+ Assert.Same(scoreQ, gotQuestions["urgency"]);
+ Assert.Same(noulQ, gotQuestions["refund"]);
+
+ // The caller's original question and criteria are untouched.
+ Assert.Equal(4, ((ChoiceQuestion)questions["intent"]).Options.Count);
+ Assert.Equal(sentinel, full["billing"]);
+
+ Assert.Equal("tech", result["intent"].AsChoice().Choice);
+ Assert.Equal(["tech", "sales"], result.Shortlist!["intent"].Labels);
+ Assert.True(result.Shortlist["intent"].Scores![0] > result.Shortlist["intent"].Scores![1]);
+ Assert.True(result.Shortlist["intent"].Scores![1] > 0);
+ Assert.Equal(2, result.Shortlist["intent"].K);
+ Assert.Equal(4, result.Shortlist["intent"].N);
+ Assert.False(result.Shortlist["intent"].Passthrough);
+ Assert.False(result.Shortlist.ContainsKey("urgency"));
+ }
+
+ [Fact]
+ public void PredictReturnsACopyRatherThanMutatingThePredictorsResult()
+ {
+ var holding = new HoldingPredictor();
+ var criteria = new Dictionary<string, object?>(Criteria);
+ var out1 = LayaShortlist.Predict(holding, "pay me", new QuestionSet { ["intent"] = Question.Choice("", criteria) },
+ _ => throw new InvalidOperationException(), k: 4);
+ Assert.NotNull(out1.Shortlist);
+ Assert.Null(holding.Held?.Shortlist);
+ }
+
+ private sealed class HoldingPredictor : ILayaPredictor
+ {
+ public LayaResult? Held;
+ public LayaResult Predict(object? state, QuestionSet questions) =>
+ Held = new LayaResult("fake", [], new Dictionary<string, Answer>(StringComparer.Ordinal), new Usage(0, 0));
+ }
+
+ [Fact]
+ public void KGreaterThanOrEqualNReachesPredictUnchanged()
+ {
+ var full = new Dictionary<string, object?> { ["billing"] = null, ["tech"] = "bugs", ["sales"] = null, ["other"] = "misc" };
+ var originalQuestion = Question.Choice("Which desk?", full);
+ var recorder = new Recorder();
+ var out1 = LayaShortlist.Predict(recorder, "I was charged twice",
+ new QuestionSet { ["intent"] = originalQuestion },
+ _ => throw new InvalidOperationException("embed_fn should not run"), k: 4);
+
+ Assert.Same(originalQuestion, recorder.Calls[0].Questions["intent"]);
+ Assert.Equal([.. full.Keys], out1.Shortlist!["intent"].Labels);
+ Assert.Null(out1.Shortlist["intent"].Scores);
+ Assert.True(out1.Shortlist["intent"].Passthrough);
+ }
+
+ [Fact]
+ public void ListCriteriaStayInRankOrderAsAList()
+ {
+ var vectors = new Dictionary<string, double[]>(StringComparer.Ordinal)
+ {
+ ["Which?\nhello"] = [1.0, 0.0],
+ ["alpha"] = [0.0, 1.0],
+ ["beta"] = [1.0, 0.0],
+ ["gamma"] = [0.0, 0.0],
+ };
+ var embed = new TableEmbed(vectors);
+ var recorder = new Recorder();
+ var questions = new QuestionSet { ["intent"] = Question.Choice("Which?", "alpha", "beta", "gamma") };
+ LayaShortlist.Predict(recorder, "hello", questions, embed.Embed, k: 2);
+
+ var received = (ChoiceQuestion)recorder.Calls[0].Questions["intent"];
+ Assert.Equal(["beta", "alpha"], received.Labels);
+ // The caller's own question is untouched.
+ Assert.Equal(["alpha", "beta", "gamma"], ((ChoiceQuestion)questions["intent"]).Labels);
+ }
+
+ // ── errors ────────────────────────────────────────────────────────────────
+
+ [Fact]
+ public void KZeroThrows() => Assert.Throws<ArgumentOutOfRangeException>(
+ () => LayaShortlist.ShortlistChoice("pay me", Criteria, ForQuery("pay me", OptionVectors), k: 0));
+
+ [Fact]
+ public void KNegativeThrows() => Assert.Throws<ArgumentOutOfRangeException>(
+ () => LayaShortlist.ShortlistChoice("pay me", Criteria, ForQuery("pay me", OptionVectors), k: -3));
+
+ [Fact]
+ public void EmptyDictCriteriaThrows() => Assert.Throws<ArgumentException>(
+ () => LayaShortlist.ShortlistChoice("pay me", new Dictionary<string, object?>(), ForQuery("pay me", OptionVectors), k: 1));
+
+ [Fact]
+ public void EmptyListCriteriaThrows() => Assert.Throws<ArgumentException>(
+ () => LayaShortlist.ShortlistChoice("pay me", Array.Empty<string>(), ForQuery("pay me", OptionVectors), k: 1));
+
+ [Fact]
+ public void DuplicateLabelThrows() => Assert.Throws<ArgumentException>(
+ () => LayaShortlist.ShortlistChoice("pay me", ["alpha", "alpha"], ForQuery("pay me", OptionVectors), k: 1));
+
+ [Fact]
+ public void MissingCriteriaOnAChoiceQuestionThrows()
+ {
+ // A choice question with no ChoiceQuestion.Options cannot exist in the .NET model
+ // (construction itself validates), so the analogous failure here is empty criteria.
+ Assert.Throws<ArgumentException>(() => Question.Choice("x", new Dictionary<string, object?>()));
+ }
+
+ private static readonly double[][] BadShapeVectors = [[0.0, 0.0, 0.0, 0.0]];
+
+ [Fact]
+ public void BadEmbedShapeThrowsAndNeverCallsPredict()
+ {
+ var recorder = new Recorder();
+ var full = new Dictionary<string, object?> { ["billing"] = null, ["tech"] = "bugs", ["sales"] = null, ["other"] = "misc" };
+ var questions = new QuestionSet { ["intent"] = Question.Choice("Which desk?", full) };
+ Assert.Throws<ArgumentException>(() =>
+ LayaShortlist.Predict(recorder, "pay me", questions, _ => BadShapeVectors, k: 2));
+ Assert.Empty(recorder.Calls);
+ }
+
+ // ── every recorded probe entry ────────────────────────────────────────────
+
+ public static IEnumerable<object[]> ShortlistProbeCases()
+ {
+ using var doc = RoutingGoldenData.Load("shortlist_probe.json");
+ return [.. doc.RootElement.EnumerateArray().Select(e => new object[] { e.GetProperty("label").GetString()! })];
+ }
+
+ private static JsonElement FindEntry(string label)
+ {
+ using var doc = RoutingGoldenData.Load("shortlist_probe.json");
+ foreach (var e in doc.RootElement.EnumerateArray())
+ if (e.GetProperty("label").GetString() == label)
+ return JsonDocument.Parse(e.GetRawText()).RootElement;
+ throw new InvalidOperationException($"no shortlist_probe.json entry labelled '{label}'");
+ }
+
+ [Theory]
+ [MemberData(nameof(ShortlistProbeCases))]
+ public void EveryShortlistProbeEntry(string label)
+ {
+ var e = FindEntry(label);
+ var state = RoutingGoldenData.ToClr(e.GetProperty("state"));
+ var k = e.GetProperty("k").GetInt32();
+ var instructionsProp = e.GetProperty("instructions");
+ var instructions = instructionsProp.ValueKind == JsonValueKind.Null ? null : RoutingGoldenData.ToClr(instructionsProp);
+
+ var criteriaElement = e.GetProperty("criteria");
+ var criteria = criteriaElement.ValueKind == JsonValueKind.Array
+ ? criteriaElement.EnumerateArray().Select(x => new KeyValuePair<string, object?>(x.GetString()!, null)).ToList()
+ : criteriaElement.EnumerateObject()
+ .Select(p => new KeyValuePair<string, object?>(p.Name, RoutingGoldenData.ToClr(p.Value))).ToList();
+
+ var passthrough = e.GetProperty("passthrough").GetBoolean();
+ var wantLabels = e.GetProperty("labels").EnumerateArray().Select(x => x.GetString()!).ToArray();
+ var n = e.GetProperty("n").GetInt32();
+
+ var recorder = new Recorder();
+ var question = Question.Choice("", criteria);
+ var questions = new QuestionSet { ["q"] = question };
+
+ LayaEmbedFunction embed = passthrough
+ ? (_ => throw new InvalidOperationException($"{label}: embed_fn should not run when k >= n"))
+ : texts => ReadVectorsMatrix(e.GetProperty("vectors"), texts.Count);
+
+ var result = LayaShortlist.Predict(recorder, state, questions, embed, k);
+ var info = result.Shortlist!["q"];
+
+ Assert.Equal(wantLabels, info.Labels);
+ Assert.Equal(k, info.K);
+ Assert.Equal(n, info.N);
+ Assert.Equal(passthrough, info.Passthrough);
+
+ if (passthrough)
+ {
+ Assert.Null(info.Scores);
+ }
+ else
+ {
+ var wantScores = e.GetProperty("scores").EnumerateArray().Select(x => x.GetDouble()).ToArray();
+ Assert.NotNull(info.Scores);
+ Assert.Equal(wantScores.Length, info.Scores!.Count);
+ for (var i = 0; i < wantScores.Length; i++)
+ Assert.Equal(wantScores[i], info.Scores[i], 1e-9);
+ }
+ }
+
+ /// <summary>
+ /// Parse a probe "vectors" matrix (first row is the query, remaining rows the options, in
+ /// criteria order). Non-finite components are encoded as the JSON strings "nan"/"inf"/"-inf"
+ /// per <c>meta.json</c>'s notes; <paramref name="expectedRows"/> is asserted against the
+ /// embed_fn call's own text count, mirroring how <c>_rank</c> always calls embed_fn with
+ /// exactly <c>1 + n</c> texts.
+ /// </summary>
+ private static double[][] ReadVectorsMatrix(JsonElement vectors, int expectedRows)
+ {
+ var rows = vectors.EnumerateArray().Select(row =>
+ row.EnumerateArray().Select(ParseComponent).ToArray()).ToArray();
+ Assert.Equal(expectedRows, rows.Length);
+ return rows;
+ }
+
+ private static double ParseComponent(JsonElement e) => e.ValueKind switch
+ {
+ JsonValueKind.String => e.GetString() switch
+ {
+ "nan" => double.NaN,
+ "inf" => double.PositiveInfinity,
+ "-inf" => double.NegativeInfinity,
+ var s => double.Parse(s!, System.Globalization.CultureInfo.InvariantCulture),
+ },
+ _ => e.GetDouble(),
+ };
+}
diff --git a/laya-dotnet/tests/Laya.Tests/ShortlistEndToEndTests.cs b/laya-dotnet/tests/Laya.Tests/ShortlistEndToEndTests.cs
new file mode 100644
index 00000000..34fb694b
--- /dev/null
+++ b/laya-dotnet/tests/Laya.Tests/ShortlistEndToEndTests.cs
@@ -0,0 +1,72 @@
+namespace Laya.Tests;
+
+/// <summary>
+/// Tier 2: the embedding shortlist over a real checkpoint end to end, for all three checkpoints.
+/// Compares <see cref="LayaShortlist.Predict"/> (using <see cref="HashingEmbedder"/>, the same
+/// deterministic demo embedder the golden vector was recorded with) against
+/// <c>case_shortlist_many_options.json</c>'s recorded <c>predict_shortlist</c> result.
+/// </summary>
+/// <remarks>
+/// Each test loads and disposes its own <see cref="LayaEngine"/> rather than sharing
+/// <see cref="AllEnginesFixture"/> (which keeps every checkpoint it has touched resident for the
+/// whole run): the ONNX artifacts are 1.3-1.7 GB apiece, and these tests must not hold more than
+/// one at a time. The class still joins <see cref="AllEnginesCollection"/>, purely so it never
+/// runs concurrently with another engine-loading test class — the native tokenizer binding is not
+/// safe to construct from two threads at once for the same checkpoint.
+/// </remarks>
+[Collection(AllEnginesCollection.Name)]
+public sealed class ShortlistEndToEndTests
+{
+ /// <summary>Tolerance for the shortlist's raw (unrounded) cosine scores.</summary>
+ private const double ScoreTolerance = 1e-9;
+
+ /// <summary>Tolerance for the model's own 4-dp probabilities/confidence, as in <see cref="PredictParityTests"/>.</summary>
+ private const double ProbTolerance = 2e-4;
+
+ public static TheoryData<LayaCheckpoint> Checkpoints => new(Enum.GetValues<LayaCheckpoint>());
+
+ [Theory]
+ [MemberData(nameof(Checkpoints))]
+ public void ShortlistManyOptionsMatchesTheRecordedResult(LayaCheckpoint checkpoint)
+ {
+ var golden = GoldenData.For(checkpoint);
+ if (!golden.Available)
+ Assert.Skip($"no golden data for '{checkpoint}'; run laya-dotnet/tools/dump_golden.py");
+ if (!TestArtifacts.For(checkpoint).HasModel || !TestArtifacts.For(checkpoint).HasTokenizer)
+ Assert.Skip($"no ONNX artifacts for '{checkpoint}'; set LAYA_ONNX_ROOT to the parent of all checkpoints");
+
+ using var doc = golden.Load("case_shortlist_many_options.json");
+ var root = doc.RootElement;
+ var state = GoldenData.ToClr(root.GetProperty("state"));
+ var questions = GoldenData.BuildQuestions(root.GetProperty("questions"));
+ var k = root.GetProperty("shortlist_k").GetInt32();
+
+ using var engine = LayaEngine.FromDirectory(TestArtifacts.For(checkpoint).Directory);
+ var result = LayaShortlist.Predict(engine, state, questions, HashingEmbedder.Embed, k);
+
+ var expected = root.GetProperty("result");
+ Assert.Equal(expected.GetProperty("model").GetString(), result.Model);
+
+ var wantShortlist = expected.GetProperty("shortlist").GetProperty("intent");
+ var gotShortlist = result.Shortlist!["intent"];
+ Assert.Equal(
+ [.. wantShortlist.GetProperty("labels").EnumerateArray().Select(e => e.GetString()!)],
+ gotShortlist.Labels);
+ Assert.Equal(wantShortlist.GetProperty("k").GetInt32(), gotShortlist.K);
+ Assert.Equal(wantShortlist.GetProperty("n").GetInt32(), gotShortlist.N);
+ Assert.Equal(wantShortlist.GetProperty("passthrough").GetBoolean(), gotShortlist.Passthrough);
+
+ var wantScores = wantShortlist.GetProperty("scores").EnumerateArray().Select(e => e.GetDouble()).ToArray();
+ Assert.NotNull(gotShortlist.Scores);
+ Assert.Equal(wantScores.Length, gotShortlist.Scores!.Count);
+ for (var i = 0; i < wantScores.Length; i++)
+ Assert.Equal(wantScores[i], gotShortlist.Scores[i], ScoreTolerance);
+
+ var wantAnswer = expected.GetProperty("answers").GetProperty("intent");
+ var gotAnswer = result["intent"].AsChoice();
+ Assert.Equal(wantAnswer.GetProperty("choice").GetString(), gotAnswer.Choice);
+ foreach (var p in wantAnswer.GetProperty("probabilities").EnumerateObject())
+ Assert.Equal(p.Value.GetDouble(), gotAnswer[p.Name], ProbTolerance);
+ Assert.Equal(wantAnswer.GetProperty("confidence").GetDouble(), result["intent"].Confidence, ProbTolerance);
+ }
+}
diff --git a/laya-dotnet/tools/hashing_embedder.py b/laya-dotnet/tools/hashing_embedder.py
new file mode 100644
index 00000000..2311435e
--- /dev/null
+++ b/laya-dotnet/tools/hashing_embedder.py
@@ -0,0 +1,101 @@
+"""Deterministic, dependency-free demo embedder for the shortlist golden vectors and samples.
+
+This is **not** a real embedding model: it is a hashed character-trigram bag-of-features
+vector, chosen only because it is a pure function of the input text and is trivial to
+reimplement bit-for-bit in C# with nothing but a string, UTF-8 encoding and 32-bit
+integer arithmetic. It is deliberately named and documented as a demo: any real deployment
+should pass `embed_fn_from_agent` (Python) or a real bi-encoder to the shortlist, not this.
+
+The .NET port MUST reproduce this exact algorithm, because
+`hashing_embedder_probe.json` and `case_shortlist_many_options.json` in the golden vectors
+were both generated with it, and the .NET sample uses it too, so its
+stdout can be diffed against the Python-recorded golden line for line.
+
+Algorithm (this docstring is the spec -- read it precisely before porting):
+
+1. Lowercase the input with a full Unicode case mapping (Python's `str.lower()`; C# must use
+ `.ToLowerInvariant()` on the *string*, which uses full Unicode case folding rules too --
+ NOT a per-UTF-16-unit simple mapping. This matters for characters whose lowercase form is
+ more than one code point, e.g. `"İ".lower() == "i̇"` (Turkish capital I with dot
+ above lowercases to 2 code points: 'i' + combining dot above).
+2. Pad the lowercased text with two ASCII space characters (U+0020) on each side:
+ `" " + text + " "`. Padding is added even when `text` is empty, so the padded text is
+ never shorter than 4 code points and there are always at least 2 windows (see step 3):
+ this is what lets short inputs like `""` or `"a"` still produce a nonzero, well-defined
+ vector instead of an empty one.
+3. Slide a window of exactly 3 **Unicode code points** (not UTF-16 code units, not grapheme
+ clusters, not bytes) across the padded text, one code point at a time. For padded text of
+ length `n` code points there are `n - 2` windows, at start offsets `0, 1, ..., n - 3`
+ inclusive. A code point outside the Basic Multilingual Plane (an "astral" character, e.g.
+ an emoji) counts as exactly one code point in this count -- a C# implementation that
+ iterates `char` (UTF-16 code units) instead of Unicode scalar values will split a
+ surrogate pair into two units and get a different window count and different window
+ contents; it must enumerate Unicode scalar values instead (e.g. via `StringInfo` /
+ `System.Globalization.StringInfo.GetTextElementEnumerator` is NOT correct either, since
+ that walks grapheme clusters -- use rune/code-point enumeration, e.g. .NET's
+ `System.Text.Rune` via `str.EnumerateRunes()`).
+4. Encode each 3-code-point window as UTF-8 bytes (the window's own bytes, not the whole
+ padded text's bytes -- re-encode the 3-code-point substring on its own each time).
+5. Hash those UTF-8 bytes with 32-bit FNV-1a:
+
+ hash = 0x811c9dc5 # FNV offset basis
+ for byte in utf8_bytes:
+ hash ^= byte
+ hash = (hash * 0x01000193) & 0xFFFFFFFF # FNV prime, wrapped to 32 bits
+
+ `hash` is an unsigned 32-bit integer throughout; the multiply must wrap (mask to
+ 0xFFFFFFFF / use unchecked arithmetic in C#) exactly like Python's explicit
+ `& 0xFFFFFFFF` above.
+6. `bucket = hash % 64` (0..63, unsigned).
+7. `sign = +1.0 if (hash >> 31) & 1 == 0 else -1.0` (the hash's top bit, as an unsigned
+ 32-bit value -- this is a sign-hashing trick to reduce feature collisions, not a
+ statement about the hash being negative).
+8. Accumulate `sign` into `vector[bucket]` for every window (i.e. `vector[bucket] += sign`;
+ multiple windows can and do land in the same bucket and their signed contributions add).
+ The accumulator is float64 throughout.
+9. No normalisation is applied. The shortlist's cosine similarity handles scale on its own,
+ and leaving the raw counts in the golden vectors makes them easier to hand-check.
+
+`embed(texts)` returns one such vector per input string, as a `(len(texts), 64)` float64
+NumPy array, in input order. It does not batch, cache, or special-case any input; every text
+is processed independently and only via the steps above.
+"""
+from typing import Sequence
+
+import numpy as np
+
+DIM = 64
+_FNV_OFFSET_BASIS = 0x811C9DC5
+_FNV_PRIME = 0x01000193
+_MASK32 = 0xFFFFFFFF
+
+
+def _fnv1a_32(data: bytes) -> int:
+ """32-bit FNV-1a over raw bytes, wrapping like an unsigned 32-bit integer."""
+ h = _FNV_OFFSET_BASIS
+ for b in data:
+ h ^= b
+ h = (h * _FNV_PRIME) & _MASK32
+ return h
+
+
+def _embed_one(text: str) -> np.ndarray:
+ vec = np.zeros(DIM, dtype=np.float64)
+ padded = " " + text.lower() + " "
+ n = len(padded)
+ for i in range(n - 2):
+ window = padded[i : i + 3]
+ h = _fnv1a_32(window.encode("utf-8"))
+ bucket = h % DIM
+ sign = 1.0 if (h >> 31) & 1 == 0 else -1.0
+ vec[bucket] += sign
+ return vec
+
+
+def embed(texts: Sequence[str]) -> np.ndarray:
+ """Embed a list of strings into a `(len(texts), 64)` float64 array. See the module docstring
+ for the exact algorithm; every other-language port must match it bit-for-bit."""
+ rows = [_embed_one("" if t is None else str(t)) for t in texts]
+ if not rows:
+ return np.zeros((0, DIM), dtype=np.float64)
+ return np.stack(rows, axis=0)