Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
feat(dotnet): add embedding shortlist for high-cardinality choice
kevin-gatimu99e0478a138f
diff --git a/laya-dotnet/src/Laya/LayaShortlist.cs b/laya-dotnet/src/Laya/LayaShortlist.csnew file mode 100644index 00000000..19fc6a9d--- /dev/null+++ b/laya-dotnet/src/Laya/LayaShortlist.cs@@ -0,0 +1,304 @@+namespace Laya;++/// <summary>+/// Embeds a batch of texts into vectors. Called once per shortlist, with the query text first,+/// followed by one text per option in criteria order.+/// </summary>+/// <remarks>+/// Mirrors Python's <c>embed_fn: Callable[[Sequence[str]], Any]</c>. Python's return value is+/// duck-typed (a NumPy array, a list of lists, or a torch tensor via <c>.detach()</c>); this port+/// asks for a plain <c>double[][]</c> instead, of shape <c>(texts.Count, dim)</c>, so a caller can+/// implement <see cref="LayaEmbedFunction"/> without taking a dependency on this SDK's own tensor+/// or NumPy-equivalent types — any bi-encoder wrapper that already returns row vectors can be+/// adapted with a one-line lambda. Non-finite components are zeroed by+/// <see cref="LayaShortlist"/> before use, matching Python's <c>np.nan_to_num</c>.+/// </remarks>+/// <param name="texts">The texts to embed, in order.</param>+/// <returns>One row per input text, all rows the same length.</returns>+public delegate double[][] LayaEmbedFunction(IReadOnlyList<string> texts);++/// <summary>+/// Shortlist metadata for one choice question: which labels were kept, their similarity scores+/// (when ranking actually ran), and the <c>k</c>/<c>n</c> it ran under. Mirrors Python's+/// <c>shortlist[qid]</c> entry.+/// </summary>+public sealed class ShortlistInfo+{+ internal ShortlistInfo(IReadOnlyList<string> labels, IReadOnlyList<double>? scores, int k, int n, bool passthrough)+ {+ Labels = labels;+ Scores = scores;+ K = k;+ N = n;+ Passthrough = passthrough;+ }++ /// <summary>The kept labels, in rank order (descending cosine similarity; ties keep the earlier label).</summary>+ public IReadOnlyList<string> Labels { get; }++ /// <summary>+ /// Cosine similarity per kept label, aligned with <see cref="Labels"/>, or+ /// <see langword="null"/> when nothing was dropped (<see cref="Passthrough"/>).+ /// </summary>+ public IReadOnlyList<double>? Scores { get; }++ /// <summary>The shortlist budget this question was ranked under.</summary>+ public int K { get; }++ /// <summary>How many labels the original criteria had.</summary>+ public int N { get; }++ /// <summary>+ /// Whether every label was kept without calling <see cref="LayaEmbedFunction"/> at all+ /// (<c>K >= N</c>).+ /// </summary>+ public bool Passthrough { get; }+}++/// <summary>+/// Opt-in embedding shortlist for high-cardinality choice questions. Ports <c>laya/shortlist.py</c>.+/// </summary>+/// <remarks>+/// <para>+/// Choice options share one token budget, so a large label set leaves only a few tokens per+/// label. <see cref="Predict"/> embeds the state and each option with a caller-supplied+/// <see cref="LayaEmbedFunction"/>, keeps the top <c>k</c>, and runs a single+/// <see cref="ILayaPredictor.Predict"/> on that reduced criteria set.+/// </para>+/// <para>+/// This does not change the decision model's forward pass and does not add a second model call+/// for ranking: ranking is pure cosine similarity on whatever vectors the caller's embedder+/// returns. Porting Python's <c>embed_fn_from_agent</c> (mean-pooling the checkpoint's own+/// encoder) is out of scope for this SDK; pass a real bi-encoder, or the deterministic+/// hashing embedder used in this SDK's tests and samples for a dependency-free demo.+/// </para>+/// </remarks>+public static class LayaShortlist+{+ /// <summary>Default shortlist size, matching Python's <c>DEFAULT_SHORTLIST_K</c>.</summary>+ public const int DefaultShortlistK = 20;++ /// <summary>+ /// Return the top-<paramref name="k"/> choice labels for <paramref name="state"/>, ranked by+ /// cosine similarity between <paramref name="state"/> (plus optional+ /// <paramref name="instructions"/>) and each option's rendered text.+ /// </summary>+ /// <remarks>+ /// <see cref="LayaEmbedFunction"/> is called once, with the query text first and then one+ /// string per option in criteria order. When <paramref name="k"/> is at least the number of+ /// labels, every label is returned in its original order and the embedder is not called.+ /// Ties keep the earlier label; a zero vector scores 0 and does not outrank a label that came+ /// before it.+ /// </remarks>+ public static IReadOnlyList<string> ShortlistChoice(+ object? state,+ IEnumerable<KeyValuePair<string, object?>> criteria,+ LayaEmbedFunction embedFn,+ int k = DefaultShortlistK,+ object? instructions = null)+ {+ ArgumentNullException.ThrowIfNull(criteria);+ ArgumentNullException.ThrowIfNull(embedFn);+ var checkedK = CheckK(k);+ // "" is a valid placeholder: RenderOptions never reads a choice question's Instructions.+ var question = Question.Choice("", criteria);+ var (labels, _, _, _) = Rank(state, question, embedFn, checkedK, instructions);+ return labels;+ }++ /// <summary>+ /// <see cref="ShortlistChoice(object, IEnumerable{KeyValuePair{string, object}}, LayaEmbedFunction, int, object)"/>+ /// over bare labels, with no descriptions.+ /// </summary>+ public static IReadOnlyList<string> ShortlistChoice(+ object? state,+ IEnumerable<string> labels,+ LayaEmbedFunction embedFn,+ int k = DefaultShortlistK,+ object? instructions = null)+ {+ ArgumentNullException.ThrowIfNull(labels);+ return ShortlistChoice(+ state, labels.Select(l => new KeyValuePair<string, object?>(l, null)), embedFn, k, instructions);+ }++ /// <summary>+ /// Shortlist each choice question in <paramref name="questions"/>, then call+ /// <paramref name="predictor"/> once over the reduced set.+ /// </summary>+ /// <remarks>+ /// <para>+ /// Non-choice questions are forwarded unchanged. A choice whose label count is <c><= k</c>+ /// is forwarded unchanged and does not call <paramref name="embedFn"/>. <paramref name="questions"/>+ /// itself is never mutated.+ /// </para>+ /// <para>+ /// The returned <see cref="LayaResult"/> is whatever <paramref name="predictor"/> returned,+ /// plus <see cref="LayaResult.Shortlist"/>. Probabilities on a shortlisted choice are over the+ /// kept labels only. When <paramref name="predictor"/> is a <see cref="LayaRouter"/>, the+ /// result carries both <see cref="LayaResult.Routing"/> and <see cref="LayaResult.Shortlist"/>.+ /// </para>+ /// </remarks>+ /// <exception cref="ArgumentException">+ /// If a choice question in <paramref name="questions"/> is missing <see cref="ChoiceQuestion"/>+ /// options, or <paramref name="embedFn"/> returns a malformed matrix.+ /// </exception>+ public static LayaResult Predict(+ ILayaPredictor predictor,+ object? state,+ QuestionSet questions,+ LayaEmbedFunction embedFn,+ int k = DefaultShortlistK)+ {+ ArgumentNullException.ThrowIfNull(predictor);+ ArgumentNullException.ThrowIfNull(questions);+ ArgumentNullException.ThrowIfNull(embedFn);+ var checkedK = CheckK(k);++ var reduced = new QuestionSet();+ var meta = new Dictionary<string, ShortlistInfo>(StringComparer.Ordinal);++ foreach (var (qid, question) in questions)+ {+ if (question is not ChoiceQuestion choice)+ {+ reduced[qid] = question;+ continue;+ }++ var (labels, scores, passthrough, n) = Rank(state, choice, embedFn, checkedK, choice.Instructions);+ meta[qid] = new ShortlistInfo(labels, scores, checkedK, n, passthrough);+ if (passthrough)+ {+ reduced[qid] = question;+ continue;+ }+ reduced[qid] = Question.Choice(choice.Instructions, SubsetOptions(choice.Options, labels));+ }++ var result = predictor.Predict(state, reduced);+ return result.WithShortlist(meta);+ }++ // ── ranking ───────────────────────────────────────────────────────────────++ private static (List<string> Labels, List<double>? Scores, bool Passthrough, int N) Rank(+ object? state, ChoiceQuestion question, LayaEmbedFunction embedFn, int k, object? instructions)+ {+ var keys = question.Options.Select(o => o.Key).ToList();+ var n = keys.Count;+ if (k >= n) return (keys, null, true, n);++ var query = QueryText(state, instructions);+ var optionTexts = SequenceBuilder.RenderOptions(question);+ var texts = new List<string>(n + 1) { query };+ texts.AddRange(optionTexts);++ var matrix = Embeddings(embedFn, texts);+ var docs = new double[n][];+ Array.Copy(matrix, 1, docs, 0, n);+ var sims = Cosine(matrix[0], docs);++ // Stable descending sort: ties keep the earlier label, matching NumPy's+ // np.argsort(-sims, kind="mergesort") — LINQ's OrderBy family is documented stable.+ var order = Enumerable.Range(0, n).OrderByDescending(i => sims[i]).Take(k).ToList();+ var labels = order.Select(i => keys[i]).ToList();+ var scores = order.Select(i => sims[i]).ToList();+ return (labels, scores, false, n);+ }++ private static IEnumerable<KeyValuePair<string, object?>> SubsetOptions(+ IReadOnlyList<KeyValuePair<string, object?>> options, IReadOnlyList<string> labels)+ {+ var byLabel = options.ToDictionary(o => o.Key, o => o.Value, StringComparer.Ordinal);+ foreach (var label in labels) yield return new(label, byLabel[label]);+ }++ /// <summary>+ /// Query text: <c>instructions + "\n" + state</c> when instructions is present and non-empty,+ /// else just the serialized state. A non-string instructions value is dumped via+ /// <see cref="PythonJson.State"/> (Python's <c>json.dumps(..., ensure_ascii=False)</c>, no+ /// <c>default=str</c> fallback) — the same dialect <c>PythonJson.State</c> already uses for a+ /// non-string state, so no separate dialect was needed for this.+ /// </summary>+ private static string QueryText(object? state, object? instructions)+ {+ var body = PythonJson.State(state);+ if (instructions is null) return body;+ if (instructions is string { Length: 0 }) return body;+ return PythonJson.State(instructions) + "\n" + body;+ }++ private static int CheckK(int k)+ {+ if (k < 1) throw new ArgumentOutOfRangeException(nameof(k), k, "k must be a positive integer");+ return k;+ }++ // ── embedding + cosine ───────────────────────────────────────────────────++ private static double[][] Embeddings(LayaEmbedFunction embedFn, IReadOnlyList<string> texts)+ {+ var raw = embedFn(texts)+ ?? throw new ArgumentException(+ $"embed_fn must return an array of shape ({texts.Count}, dim), got null", nameof(embedFn));+ if (raw.Length != texts.Count)+ throw new ArgumentException(+ $"embed_fn must return an array of shape ({texts.Count}, dim), got ({raw.Length}, ...)",+ nameof(embedFn));++ var dim = raw.Length > 0 ? raw[0]?.Length ?? -1 : -1;+ if (dim < 1)+ throw new ArgumentException(+ $"embed_fn must return an array of shape ({texts.Count}, dim) with dim >= 1", nameof(embedFn));++ var cleaned = new double[raw.Length][];+ for (var i = 0; i < raw.Length; i++)+ {+ var row = raw[i]+ ?? throw new ArgumentException($"embed_fn returned a null row at index {i}", nameof(embedFn));+ if (row.Length != dim)+ throw new ArgumentException(+ $"embed_fn must return an array of shape ({texts.Count}, dim); row {i} has length "+ + $"{row.Length}, expected {dim}", nameof(embedFn));++ var outRow = new double[dim];+ for (var j = 0; j < dim; j++)+ {+ var v = row[j];+ // np.nan_to_num: only non-finite COMPONENTS are zeroed, not the whole vector.+ outRow[j] = double.IsNaN(v) || double.IsInfinity(v) ? 0.0 : v;+ }+ cleaned[i] = outRow;+ }+ return cleaned;+ }++ private static double[] Cosine(double[] query, double[][] docs)+ {+ var sims = new double[docs.Length];+ var qn = Norm(query);+ if (qn == 0.0) return sims;++ for (var i = 0; i < docs.Length; i++)+ {+ var denom = Norm(docs[i]) * qn;+ if (denom > 0.0) sims[i] = Dot(docs[i], query) / denom;+ }+ return sims;+ }++ private static double Norm(double[] v)+ {+ var sum = 0.0;+ foreach (var x in v) sum += x * x;+ return Math.Sqrt(sum);+ }++ private static double Dot(double[] a, double[] b)+ {+ var sum = 0.0;+ for (var i = 0; i < a.Length; i++) sum += a[i] * b[i];+ return sum;+ }+}diff --git a/laya-dotnet/tests/Laya.Tests/HashingEmbedder.cs b/laya-dotnet/tests/Laya.Tests/HashingEmbedder.csnew file mode 100644index 00000000..bf9a87ed--- /dev/null+++ b/laya-dotnet/tests/Laya.Tests/HashingEmbedder.cs@@ -0,0 +1,82 @@+using System.Text;++namespace Laya.Tests;++/// <summary>+/// Deterministic, dependency-free demo embedder for shortlist tests. Ports+/// <c>laya-dotnet/tools/hashing_embedder.py</c> bit-for-bit: a hashed character-trigram bag-of-features+/// vector, chosen only because it is a pure function of the input text and is trivial to+/// reimplement exactly with nothing but a string, UTF-8 encoding and 32-bit integer arithmetic.+/// <b>This is not a real embedding model</b> — it exists solely so the golden vectors and this+/// SDK's tests can pin an exact, reproducible <see cref="LayaEmbedFunction"/> without a real+/// bi-encoder or a Hub download. See <c>hashing_embedder_probe.json</c> and+/// <c>case_shortlist_many_options.json</c>, which were both recorded with this exact algorithm.+/// </summary>+/// <remarks>+/// Algorithm (mirrors the Python docstring precisely):+/// <list type="number">+/// <item>Lowercase with full Unicode case mapping — <see cref="LanguageDetection.PythonLower"/>,+/// not <see cref="string.ToLowerInvariant()"/> directly, because this project builds with+/// <c>InvariantGlobalization</c> enabled, under which plain <c>ToLowerInvariant</c> leaves U+0130+/// (Turkish capital I with dot above) untouched instead of expanding it to "i" + combining dot,+/// as Python's <c>str.lower()</c> does.</item>+/// <item>Pad with two ASCII spaces on each side.</item>+/// <item>Slide a window of exactly 3 Unicode code points (via <see cref="System.Text.Rune"/>,+/// never UTF-16 code units) across the padded text, one code point at a time.</item>+/// <item>UTF-8-encode each window's own 3 code points (re-encoded fresh each time, not sliced+/// from the whole padded text's byte buffer).</item>+/// <item>Hash those bytes with 32-bit FNV-1a (unsigned, wrapping multiply).</item>+/// <item><c>bucket = hash % 64</c>.</item>+/// <item><c>sign = +1</c> when the hash's top bit is 0, else <c>-1</c>.</item>+/// <item>Accumulate <c>sign</c> into <c>vector[bucket]</c> for every window (float64 throughout).</item>+/// <item>No normalisation.</item>+/// </list>+/// </remarks>+public static class HashingEmbedder+{+ /// <summary>Vector width, matching Python's <c>DIM</c>.</summary>+ public const int Dimension = 64;++ private const uint FnvOffsetBasis = 0x811c9dc5;+ private const uint FnvPrime = 0x01000193;++ /// <summary>Embed a batch of texts, one <see cref="Dimension"/>-length vector per input, in order.</summary>+ public static double[][] Embed(IReadOnlyList<string> texts)+ {+ ArgumentNullException.ThrowIfNull(texts);+ var result = new double[texts.Count][];+ for (var i = 0; i < texts.Count; i++) result[i] = EmbedOne(texts[i]);+ return result;+ }++ /// <summary>Embed a single text into a <see cref="Dimension"/>-length vector.</summary>+ public static double[] EmbedOne(string? text)+ {+ var vector = new double[Dimension];+ var padded = " " + LanguageDetection.PythonLower(text ?? string.Empty) + " ";+ var runes = padded.EnumerateRunes().ToArray();+ var windowCount = runes.Length - 2; // always >= 2: padding guarantees at least 4 code points++ for (var i = 0; i < windowCount; i++)+ {+ var window = string.Concat(runes[i].ToString(), runes[i + 1].ToString(), runes[i + 2].ToString());+ var hash = Fnv1a32(Encoding.UTF8.GetBytes(window));+ var bucket = (int)(hash % Dimension);+ var sign = (hash >> 31 & 1) == 0 ? 1.0 : -1.0;+ vector[bucket] += sign;+ }+ return vector;+ }++ /// <summary>32-bit FNV-1a over raw bytes, wrapping like an unsigned 32-bit integer.</summary>+ private static uint Fnv1a32(byte[] data)+ {+ var hash = FnvOffsetBasis;+ foreach (var b in data)+ {+ hash ^= b;+ hash *= FnvPrime; // uint * uint wraps silently (unchecked) by default in C#+ }+ return hash;+ }+}diff --git a/laya-dotnet/tests/Laya.Tests/HashingEmbedderTests.cs b/laya-dotnet/tests/Laya.Tests/HashingEmbedderTests.csnew file mode 100644index 00000000..7784e6dd--- /dev/null+++ b/laya-dotnet/tests/Laya.Tests/HashingEmbedderTests.cs@@ -0,0 +1,64 @@+using System.Text.Json;++namespace Laya.Tests;++/// <summary>+/// Tier 1: the hashing embedder against every entry of+/// <c>golden/routing/hashing_embedder_probe.json</c>, which was generated by+/// <c>laya-dotnet/tools/hashing_embedder.py</c>. Every language's port must agree bit-for-bit, because the+/// shortlist goldens and every SDK's sample were recorded with it.+/// </summary>+public sealed class HashingEmbedderTests+{+ public static IEnumerable<object[]> ProbeCases()+ {+ using var doc = RoutingGoldenData.Load("hashing_embedder_probe.json");+ // Index by position, not by text: several probe entries share the same text after+ // trimming display, and whitespace-only text has no useful theory-case label anyway.+ return+ [+ .. Enumerable.Range(0, doc.RootElement.GetArrayLength()).Select(i => new object[] { i }),+ ];+ }++ [Theory]+ [MemberData(nameof(ProbeCases))]+ public void EveryProbeEntry(int index)+ {+ using var doc = RoutingGoldenData.Load("hashing_embedder_probe.json");+ var entry = doc.RootElement[index];+ var text = entry.GetProperty("text").GetString()!;+ var expected = entry.GetProperty("vector").EnumerateArray().Select(e => e.GetDouble()).ToArray();++ var actual = HashingEmbedder.EmbedOne(text);++ Assert.Equal(HashingEmbedder.Dimension, actual.Length);+ Assert.Equal(expected.Length, actual.Length);+ for (var i = 0; i < expected.Length; i++)+ Assert.True(expected[i] == actual[i],+ $"text {JsonSerializer.Serialize(text)} bucket {i}: python {expected[i]}, dotnet {actual[i]}");+ }++ [Fact]+ public void EmbedBatchesInOrderAndMatchesEmbedOne()+ {+ var texts = new[] { "alpha", "beta", "gamma" };+ var batch = HashingEmbedder.Embed(texts);+ Assert.Equal(texts.Length, batch.Length);+ for (var i = 0; i < texts.Length; i++)+ Assert.Equal(HashingEmbedder.EmbedOne(texts[i]), batch[i]);+ }++ [Fact]+ public void EmptyBatchReturnsEmptyResult() => Assert.Empty(HashingEmbedder.Embed([]));++ [Fact]+ public void TurkishCapitalIWithDotUsesTheTwoCodePointLowercasing()+ {+ // Regression guard for the InvariantGlobalization pitfall documented on HashingEmbedder:+ // plain ToLowerInvariant() would leave U+0130 untouched instead of expanding it.+ var istanbul = HashingEmbedder.EmbedOne("İstanbul");+ var alreadyLower = HashingEmbedder.EmbedOne("i̇stanbul"); // "i" + combining dot above+ Assert.Equal(alreadyLower, istanbul);+ }+}diff --git a/laya-dotnet/tests/Laya.Tests/LayaShortlistTests.cs b/laya-dotnet/tests/Laya.Tests/LayaShortlistTests.csnew file mode 100644index 00000000..f4b60078--- /dev/null+++ b/laya-dotnet/tests/Laya.Tests/LayaShortlistTests.cs@@ -0,0 +1,439 @@+using System.Text.Json;++namespace Laya.Tests;++/// <summary>+/// Tier 1: the embedding shortlist, which needs no artifacts. A mock <see cref="ILayaPredictor"/>+/// stands in for the decision model, so these tests never construct one. Named cases mirror+/// <c>tests/test_shortlist.py</c> (excluding the torch mean-pool block, which ports Python's+/// <c>embed_fn_from_agent</c> — out of scope per the porting plan); <see cref="EveryShortlistProbeEntry"/>+/// consumes every entry of <c>golden/routing/shortlist_probe.json</c>.+/// </summary>+public sealed class LayaShortlistTests+{+ // Vectors: query [1, 0]. alpha and delta tie at cosine 1; gamma is 0.6; beta is 0.+ // Stable order must keep alpha ahead of delta.+ private static readonly Dictionary<string, object?> Criteria = new(StringComparer.Ordinal)+ {+ ["alpha"] = null, ["beta"] = "", ["gamma"] = "mid", ["delta"] = "same",+ };++ private static readonly Dictionary<string, double[]> OptionVectors = new(StringComparer.Ordinal)+ {+ ["alpha"] = [1.0, 0.0],+ ["beta"] = [0.0, 1.0],+ ["gamma: mid"] = [0.6, 0.8],+ ["delta: same"] = [1.0, 0.0],+ };++ private sealed class TableEmbed(IReadOnlyDictionary<string, double[]> vectors)+ {+ public List<List<string>> Calls { get; } = [];++ public double[][] Embed(IReadOnlyList<string> texts)+ {+ Calls.Add([.. texts]);+ var missing = texts.Where(t => !vectors.ContainsKey(t)).ToList();+ if (missing.Count > 0) throw new InvalidOperationException($"unexpected texts: {string.Join(", ", missing)}");+ return [.. texts.Select(t => vectors[t])];+ }+ }++ private static LayaEmbedFunction ForQuery(string queryText, IReadOnlyDictionary<string, double[]> optionVectors)+ {+ var full = new Dictionary<string, double[]>(optionVectors, StringComparer.Ordinal) { [queryText] = [1.0, 0.0] };+ return new TableEmbed(full).Embed;+ }++ // ── deterministic top-k ──────────────────────────────────────────────────++ [Fact]+ public void TopKKeepsTheCosineTieInInputOrder()+ {+ var embed = new TableEmbed(new Dictionary<string, double[]>(OptionVectors, StringComparer.Ordinal) { ["pay me"] = [1.0, 0.0] });+ var labels = LayaShortlist.ShortlistChoice("pay me", Criteria, embed.Embed, k: 2);+ Assert.Equal(["alpha", "delta"], labels);+ Assert.Single(embed.Calls);+ Assert.Equal("pay me", embed.Calls[0][0]);+ Assert.Equal(["alpha", "beta", "gamma: mid", "delta: same"], embed.Calls[0].Skip(1));+ }++ [Fact]+ public void K1IsTheEarliestMax()+ {+ var embed = ForQuery("pay me", OptionVectors);+ Assert.Equal(["alpha"], LayaShortlist.ShortlistChoice("pay me", Criteria, embed, k: 1));+ }++ [Fact]+ public void K3AppendsTheNextCosine()+ {+ var embed = ForQuery("pay me", OptionVectors);+ Assert.Equal(["alpha", "delta", "gamma"], LayaShortlist.ShortlistChoice("pay me", Criteria, embed, k: 3));+ }++ [Fact]+ public void ZeroQueryKeepsOriginalOrder()+ {+ // Every cosine is 0 when the query vector is 0, so the earliest labels win.+ var vectors = new Dictionary<string, double[]>(StringComparer.Ordinal)+ {+ ["pay me"] = [0.0, 0.0],+ ["alpha"] = [1.0, 0.0],+ ["beta"] = [0.0, 1.0],+ ["gamma: mid"] = [0.6, 0.8],+ ["delta: same"] = [3.0, 4.0],+ };+ var embed = new TableEmbed(vectors);+ Assert.Equal(["alpha", "beta"], LayaShortlist.ShortlistChoice("pay me", Criteria, embed.Embed, k: 2));+ }++ [Fact]+ public void NanVectorSortsBehindAFiniteMatch()+ {+ // A non-finite option vector is treated as 0 and loses to a real match.+ var vectors = new Dictionary<string, double[]>(StringComparer.Ordinal)+ {+ ["pay me"] = [1.0, 0.0],+ ["alpha"] = [double.NaN, double.NaN],+ ["beta"] = [1.0, 0.0],+ };+ var embed = new TableEmbed(vectors);+ var criteria = new Dictionary<string, object?> { ["alpha"] = null, ["beta"] = null };+ Assert.Equal(["beta"], LayaShortlist.ShortlistChoice("pay me", criteria, embed.Embed, k: 1));+ }++ // ── list criteria and instructions ───────────────────────────────────────++ [Fact]+ public void InstructionsChangeTheQueryAndTheWinner()+ {+ var vectors = new Dictionary<string, double[]>(StringComparer.Ordinal)+ {+ ["Classify\npay me"] = [0.0, 1.0],+ ["alpha"] = [1.0, 0.0],+ ["beta"] = [0.0, 1.0],+ ["gamma"] = [0.0, 0.2],+ };+ var embed = new TableEmbed(vectors);+ var labels = LayaShortlist.ShortlistChoice("pay me", ["alpha", "beta", "gamma"], embed.Embed, k: 2, instructions: "Classify");+ Assert.Equal(["beta", "gamma"], labels);+ Assert.Equal("Classify\npay me", embed.Calls[0][0]);+ Assert.Equal(["alpha", "beta", "gamma"], embed.Calls[0].Skip(1));+ }++ [Fact]+ public void DictStateIsSerialized()+ {+ var state = new Dictionary<string, object?> { ["text"] = "hi" };+ var vectors = new Dictionary<string, double[]>(StringComparer.Ordinal)+ {+ ["Classify\n{\"text\": \"hi\"}"] = [1.0, 0.0],+ ["alpha"] = [1.0, 0.0],+ ["beta"] = [0.0, 1.0],+ };+ var embed = new TableEmbed(vectors);+ var labels = LayaShortlist.ShortlistChoice(state, ["alpha", "beta"], embed.Embed, k: 1, instructions: "Classify");+ Assert.Equal(["alpha"], labels);+ Assert.Equal("Classify\n{\"text\": \"hi\"}", embed.Calls[0][0]);+ }++ [Fact]+ public void ZeroAndFalseCriterionValuesStayInTheOptionText()+ {+ // 0 and False are real criterion values, so they are part of the embedded text.+ var rich = new Dictionary<string, object?>+ {+ ["zero"] = 0L, ["no"] = false, ["bare"] = null,+ ["named"] = new Dictionary<string, object?> { ["desc"] = "payments" },+ };+ var richRendered = SequenceBuilder.RenderOptions(Question.Choice("", rich));+ var vectors = new Dictionary<string, double[]>(StringComparer.Ordinal) { ["pay me"] = [1.0, 0.0] };+ foreach (var t in richRendered) vectors[t] = [1.0, 0.0];+ var embed = new TableEmbed(vectors);+ LayaShortlist.ShortlistChoice("pay me", rich, embed.Embed, k: 1);+ Assert.Equal(richRendered, embed.Calls[0].Skip(1));+ }++ // ── k >= n pass-through ───────────────────────────────────────────────────++ [Fact]+ public void KEqualsNReturnsEveryLabelInOrderWithoutCallingEmbedFn()+ {+ LayaEmbedFunction boom = _ => throw new InvalidOperationException("embed_fn should not run when k >= n");+ Assert.Equal([.. Criteria.Keys], LayaShortlist.ShortlistChoice("pay me", Criteria, boom, k: 4));+ }++ [Fact]+ public void KGreaterThanNReturnsEveryLabelInOrderWithoutCallingEmbedFn()+ {+ LayaEmbedFunction boom = _ => throw new InvalidOperationException("embed_fn should not run when k >= n");+ Assert.Equal([.. Criteria.Keys], LayaShortlist.ShortlistChoice("pay me", Criteria, boom, k: 20));+ }++ // ── Predict: mock predictor sees only k criteria ─────────────────────────++ private sealed class Recorder : ILayaPredictor+ {+ public List<(object? State, QuestionSet Questions)> Calls { get; } = [];++ public LayaResult Predict(object? state, QuestionSet questions)+ {+ Calls.Add((state, questions));+ var order = new List<string>();+ var byId = new Dictionary<string, Answer>(StringComparer.Ordinal);+ foreach (var (qid, q) in questions)+ {+ order.Add(qid);+ if (q is ChoiceQuestion choice)+ {+ var labels = choice.Labels;+ var probs = labels.Select((l, i) => new KeyValuePair<string, double>(l, i == 0 ? 1.0 : 0.0)).ToList();+ byId[qid] = new ChoiceAnswer(labels[0], probs, 1.0, 1.0, new ActionInfo(1.0));+ }+ }+ return new LayaResult("fake", order, byId, new Usage(0, 0));+ }+ }++ [Fact]+ public void PredictShortlistsChoiceCriteriaToTheTopK()+ {+ var sentinel = new Dictionary<string, object?> { ["desc"] = "payments" };+ var full = new Dictionary<string, object?> { ["billing"] = sentinel, ["tech"] = "bugs", ["sales"] = null, ["other"] = "misc" };+ var vectors = new Dictionary<string, double[]>(StringComparer.Ordinal)+ {+ ["Which desk?\nI was charged twice"] = [1.0, 0.0],+ ["billing: {\"desc\": \"payments\"}"] = [0.0, 1.0],+ ["tech: bugs"] = [1.0, 0.0],+ ["sales"] = [0.2, 0.2],+ ["other: misc"] = [0.0, 1.0],+ };+ var embed = new TableEmbed(vectors);+ var scoreQ = Question.Score("How urgent?", "low", "mid", "high", "now");+ var noulQ = Question.Noul("Is a refund requested?");+ var questions = new QuestionSet+ {+ ["intent"] = Question.Choice("Which desk?", full),+ ["urgency"] = scoreQ,+ ["refund"] = noulQ,+ };+ var recorder = new Recorder();++ var result = LayaShortlist.Predict(recorder, "I was charged twice", questions, embed.Embed, k: 2);++ Assert.Single(recorder.Calls);+ var gotQuestions = recorder.Calls[0].Questions;+ Assert.Same("I was charged twice", recorder.Calls[0].State);+ var reducedIntent = (ChoiceQuestion)gotQuestions["intent"];+ Assert.Equal(["tech", "sales"], reducedIntent.Labels);+ Assert.Equal("bugs", reducedIntent.Options.First(o => o.Key == "tech").Value);+ Assert.Same(scoreQ, gotQuestions["urgency"]);+ Assert.Same(noulQ, gotQuestions["refund"]);++ // The caller's original question and criteria are untouched.+ Assert.Equal(4, ((ChoiceQuestion)questions["intent"]).Options.Count);+ Assert.Equal(sentinel, full["billing"]);++ Assert.Equal("tech", result["intent"].AsChoice().Choice);+ Assert.Equal(["tech", "sales"], result.Shortlist!["intent"].Labels);+ Assert.True(result.Shortlist["intent"].Scores![0] > result.Shortlist["intent"].Scores![1]);+ Assert.True(result.Shortlist["intent"].Scores![1] > 0);+ Assert.Equal(2, result.Shortlist["intent"].K);+ Assert.Equal(4, result.Shortlist["intent"].N);+ Assert.False(result.Shortlist["intent"].Passthrough);+ Assert.False(result.Shortlist.ContainsKey("urgency"));+ }++ [Fact]+ public void PredictReturnsACopyRatherThanMutatingThePredictorsResult()+ {+ var holding = new HoldingPredictor();+ var criteria = new Dictionary<string, object?>(Criteria);+ var out1 = LayaShortlist.Predict(holding, "pay me", new QuestionSet { ["intent"] = Question.Choice("", criteria) },+ _ => throw new InvalidOperationException(), k: 4);+ Assert.NotNull(out1.Shortlist);+ Assert.Null(holding.Held?.Shortlist);+ }++ private sealed class HoldingPredictor : ILayaPredictor+ {+ public LayaResult? Held;+ public LayaResult Predict(object? state, QuestionSet questions) =>+ Held = new LayaResult("fake", [], new Dictionary<string, Answer>(StringComparer.Ordinal), new Usage(0, 0));+ }++ [Fact]+ public void KGreaterThanOrEqualNReachesPredictUnchanged()+ {+ var full = new Dictionary<string, object?> { ["billing"] = null, ["tech"] = "bugs", ["sales"] = null, ["other"] = "misc" };+ var originalQuestion = Question.Choice("Which desk?", full);+ var recorder = new Recorder();+ var out1 = LayaShortlist.Predict(recorder, "I was charged twice",+ new QuestionSet { ["intent"] = originalQuestion },+ _ => throw new InvalidOperationException("embed_fn should not run"), k: 4);++ Assert.Same(originalQuestion, recorder.Calls[0].Questions["intent"]);+ Assert.Equal([.. full.Keys], out1.Shortlist!["intent"].Labels);+ Assert.Null(out1.Shortlist["intent"].Scores);+ Assert.True(out1.Shortlist["intent"].Passthrough);+ }++ [Fact]+ public void ListCriteriaStayInRankOrderAsAList()+ {+ var vectors = new Dictionary<string, double[]>(StringComparer.Ordinal)+ {+ ["Which?\nhello"] = [1.0, 0.0],+ ["alpha"] = [0.0, 1.0],+ ["beta"] = [1.0, 0.0],+ ["gamma"] = [0.0, 0.0],+ };+ var embed = new TableEmbed(vectors);+ var recorder = new Recorder();+ var questions = new QuestionSet { ["intent"] = Question.Choice("Which?", "alpha", "beta", "gamma") };+ LayaShortlist.Predict(recorder, "hello", questions, embed.Embed, k: 2);++ var received = (ChoiceQuestion)recorder.Calls[0].Questions["intent"];+ Assert.Equal(["beta", "alpha"], received.Labels);+ // The caller's own question is untouched.+ Assert.Equal(["alpha", "beta", "gamma"], ((ChoiceQuestion)questions["intent"]).Labels);+ }++ // ── errors ────────────────────────────────────────────────────────────────++ [Fact]+ public void KZeroThrows() => Assert.Throws<ArgumentOutOfRangeException>(+ () => LayaShortlist.ShortlistChoice("pay me", Criteria, ForQuery("pay me", OptionVectors), k: 0));++ [Fact]+ public void KNegativeThrows() => Assert.Throws<ArgumentOutOfRangeException>(+ () => LayaShortlist.ShortlistChoice("pay me", Criteria, ForQuery("pay me", OptionVectors), k: -3));++ [Fact]+ public void EmptyDictCriteriaThrows() => Assert.Throws<ArgumentException>(+ () => LayaShortlist.ShortlistChoice("pay me", new Dictionary<string, object?>(), ForQuery("pay me", OptionVectors), k: 1));++ [Fact]+ public void EmptyListCriteriaThrows() => Assert.Throws<ArgumentException>(+ () => LayaShortlist.ShortlistChoice("pay me", Array.Empty<string>(), ForQuery("pay me", OptionVectors), k: 1));++ [Fact]+ public void DuplicateLabelThrows() => Assert.Throws<ArgumentException>(+ () => LayaShortlist.ShortlistChoice("pay me", ["alpha", "alpha"], ForQuery("pay me", OptionVectors), k: 1));++ [Fact]+ public void MissingCriteriaOnAChoiceQuestionThrows()+ {+ // A choice question with no ChoiceQuestion.Options cannot exist in the .NET model+ // (construction itself validates), so the analogous failure here is empty criteria.+ Assert.Throws<ArgumentException>(() => Question.Choice("x", new Dictionary<string, object?>()));+ }++ private static readonly double[][] BadShapeVectors = [[0.0, 0.0, 0.0, 0.0]];++ [Fact]+ public void BadEmbedShapeThrowsAndNeverCallsPredict()+ {+ var recorder = new Recorder();+ var full = new Dictionary<string, object?> { ["billing"] = null, ["tech"] = "bugs", ["sales"] = null, ["other"] = "misc" };+ var questions = new QuestionSet { ["intent"] = Question.Choice("Which desk?", full) };+ Assert.Throws<ArgumentException>(() =>+ LayaShortlist.Predict(recorder, "pay me", questions, _ => BadShapeVectors, k: 2));+ Assert.Empty(recorder.Calls);+ }++ // ── every recorded probe entry ────────────────────────────────────────────++ public static IEnumerable<object[]> ShortlistProbeCases()+ {+ using var doc = RoutingGoldenData.Load("shortlist_probe.json");+ return [.. doc.RootElement.EnumerateArray().Select(e => new object[] { e.GetProperty("label").GetString()! })];+ }++ private static JsonElement FindEntry(string label)+ {+ using var doc = RoutingGoldenData.Load("shortlist_probe.json");+ foreach (var e in doc.RootElement.EnumerateArray())+ if (e.GetProperty("label").GetString() == label)+ return JsonDocument.Parse(e.GetRawText()).RootElement;+ throw new InvalidOperationException($"no shortlist_probe.json entry labelled '{label}'");+ }++ [Theory]+ [MemberData(nameof(ShortlistProbeCases))]+ public void EveryShortlistProbeEntry(string label)+ {+ var e = FindEntry(label);+ var state = RoutingGoldenData.ToClr(e.GetProperty("state"));+ var k = e.GetProperty("k").GetInt32();+ var instructionsProp = e.GetProperty("instructions");+ var instructions = instructionsProp.ValueKind == JsonValueKind.Null ? null : RoutingGoldenData.ToClr(instructionsProp);++ var criteriaElement = e.GetProperty("criteria");+ var criteria = criteriaElement.ValueKind == JsonValueKind.Array+ ? criteriaElement.EnumerateArray().Select(x => new KeyValuePair<string, object?>(x.GetString()!, null)).ToList()+ : criteriaElement.EnumerateObject()+ .Select(p => new KeyValuePair<string, object?>(p.Name, RoutingGoldenData.ToClr(p.Value))).ToList();++ var passthrough = e.GetProperty("passthrough").GetBoolean();+ var wantLabels = e.GetProperty("labels").EnumerateArray().Select(x => x.GetString()!).ToArray();+ var n = e.GetProperty("n").GetInt32();++ var recorder = new Recorder();+ var question = Question.Choice("", criteria);+ var questions = new QuestionSet { ["q"] = question };++ LayaEmbedFunction embed = passthrough+ ? (_ => throw new InvalidOperationException($"{label}: embed_fn should not run when k >= n"))+ : texts => ReadVectorsMatrix(e.GetProperty("vectors"), texts.Count);++ var result = LayaShortlist.Predict(recorder, state, questions, embed, k);+ var info = result.Shortlist!["q"];++ Assert.Equal(wantLabels, info.Labels);+ Assert.Equal(k, info.K);+ Assert.Equal(n, info.N);+ Assert.Equal(passthrough, info.Passthrough);++ if (passthrough)+ {+ Assert.Null(info.Scores);+ }+ else+ {+ var wantScores = e.GetProperty("scores").EnumerateArray().Select(x => x.GetDouble()).ToArray();+ Assert.NotNull(info.Scores);+ Assert.Equal(wantScores.Length, info.Scores!.Count);+ for (var i = 0; i < wantScores.Length; i++)+ Assert.Equal(wantScores[i], info.Scores[i], 1e-9);+ }+ }++ /// <summary>+ /// Parse a probe "vectors" matrix (first row is the query, remaining rows the options, in+ /// criteria order). Non-finite components are encoded as the JSON strings "nan"/"inf"/"-inf"+ /// per <c>meta.json</c>'s notes; <paramref name="expectedRows"/> is asserted against the+ /// embed_fn call's own text count, mirroring how <c>_rank</c> always calls embed_fn with+ /// exactly <c>1 + n</c> texts.+ /// </summary>+ private static double[][] ReadVectorsMatrix(JsonElement vectors, int expectedRows)+ {+ var rows = vectors.EnumerateArray().Select(row =>+ row.EnumerateArray().Select(ParseComponent).ToArray()).ToArray();+ Assert.Equal(expectedRows, rows.Length);+ return rows;+ }++ private static double ParseComponent(JsonElement e) => e.ValueKind switch+ {+ JsonValueKind.String => e.GetString() switch+ {+ "nan" => double.NaN,+ "inf" => double.PositiveInfinity,+ "-inf" => double.NegativeInfinity,+ var s => double.Parse(s!, System.Globalization.CultureInfo.InvariantCulture),+ },+ _ => e.GetDouble(),+ };+}diff --git a/laya-dotnet/tests/Laya.Tests/ShortlistEndToEndTests.cs b/laya-dotnet/tests/Laya.Tests/ShortlistEndToEndTests.csnew file mode 100644index 00000000..34fb694b--- /dev/null+++ b/laya-dotnet/tests/Laya.Tests/ShortlistEndToEndTests.cs@@ -0,0 +1,72 @@+namespace Laya.Tests;++/// <summary>+/// Tier 2: the embedding shortlist over a real checkpoint end to end, for all three checkpoints.+/// Compares <see cref="LayaShortlist.Predict"/> (using <see cref="HashingEmbedder"/>, the same+/// deterministic demo embedder the golden vector was recorded with) against+/// <c>case_shortlist_many_options.json</c>'s recorded <c>predict_shortlist</c> result.+/// </summary>+/// <remarks>+/// Each test loads and disposes its own <see cref="LayaEngine"/> rather than sharing+/// <see cref="AllEnginesFixture"/> (which keeps every checkpoint it has touched resident for the+/// whole run): the ONNX artifacts are 1.3-1.7 GB apiece, and these tests must not hold more than+/// one at a time. The class still joins <see cref="AllEnginesCollection"/>, purely so it never+/// runs concurrently with another engine-loading test class — the native tokenizer binding is not+/// safe to construct from two threads at once for the same checkpoint.+/// </remarks>+[Collection(AllEnginesCollection.Name)]+public sealed class ShortlistEndToEndTests+{+ /// <summary>Tolerance for the shortlist's raw (unrounded) cosine scores.</summary>+ private const double ScoreTolerance = 1e-9;++ /// <summary>Tolerance for the model's own 4-dp probabilities/confidence, as in <see cref="PredictParityTests"/>.</summary>+ private const double ProbTolerance = 2e-4;++ public static TheoryData<LayaCheckpoint> Checkpoints => new(Enum.GetValues<LayaCheckpoint>());++ [Theory]+ [MemberData(nameof(Checkpoints))]+ public void ShortlistManyOptionsMatchesTheRecordedResult(LayaCheckpoint checkpoint)+ {+ var golden = GoldenData.For(checkpoint);+ if (!golden.Available)+ Assert.Skip($"no golden data for '{checkpoint}'; run laya-dotnet/tools/dump_golden.py");+ if (!TestArtifacts.For(checkpoint).HasModel || !TestArtifacts.For(checkpoint).HasTokenizer)+ Assert.Skip($"no ONNX artifacts for '{checkpoint}'; set LAYA_ONNX_ROOT to the parent of all checkpoints");++ using var doc = golden.Load("case_shortlist_many_options.json");+ var root = doc.RootElement;+ var state = GoldenData.ToClr(root.GetProperty("state"));+ var questions = GoldenData.BuildQuestions(root.GetProperty("questions"));+ var k = root.GetProperty("shortlist_k").GetInt32();++ using var engine = LayaEngine.FromDirectory(TestArtifacts.For(checkpoint).Directory);+ var result = LayaShortlist.Predict(engine, state, questions, HashingEmbedder.Embed, k);++ var expected = root.GetProperty("result");+ Assert.Equal(expected.GetProperty("model").GetString(), result.Model);++ var wantShortlist = expected.GetProperty("shortlist").GetProperty("intent");+ var gotShortlist = result.Shortlist!["intent"];+ Assert.Equal(+ [.. wantShortlist.GetProperty("labels").EnumerateArray().Select(e => e.GetString()!)],+ gotShortlist.Labels);+ Assert.Equal(wantShortlist.GetProperty("k").GetInt32(), gotShortlist.K);+ Assert.Equal(wantShortlist.GetProperty("n").GetInt32(), gotShortlist.N);+ Assert.Equal(wantShortlist.GetProperty("passthrough").GetBoolean(), gotShortlist.Passthrough);++ var wantScores = wantShortlist.GetProperty("scores").EnumerateArray().Select(e => e.GetDouble()).ToArray();+ Assert.NotNull(gotShortlist.Scores);+ Assert.Equal(wantScores.Length, gotShortlist.Scores!.Count);+ for (var i = 0; i < wantScores.Length; i++)+ Assert.Equal(wantScores[i], gotShortlist.Scores[i], ScoreTolerance);++ var wantAnswer = expected.GetProperty("answers").GetProperty("intent");+ var gotAnswer = result["intent"].AsChoice();+ Assert.Equal(wantAnswer.GetProperty("choice").GetString(), gotAnswer.Choice);+ foreach (var p in wantAnswer.GetProperty("probabilities").EnumerateObject())+ Assert.Equal(p.Value.GetDouble(), gotAnswer[p.Name], ProbTolerance);+ Assert.Equal(wantAnswer.GetProperty("confidence").GetDouble(), result["intent"].Confidence, ProbTolerance);+ }+}diff --git a/laya-dotnet/tools/hashing_embedder.py b/laya-dotnet/tools/hashing_embedder.pynew file mode 100644index 00000000..2311435e--- /dev/null+++ b/laya-dotnet/tools/hashing_embedder.py@@ -0,0 +1,101 @@+"""Deterministic, dependency-free demo embedder for the shortlist golden vectors and samples.++This is **not** a real embedding model: it is a hashed character-trigram bag-of-features+vector, chosen only because it is a pure function of the input text and is trivial to+reimplement bit-for-bit in C# with nothing but a string, UTF-8 encoding and 32-bit+integer arithmetic. It is deliberately named and documented as a demo: any real deployment+should pass `embed_fn_from_agent` (Python) or a real bi-encoder to the shortlist, not this.++The .NET port MUST reproduce this exact algorithm, because+`hashing_embedder_probe.json` and `case_shortlist_many_options.json` in the golden vectors+were both generated with it, and the .NET sample uses it too, so its+stdout can be diffed against the Python-recorded golden line for line.++Algorithm (this docstring is the spec -- read it precisely before porting):++1. Lowercase the input with a full Unicode case mapping (Python's `str.lower()`; C# must use+ `.ToLowerInvariant()` on the *string*, which uses full Unicode case folding rules too --+ NOT a per-UTF-16-unit simple mapping. This matters for characters whose lowercase form is+ more than one code point, e.g. `"İ".lower() == "i̇"` (Turkish capital I with dot+ above lowercases to 2 code points: 'i' + combining dot above).+2. Pad the lowercased text with two ASCII space characters (U+0020) on each side:+ `" " + text + " "`. Padding is added even when `text` is empty, so the padded text is+ never shorter than 4 code points and there are always at least 2 windows (see step 3):+ this is what lets short inputs like `""` or `"a"` still produce a nonzero, well-defined+ vector instead of an empty one.+3. Slide a window of exactly 3 **Unicode code points** (not UTF-16 code units, not grapheme+ clusters, not bytes) across the padded text, one code point at a time. For padded text of+ length `n` code points there are `n - 2` windows, at start offsets `0, 1, ..., n - 3`+ inclusive. A code point outside the Basic Multilingual Plane (an "astral" character, e.g.+ an emoji) counts as exactly one code point in this count -- a C# implementation that+ iterates `char` (UTF-16 code units) instead of Unicode scalar values will split a+ surrogate pair into two units and get a different window count and different window+ contents; it must enumerate Unicode scalar values instead (e.g. via `StringInfo` /+ `System.Globalization.StringInfo.GetTextElementEnumerator` is NOT correct either, since+ that walks grapheme clusters -- use rune/code-point enumeration, e.g. .NET's+ `System.Text.Rune` via `str.EnumerateRunes()`).+4. Encode each 3-code-point window as UTF-8 bytes (the window's own bytes, not the whole+ padded text's bytes -- re-encode the 3-code-point substring on its own each time).+5. Hash those UTF-8 bytes with 32-bit FNV-1a:++ hash = 0x811c9dc5 # FNV offset basis+ for byte in utf8_bytes:+ hash ^= byte+ hash = (hash * 0x01000193) & 0xFFFFFFFF # FNV prime, wrapped to 32 bits++ `hash` is an unsigned 32-bit integer throughout; the multiply must wrap (mask to+ 0xFFFFFFFF / use unchecked arithmetic in C#) exactly like Python's explicit+ `& 0xFFFFFFFF` above.+6. `bucket = hash % 64` (0..63, unsigned).+7. `sign = +1.0 if (hash >> 31) & 1 == 0 else -1.0` (the hash's top bit, as an unsigned+ 32-bit value -- this is a sign-hashing trick to reduce feature collisions, not a+ statement about the hash being negative).+8. Accumulate `sign` into `vector[bucket]` for every window (i.e. `vector[bucket] += sign`;+ multiple windows can and do land in the same bucket and their signed contributions add).+ The accumulator is float64 throughout.+9. No normalisation is applied. The shortlist's cosine similarity handles scale on its own,+ and leaving the raw counts in the golden vectors makes them easier to hand-check.++`embed(texts)` returns one such vector per input string, as a `(len(texts), 64)` float64+NumPy array, in input order. It does not batch, cache, or special-case any input; every text+is processed independently and only via the steps above.+"""+from typing import Sequence++import numpy as np++DIM = 64+_FNV_OFFSET_BASIS = 0x811C9DC5+_FNV_PRIME = 0x01000193+_MASK32 = 0xFFFFFFFF+++def _fnv1a_32(data: bytes) -> int:+ """32-bit FNV-1a over raw bytes, wrapping like an unsigned 32-bit integer."""+ h = _FNV_OFFSET_BASIS+ for b in data:+ h ^= b+ h = (h * _FNV_PRIME) & _MASK32+ return h+++def _embed_one(text: str) -> np.ndarray:+ vec = np.zeros(DIM, dtype=np.float64)+ padded = " " + text.lower() + " "+ n = len(padded)+ for i in range(n - 2):+ window = padded[i : i + 3]+ h = _fnv1a_32(window.encode("utf-8"))+ bucket = h % DIM+ sign = 1.0 if (h >> 31) & 1 == 0 else -1.0+ vec[bucket] += sign+ return vec+++def embed(texts: Sequence[str]) -> np.ndarray:+ """Embed a list of strings into a `(len(texts), 64)` float64 array. See the module docstring+ for the exact algorithm; every other-language port must match it bit-for-bit."""+ rows = [_embed_one("" if t is None else str(t)) for t in texts]+ if not rows:+ return np.zeros((0, DIM), dtype=np.float64)+ return np.stack(rows, axis=0)