Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

8.8 KB
Raw Blame
using Laya.Tokenization;

namespace Laya.Tests;

/// <summary>
/// Tier 1: <see cref="HfTokenizer"/>'s special-token resolution against a minimal, synthetic
/// <c>tokenizer.json</c> in a temp directory, covering the split layout's missing
/// <c>tokenizer_config.json</c> without needing a real checkpoint.
/// </summary>
/// <remarks>
/// The fixture below is not a usable tokenizer (its "vocabulary" only has enough entries to prove
/// which ids got resolved), but it is enough to exercise <c>HfTokenizer</c>'s constructor end to
/// end: the native <c>Tokenizers.DotNet</c> wrapper does load it, since a WordPiece model only
/// needs a <c>vocab</c> map and an <c>unk_token</c> to be valid.
/// </remarks>
public sealed class HfTokenizerFallbackTests : IDisposable
{
    private readonly string _dir;

    public HfTokenizerFallbackTests()
    {
        _dir = Path.Combine(Path.GetTempPath(), "laya-tokenizer-tests-" + Guid.NewGuid().ToString("N"));
        Directory.CreateDirectory(_dir);
    }

    public void Dispose()
    {
        try { Directory.Delete(_dir, recursive: true); }
        catch (IOException) { /* best effort */ }
    }

    private static string AddedToken(string content, int id) =>
        $$"""{"id": {{id}}, "content": "{{content}}", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}""";

    /// <summary>A minimal but loadable WordPiece tokenizer.json using ModernBERT-style bracketed tokens.</summary>
    private const string BracketedVocab =
        """{"[PAD]":0,"[UNK]":1,"[CLS]":2,"[SEP]":3,"[MASK]":4,"hello":5}""";

    private void WriteTokenizerJson(string addedTokensJson, string vocabJson)
    {
        var json = $$"""
            {
              "version": "1.0",
              "truncation": null,
              "padding": null,
              "added_tokens": [{{addedTokensJson}}],
              "normalizer": null,
              "pre_tokenizer": {"type": "Whitespace"},
              "post_processor": null,
              "decoder": null,
              "model": {
                "type": "WordPiece",
                "unk_token": "[UNK]",
                "continuing_subword_prefix": "##",
                "max_input_chars_per_word": 100,
                "vocab": {{vocabJson}}
              }
            }
            """;
        File.WriteAllText(Path.Combine(_dir, "tokenizer.json"), json);
    }

    [Fact]
    public void DecodePreservesSpecialTokensAsPythonDoesForScanWindows()
    {
        var added = string.Join(",", [
            AddedToken("[PAD]", 0), AddedToken("[UNK]", 1), AddedToken("[CLS]", 2),
            AddedToken("[SEP]", 3), AddedToken("[MASK]", 4),
        ]);
        WriteTokenizerJson(added, BracketedVocab);
        using var tok = new HfTokenizer(Path.Combine(_dir, "tokenizer.json"));
        Assert.Equal("[CLS] hello [SEP]", tok.Decode([2, 5, 3]));
        Assert.Equal("", tok.Decode([]));
    }

    [Fact]
    public void DecoderCachesCanInitializeConcurrentlyWithoutChangingTheSourceOrEncoding()
    {
        var added = string.Join(",", [
            AddedToken("[PAD]", 0), AddedToken("[UNK]", 1), AddedToken("[CLS]", 2),
            AddedToken("[SEP]", 3), AddedToken("[MASK]", 4),
        ]);
        WriteTokenizerJson(added, BracketedVocab);
        var path = Path.Combine(_dir, "tokenizer.json");
        var original = File.ReadAllText(path);
        using var first = new HfTokenizer(path);
        using var second = new HfTokenizer(path);
        Parallel.Invoke(
            () => Assert.Equal("[CLS] hello [SEP]", first.Decode([2, 5, 3])),
            () => Assert.Equal("[CLS] hello [SEP]", second.Decode([2, 5, 3])));
        Assert.Equal(new[] { 2, 5, 3 }, first.Encode("[CLS] hello [SEP]"));
        Assert.Equal(original, File.ReadAllText(path));
    }

    [Fact]
    public void NoTokenizerConfig_ResolvesBracketedSpecialTokensFromAddedTokens()
    {
        // Split layout: no tokenizer_config.json beside tokenizer.json. Every role must be
        // resolved purely from added_tokens' conventional spellings.
        var added = string.Join(",", [
            AddedToken("[PAD]", 0), AddedToken("[UNK]", 1), AddedToken("[CLS]", 2),
            AddedToken("[SEP]", 3), AddedToken("[MASK]", 4),
        ]);
        WriteTokenizerJson(added, BracketedVocab);

        using var tok = new HfTokenizer(Path.Combine(_dir, "tokenizer.json"));

        Assert.Equal(0, tok.PadId);
        Assert.Equal(1, tok.UnkId);
        Assert.Equal(2, tok.ClsId);
        Assert.Equal(3, tok.SepId);
        Assert.Equal(4, tok.MaskId);
        Assert.Equal("[MASK]", tok.MaskToken);
    }

    [Fact]
    public void NoTokenizerConfig_ResolvesBosEosSpellingForClsAndSep()
    {
        // Some split exports use the BOS/EOS convention for CLS/SEP instead of the bracketed
        // ModernBERT form (e.g. mmBERT-style vocabularies). FirstKnownToken must fall through to
        // that spelling when the bracketed one is absent.
        var vocab = """{"<pad>":0,"<unk>":1,"<bos>":2,"<eos>":3,"<mask>":4,"hello":5}""";
        var added = string.Join(",", [
            AddedToken("<pad>", 0), AddedToken("<unk>", 1), AddedToken("<bos>", 2),
            AddedToken("<eos>", 3), AddedToken("<mask>", 4),
        ]);
        WriteTokenizerJson(added, vocab);

        using var tok = new HfTokenizer(Path.Combine(_dir, "tokenizer.json"));

        Assert.Equal(0, tok.PadId);
        Assert.Equal(1, tok.UnkId);
        Assert.Equal(2, tok.ClsId);
        Assert.Equal(3, tok.SepId);
        Assert.Equal(4, tok.MaskId);
        Assert.Equal("<mask>", tok.MaskToken);
    }

    [Fact]
    public void NoTokenizerConfig_TakesClsAndSepFromThePostProcessorTemplate()
    {
        // The real multilingual (mmBERT) vocabulary holds "<s>"/"</s>" as ordinary tokens as well
        // as its real CLS/SEP "<bos>"/"<eos>" (ids 2 and 1). The post_processor template names the
        // right pair, so it must win over any spelling guess.
        var vocab = """{"<pad>":0,"<eos>":1,"<bos>":2,"<unk>":3,"<mask>":4,"<s>":5,"</s>":6,"hello":7}""";
        var added = string.Join(",", [
            AddedToken("<pad>", 0), AddedToken("<eos>", 1), AddedToken("<bos>", 2),
            AddedToken("<unk>", 3), AddedToken("<mask>", 4), AddedToken("<s>", 5), AddedToken("</s>", 6),
        ]);
        WriteTokenizerJson(added, vocab);
        var path = Path.Combine(_dir, "tokenizer.json");
        File.WriteAllText(path, File.ReadAllText(path).Replace("\"post_processor\": null", """
            "post_processor": {"type": "TemplateProcessing", "single": [
              {"SpecialToken": {"id": "<bos>", "type_id": 0}},
              {"Sequence": {"id": "A", "type_id": 0}},
              {"SpecialToken": {"id": "<eos>", "type_id": 0}}]}
            """));

        using var tok = new HfTokenizer(path);

        Assert.Equal(2, tok.ClsId);
        Assert.Equal(1, tok.SepId);
        Assert.Equal(0, tok.PadId);
        Assert.Equal(3, tok.UnkId);
        Assert.Equal(4, tok.MaskId);
    }

    [Fact]
    public void NoTokenizerConfig_NoRecognisedSpelling_ThrowsNamingTheRole()
    {
        // A vocabulary that uses none of the conventional spellings for cls_token must fail
        // loudly and name which role could not be resolved, rather than silently picking an
        // unrelated id or crashing somewhere downstream with no context.
        var vocab = """{"[PAD]":0,"[UNK]":1,"[SEP]":3,"[MASK]":4,"hello":5}""";
        var added = string.Join(",", [
            AddedToken("[PAD]", 0), AddedToken("[UNK]", 1),
            AddedToken("[SEP]", 3), AddedToken("[MASK]", 4),
        ]);
        WriteTokenizerJson(added, vocab);

        var ex = Assert.Throws<InvalidOperationException>(
            () => new HfTokenizer(Path.Combine(_dir, "tokenizer.json")));

        Assert.Contains("cls_token", ex.Message);
    }

    [Fact]
    public void TokenizerConfigPresent_UsesItInsteadOfGuessing()
    {
        // Fused layout: tokenizer_config.json is present, so the config-driven path (not the
        // fallback) must be the one that runs, even though the fallback's conventional spellings
        // would also happen to resolve on this vocabulary.
        var added = string.Join(",", [
            AddedToken("[PAD]", 0), AddedToken("[UNK]", 1), AddedToken("[CLS]", 2),
            AddedToken("[SEP]", 3), AddedToken("[MASK]", 4),
        ]);
        WriteTokenizerJson(added, BracketedVocab);
        File.WriteAllText(Path.Combine(_dir, "tokenizer_config.json"), """
            {
              "cls_token": "[CLS]",
              "sep_token": "[SEP]",
              "pad_token": "[PAD]",
              "unk_token": "[UNK]",
              "mask_token": "[MASK]"
            }
            """);

        using var tok = new HfTokenizer(Path.Combine(_dir, "tokenizer.json"));

        Assert.Equal(0, tok.PadId);
        Assert.Equal(2, tok.ClsId);
        Assert.Equal(3, tok.SepId);
        Assert.Equal(4, tok.MaskId);
        Assert.Equal(1, tok.UnkId);
    }
}