Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

12.3 KB
Raw Blame
package tokenizer

import (
	"encoding/json"
	"errors"
	"os"
	"strconv"
	"strings"
	"testing"
)

// A tiny byte-level tokenizer built in the test, so the loader is checked without a checkpoint.
func byteLevelSpec(extra string) string {
	vocab := make([]string, 0, 260)
	for b := 0; b < 256; b++ {
		key, _ := json.Marshal(string(byteAlphabet.toRune[b]))
		vocab = append(vocab, string(key)+`:`+strconv.Itoa(b))
	}
	vocab = append(vocab, `"ab":256`)
	return `{"model":{"type":"BPE","vocab":{` + strings.Join(vocab, ",") + `},"merges":["a b"]},` +
		`"normalizer":null,"pre_tokenizer":{"type":"ByteLevel","add_prefix_space":false,"use_regex":true}` + extra + `}`
}

func TestSmallTokenizerRoundTrips(t *testing.T) {
	tok, err := Parse([]byte(byteLevelSpec(`,"decoder":{"type":"ByteLevel"}`)), nil)
	if err != nil {
		t.Fatal(err)
	}
	ids := tok.Encode("ab ab")
	if got := tok.Decode(ids, true); got != "ab ab" {
		t.Errorf("round trip = %q (ids %v)", got, ids)
	}
	if ids[0] != 256 {
		t.Errorf("the a b merge must fire, got %v", ids)
	}
	if tok.VocabSize() != 257 || tok.HighestID() != 256 {
		t.Errorf("VocabSize = %d, HighestID = %d", tok.VocabSize(), tok.HighestID())
	}
	if s, ok := tok.IDToToken(256); !ok || s != "ab" {
		t.Errorf("IDToToken(256) = %q, %v", s, ok)
	}
	if _, ok := tok.IDToToken(-1); ok {
		t.Error("a negative id has no token")
	}
	if id, ok := tok.TokenToID("ab"); !ok || id != 256 {
		t.Errorf("TokenToID = %d, %v", id, ok)
	}
}

func TestLoaderRefuses(t *testing.T) {
	good := byteLevelSpec("")
	for name, c := range map[string]struct {
		spec        string
		unsupported bool
	}{
		"a non-BPE model":                              {strings.Replace(good, `"BPE"`, `"WordPiece"`, 1), true},
		"an unknown normalizer":                        {strings.Replace(good, `"normalizer":null`, `"normalizer":{"type":"BertNormalizer"}`, 1), true},
		"an unknown pre-tokenizer":                     {strings.Replace(good, `"pre_tokenizer":{"type":"ByteLevel"`, `"pre_tokenizer":{"type":"Whitespace"`, 1), true},
		"an unknown decoder":                           {byteLevelSpec(`,"decoder":{"type":"WordPiece"}`), true},
		"a merge naming a symbol the vocabulary lacks": {strings.Replace(good, `"a b"`, `"a zz"`, 1), false},
		"a merge whose result is missing":              {strings.Replace(good, `"a b"`, `"a c"`, 1), false},
		"a merge that is not a pair":                   {strings.Replace(good, `"a b"`, `["a"]`, 1), false},
		"a space-joined merge with an empty half":      {strings.Replace(good, `"a b"`, `"a "`, 1), false},
		"an added token with no id":                    {byteLevelSpec(`,"added_tokens":[{"content":"x"}]`), false},
		"two added tokens sharing an id":               {byteLevelSpec(`,"added_tokens":[{"content":"x","id":300},{"content":"y","id":300}]`), false},
		"a vocabulary giving one id to two tokens":     {strings.Replace(good, `"ab":256`, `"ab":0`, 1), false},
		"no vocabulary":                                {`{"model":{"type":"BPE","merges":[]}}`, false},
		"not JSON":                                     {`{`, false},
	} {
		t.Run(name, func(t *testing.T) {
			_, err := Parse([]byte(c.spec), nil)
			if err == nil {
				t.Fatal("must be refused, not loaded")
			}
			if got := errors.Is(err, ErrUnsupported); got != c.unsupported {
				t.Errorf("errors.Is(err, ErrUnsupported) = %v for %v", got, err)
			}
		})
	}
}

func TestLoaderRefusesALossyTokenizer(t *testing.T) {
	// No unk_token, no byte fallback, and not byte-level: a character outside the vocabulary
	// would vanish. The same vocabulary missing alphabet symbols, byte-level, is refused too.
	lossy := `{"model":{"type":"BPE","vocab":{"a":0},"merges":[]}}`
	if _, err := Parse([]byte(lossy), nil); !errors.Is(err, ErrUnsupported) {
		t.Errorf("a non-byte-level tokenizer that can drop text: %v", err)
	}
	partial := `{"model":{"type":"BPE","vocab":{"a":0},"merges":[]},"pre_tokenizer":{"type":"ByteLevel"}}`
	if _, err := Parse([]byte(partial), nil); err == nil || !strings.Contains(err.Error(), "absent from its vocabulary") {
		t.Errorf("a byte-level tokenizer missing alphabet symbols: %v", err)
	}
	withUnk := `{"model":{"type":"BPE","vocab":{"a":0,"<unk>":1},"merges":[],"unk_token":"<unk>"}}`
	// Expected ids are what the crate returns for the same model.
	for _, c := range []struct {
		fuse bool
		text string
		want []int32
	}{
		{false, "azz a", []int32{0, 1, 1, 1, 0}},
		{false, "zzz", []int32{1, 1, 1}},
		{false, "zz a zz", []int32{1, 1, 1, 0, 1, 1, 1}},
		{true, "azz a", []int32{0, 1, 0}},
		{true, "zzz", []int32{1}},
		{true, "zz a zz", []int32{1, 0, 1}},
	} {
		spec := withUnk
		if c.fuse {
			spec = strings.Replace(withUnk, `"unk_token"`, `"fuse_unk":true,"unk_token"`, 1)
		}
		tok, err := Parse([]byte(spec), nil)
		if err != nil {
			t.Fatal(err)
		}
		if got := tok.Encode(c.text); !equalIDs(got, c.want) {
			t.Errorf("fuse_unk=%v Encode(%q) = %v, want %v", c.fuse, c.text, got, c.want)
		}
	}
}

func TestAddedTokens(t *testing.T) {
	spec := byteLevelSpec(`,"added_tokens":[` +
		`{"content":"<S>","id":300,"special":true,"normalized":false},` +
		`{"content":"<SS>","id":301,"special":true,"normalized":false},` +
		`{"content":"[M]","id":302,"special":true,"lstrip":true,"rstrip":true},` +
		`{"content":"word","id":303,"single_word":true},` +
		`{"content":"éclair","id":304,"normalized":true},` +
		`{"content":"§","id":305},{"content":"!","id":306}],` +
		`"decoder":{"type":"ByteLevel"}`)
	tok, err := Parse([]byte(spec), nil)
	if err != nil {
		t.Fatal(err)
	}
	if got := tok.Encode("<SS><S>"); !equalIDs(got, []int32{301, 300}) {
		t.Errorf("the longest token wins at a position, got %v", got)
	}
	if got := tok.Encode("a   [M]   b"); !equalIDs(got, []int32{97, 302, 98}) {
		t.Errorf("lstrip and rstrip absorb the white space around a token, got %v", got)
	}
	if got := tok.Encode("word"); !equalIDs(got, []int32{303}) {
		t.Errorf("a single_word token alone, got %v", got)
	}
	if got := tok.Encode("swords word."); len(got) < 3 || got[len(got)-2] != 303 {
		t.Errorf("a single_word token inside a word is not matched but a free one is, got %v", got)
	}
	if got := tok.Encode("éclair"); !equalIDs(got, []int32{304}) {
		t.Errorf("a normalized token matches after normalization, got %v", got)
	}
	if got := tok.Encode("a\u00a7b!"); !equalIDs(got, []int32{97, 305, 98, 306}) {
		t.Errorf("one-character tokens, ASCII and not, got %v", got)
	}
	if got := tok.Encode("<S"); !equalIDs(got, []int32{60, 83}) {
		t.Errorf("a prefix of a token is not the token, got %v", got)
	}
	if got := tok.Decode([]int32{300, 97, 301}, true); got != "a" {
		t.Errorf("special tokens are skipped on request, got %q", got)
	}
	if got := tok.Decode([]int32{300, 97}, false); got != "<S>a" {
		t.Errorf("and kept otherwise, got %q", got)
	}
	if id, ok := tok.SpecialID("<S>"); !ok || id != 300 {
		t.Errorf("SpecialID = %d, %v", id, ok)
	}
	if _, ok := tok.SpecialID("word"); ok {
		t.Error("an added token that is not special is not in the special table")
	}
	if n := len(tok.SpecialTokens()); n != 3 {
		t.Errorf("%d special tokens", n)
	}
}

func TestRolesFromTokenizerConfig(t *testing.T) {
	dir := t.TempDir()
	writeFile(t, dir+"/tokenizer.json", byteLevelSpec(`,"added_tokens":[{"content":"<c>","id":300,"special":true},{"content":"<m>","id":301,"special":true}]`))
	writeFile(t, dir+"/tokenizer_config.json", `{"cls_token":"<c>","mask_token":{"content":"<m>"},"pad_token":null}`)
	tok, err := LoadFS(os.DirFS(dir), ".")
	if err != nil {
		t.Fatal(err)
	}
	if id, ok := tok.CLS(); !ok || id != 300 {
		t.Errorf("CLS = %d, %v", id, ok)
	}
	if id, ok := tok.Mask(); !ok || id != 301 || tok.MaskToken() != "<m>" {
		t.Errorf("Mask = %d, %v, %q", id, ok, tok.MaskToken())
	}
	if _, ok := tok.Pad(); ok {
		t.Error("a null role is absent")
	}
	if _, ok := tok.SEP(); ok {
		t.Error("an unnamed role is absent")
	}
	writeFile(t, dir+"/tokenizer_config.json", `{"cls_token":"<nope>"}`)
	if _, err := LoadFS(os.DirFS(dir), "."); err == nil {
		t.Error("a role naming a token the vocabulary lacks must be refused")
	}
	writeFile(t, dir+"/tokenizer_config.json", `{"cls_token":5}`)
	if _, err := LoadFS(os.DirFS(dir), "."); err == nil {
		t.Error("a role that is not a string or an object must be refused")
	}
	// Load prefers <dir>/tokenizer/ when it holds a tokenizer.json.
	model := t.TempDir()
	writeFile(t, model+"/tokenizer/tokenizer.json", byteLevelSpec(""))
	if _, err := LoadFS(os.DirFS(model), "."); err != nil {
		t.Errorf("LoadFS(model dir): %v", err)
	}
	sub := t.TempDir()
	writeFile(t, sub+"/models/m/tokenizer.json", byteLevelSpec(""))
	if _, err := LoadFS(os.DirFS(sub), "models/m"); err != nil {
		t.Errorf("LoadFS(sub directory): %v", err)
	}
	if _, err := LoadFS(os.DirFS(t.TempDir()), "."); err == nil {
		t.Error("a directory with no tokenizer must be an error")
	}
}

func TestNormalizerAndPreTokenizerRefusals(t *testing.T) {
	for name, spec := range map[string]string{
		"an unknown normalizer":          `{"type":"BertNormalizer"}`,
		"a typeless normalizer":          `{}`,
		"a Replace with no content":      `{"type":"Replace","pattern":{"String":"a"}}`,
		"a Replace of the empty string":  `{"type":"Replace","pattern":{"String":""},"content":"x"}`,
		"a Replace with no pattern kind": `{"type":"Replace","pattern":{},"content":"x"}`,
		"a bad regex":                    `{"type":"Replace","pattern":{"Regex":"("},"content":"x"}`,
		"a Sequence with no steps":       `{"type":"Sequence"}`,
		"a Sequence of non-objects":      `{"type":"Sequence","normalizers":[1]}`,
	} {
		t.Run(name, func(t *testing.T) {
			n, err := parseNode([]byte(spec), "normalizer")
			if err != nil {
				t.Fatal(err)
			}
			if _, err := parseNormalizer(n); err == nil {
				t.Error("must be refused")
			}
		})
	}
	// The reference reads \s, \d, \w and \b as Unicode; Go's RE2 reads them as ASCII.
	for _, expr := range []string{`\\s+`, `a\\bb`, `[\\d]`, `\\w`, `\\p{L}`} {
		n, _ := parseNode([]byte(`{"type":"Replace","pattern":{"Regex":"`+expr+`"},"content":"x"}`), "normalizer")
		if _, err := parseNormalizer(n); !errors.Is(err, ErrUnsupported) {
			t.Errorf("the regex %s must be refused as ErrUnsupported, got %v", expr, err)
		}
	}
	n, _ := parseNode([]byte(`{"type":"Replace","pattern":{"Regex":"\\\\\\\\s"},"content":"x"}`), "normalizer")
	if norm, err := parseNormalizer(n); err != nil || norm(`a\\s`) != `ax` {
		t.Errorf("an escaped backslash before s is a literal, not a class: %v", err)
	}
	for name, spec := range map[string]string{
		"an unknown pre-tokenizer":    `{"type":"Whitespace"}`,
		"a typeless pre-tokenizer":    `{}`,
		"a two-character Metaspace":   `{"type":"Metaspace","replacement":"ab"}`,
		"an unknown Metaspace scheme": `{"type":"Metaspace","prepend_scheme":"sometimes"}`,
		"a Sequence of non-objects":   `{"type":"Sequence","pretokenizers":[1]}`,
	} {
		t.Run(name, func(t *testing.T) {
			n, _ := parseNode([]byte(spec), "pre_tokenizer")
			if _, err := parsePreTokenizer(n); err == nil {
				t.Error("must be refused")
			}
		})
	}
	n, _ = parseNode([]byte(`{"type":"Whitespace"}`), "pre_tokenizer")
	if _, err := parsePreTokenizer(n); !errors.Is(err, ErrUnsupported) {
		t.Errorf("an unimplemented pre-tokenizer must wrap ErrUnsupported, got %v", err)
	}
}

func TestPreTokenizerSequence(t *testing.T) {
	n, _ := parseNode([]byte(`{"type":"Sequence","pretokenizers":[{"type":"Metaspace","prepend_scheme":"never"},{"type":"ByteLevel","add_prefix_space":false}]}`), "pre_tokenizer")
	p, err := parsePreTokenizer(n)
	if err != nil {
		t.Fatal(err)
	}
	// The crate gives ['ab', 'âĸģ', 'cd'] for the same sequence.
	if got := strings.Join(p.preTokenize("ab cd"), "|"); got != "ab|\u00e2\u0138\u0123|cd" {
		t.Errorf("sequence = %q", got)
	}
	if got := (wholeText{}).preTokenize(""); got != nil {
		t.Errorf("empty text has no pieces, got %v", got)
	}
}

func TestPreTokenizerCornerCases(t *testing.T) {
	p := byteLevel{addPrefixSpace: false, useRegex: true}
	for _, c := range []struct {
		in   string
		want []string
	}{
		// An inner white-space run gives its last character back to the next piece.
		{"a  b\t\tc\n\nd\r\ne   f", []string{"a", "Ġ", "Ġb", "ĉ", "ĉ", "c", "Ċ", "Ċ", "d", "č", "Ċ", "e", "ĠĠ", "Ġf"}},
		{"abc123", []string{"abc", "123"}},
		{"123abc", []string{"123", "abc"}},
		{"a 1 b", []string{"a", "Ġ1", "Ġb"}},
		{"x's value", []string{"x", "'s", "Ġvalue"}},
		{"", nil},
	} {
		got := p.preTokenize(c.in)
		if strings.Join(got, "|") != strings.Join(c.want, "|") || len(got) != len(c.want) {
			t.Errorf("preTokenize(%q) = %q, want %q", c.in, got, c.want)
		}
	}
}