Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

3.4 KB
Raw Blame
package tokenizer

import (
	"testing"

	"github.com/NandhaKishorM/laya/laya-go/internal/fixtures"
)

var shipped = [...]string{"english", "multilingual"}

// Every corpus entry is byte-identical to the ids Python's tokenizer gives, on both checkpoints.
func TestCorpusIDsMatchPython(t *testing.T) {
	fx := fixtures.Load(t, "tokenizer_ids.json")
	corpusText := fixtures.Obj(t, fx.Must("corpus_text"))
	total := 0
	for _, name := range shipped {
		side := fixtures.Obj(t, fx.Must(name))
		if _, skipped := side.Get("skipped"); skipped {
			t.Fatalf("the golden for %s is a skip marker; regenerate it with the checkpoints present", name)
		}
		tok := checkpoint(t, name)
		t.Run(name+"/identity", func(t *testing.T) {
			id := fixtures.Field(t, side, "identity")
			if got := fixtures.Str(t, fixtures.Field(t, id, "model_type")); got != "BPE" {
				t.Fatalf("model_type = %q", got)
			}
			want := fixtures.Int(t, fixtures.Field(t, id, "vocab_size"))
			if extra, ok := fixtures.Obj(t, id).Get("added_tokens_outside_vocab"); ok && extra != nil {
				want += fixtures.Int(t, extra)
			}
			if tok.VocabSize() != want {
				t.Errorf("VocabSize = %d, want the vocabulary plus added tokens (%d)", tok.VocabSize(), want)
			}
		})
		t.Run(name+"/specials", func(t *testing.T) {
			sp := fixtures.Field(t, side, "special_ids")
			for role, get := range map[string]func() (int32, bool){
				"cls": tok.CLS, "sep": tok.SEP, "mask": tok.Mask, "pad": tok.Pad,
			} {
				got, ok := get()
				if want := int32(fixtures.Int(t, fixtures.Field(t, sp, role))); !ok || got != want {
					t.Errorf("%s id = %d (found %v), want %d", role, got, ok, want)
				}
			}
			if got, want := tok.MaskToken(), fixtures.Str(t, fixtures.Field(t, sp, "mask_token")); got != want {
				t.Errorf("MaskToken = %q, want %q", got, want)
			}
		})
		for _, entry := range fixtures.Obj(t, fixtures.Field(t, side, "corpus")) {
			total++
			text := fixtures.Str(t, corpusText.Must(entry.Key))
			want := fixtures.Ints(t, entry.Value)
			t.Run(name+"/"+entry.Key, func(t *testing.T) {
				if got := tok.Encode(text); !equalIDs(got, want) {
					t.Errorf("ids for %q: %s", entry.Key, firstDifference(got, want))
				}
			})
		}
		t.Run(name+"/option-truncation-48", func(t *testing.T) {
			trunc := fixtures.Field(t, side, "option_truncation")
			text := " " + fixtures.Str(t, fixtures.Field(t, trunc, "text"))
			uncapped := fixtures.Ints(t, fixtures.Field(t, trunc, "uncapped"))
			capped := fixtures.Ints(t, fixtures.Field(t, trunc, "capped_48"))
			if got := tok.EncodeLimit(text, -1); !equalIDs(got, uncapped) {
				t.Errorf("uncapped: %s", firstDifference(got, uncapped))
			}
			if got := tok.EncodeLimit(text, 48); !equalIDs(got, capped) {
				t.Errorf("capped at 48: %s", firstDifference(got, capped))
			}
		})
	}
	if total < 70 {
		t.Errorf("only %d corpus entries ran, expected the whole corpus on both checkpoints", total)
	}
}

func TestEncodeLimitEdges(t *testing.T) {
	tok := checkpoint(t, "english")
	if got := tok.EncodeLimit("hello world", 0); len(got) != 0 {
		t.Errorf("limit 0 gave %v", got)
	}
	full := tok.Encode("hello world, this is a longer sentence")
	for n := 1; n <= len(full); n++ {
		if got := tok.EncodeLimit("hello world, this is a longer sentence", n); !equalIDs(got, full[:n]) {
			t.Errorf("limit %d is not a prefix of the full encoding", n)
		}
	}
	if got := tok.EncodeLimit("hello", 100); !equalIDs(got, tok.Encode("hello")) {
		t.Error("a limit above the length changes nothing")
	}
}