Skip to content

Get up and running with Kimi, GLM, MiniMax, DeepSeek, gpt-oss, Qwen, Gemma and other models.

detect chat template from configs that contain lists

Michael Yang3eb08377f82a
diff --git a/convert/tokenizer.go b/convert/tokenizer.go
index 653df6d28..429d36e7c 100644
--- a/convert/tokenizer.go
+++ b/convert/tokenizer.go
@@ -100,8 +100,21 @@ func parseTokenizer(fsys fs.FS, specialTokenTypes []string) (*Tokenizer, error)
}
if template, ok := p["chat_template"]; ok {
- if err := json.Unmarshal(template, &t.Template); err != nil {
- return nil, err
+ var s []struct {
+ Name string `json:"name"`
+ Template string `json:"template"`
+ }
+ if err := json.Unmarshal(template, &t.Template); err == nil {
+ // noop
+ } else if err := json.Unmarshal(template, &s); err == nil {
+ for _, e := range s {
+ if e.Name == "default" {
+ t.Template = e.Template
+ break
+ }
+ }
+ } else {
+ return nil, fmt.Errorf("invalid chat_template: %w", err)
}
}
diff --git a/convert/tokenizer_test.go b/convert/tokenizer_test.go
new file mode 100644
index 000000000..ed0175a42
--- /dev/null
+++ b/convert/tokenizer_test.go
@@ -0,0 +1,96 @@
+package convert
+
+import (
+ "io"
+ "io/fs"
+ "os"
+ "path/filepath"
+ "strings"
+ "testing"
+
+ "github.com/google/go-cmp/cmp"
+)
+
+func createTokenizerFS(t *testing.T, dir string, files map[string]io.Reader) fs.FS {
+ t.Helper()
+
+ for k, v := range files {
+ if err := func() error {
+ f, err := os.Create(filepath.Join(dir, k))
+ if err != nil {
+ return err
+ }
+ defer f.Close()
+
+ if _, err := io.Copy(f, v); err != nil {
+ return err
+ }
+
+ return nil
+ }(); err != nil {
+ t.Fatalf("unexpected error: %v", err)
+ }
+ }
+
+ return os.DirFS(dir)
+}
+
+func TestParseTokenizer(t *testing.T) {
+ cases := []struct {
+ name string
+ fsys fs.FS
+ specialTokenTypes []string
+ want *Tokenizer
+ }{
+ {
+ name: "string chat template",
+ fsys: createTokenizerFS(t, t.TempDir(), map[string]io.Reader{
+ "tokenizer.json": strings.NewReader(`{}`),
+ "tokenizer_config.json": strings.NewReader(`{
+ "chat_template": "<default template>"
+ }`),
+ }),
+ want: &Tokenizer{
+ Vocabulary: &Vocabulary{Model: "gpt2"},
+ Pre: "default",
+ Template: "<default template>",
+ },
+ },
+ {
+ name: "list chat template",
+ fsys: createTokenizerFS(t, t.TempDir(), map[string]io.Reader{
+ "tokenizer.json": strings.NewReader(`{}`),
+ "tokenizer_config.json": strings.NewReader(`{
+ "chat_template": [
+ {
+ "name": "default",
+ "template": "<default template>"
+ },
+ {
+ "name": "tools",
+ "template": "<tools template>"
+ }
+ ]
+ }`),
+ }),
+ want: &Tokenizer{
+ Vocabulary: &Vocabulary{Model: "gpt2"},
+ Pre: "default",
+ Template: "<default template>",
+ },
+ },
+ }
+
+ for _, tt := range cases {
+ t.Run(tt.name, func(t *testing.T) {
+ tokenizer, err := parseTokenizer(tt.fsys, tt.specialTokenTypes)
+ if err != nil {
+ t.Fatalf("unexpected error: %v", err)
+ }
+
+ if diff := cmp.Diff(tt.want, tokenizer); diff != "" {
+ t.Errorf("unexpected tokenizer (-want +got):\n%s", diff)
+ }
+ })
+ }
+}