Get up and running with Kimi, GLM, MiniMax, DeepSeek, gpt-oss, Qwen, Gemma and other models.
model: validate left and right pairs before merging them
jmorganca20e35938630e
diff --git a/model/process_text_spm.go b/model/process_text_spm.goindex 9d61746a2..68e3ed015 100644--- a/model/process_text_spm.go+++ b/model/process_text_spm.go@@ -169,6 +169,10 @@ func (spm SentencePieceModel) Encode(s string, addSpecial bool) ([]int32, error)continue}+ if id := spm.vocab.Encode(string(left.runes) + string(right.runes)); id < 0 {+ continue+ }+merges[pair.a].runes = append(left.runes, right.runes...)merges[pair.b].runes = nilmerges[pair.a].n = right.n