Skip to content

Get up and running with Kimi, GLM, MiniMax, DeepSeek, gpt-oss, Qwen, Gemma and other models.

use `int32_t` for call to tokenize (#4738)

Jeffrey Morgan763bb65dbb88
diff --git a/llm/llm.go b/llm/llm.go
index ead691f06..4492d39ff 100644
--- a/llm/llm.go
+++ b/llm/llm.go
@@ -63,12 +63,27 @@ func (llm *llamaModel) Tokenize(s string) []int {
cs := C.CString(s)
defer C.free(unsafe.Pointer(cs))
- tokens := make([]int, len(s)+2)
- if n := C.llama_tokenize(llm.m, cs, C.int(len(s)), (*C.llama_token)(unsafe.Pointer(&tokens[0])), C.int(len(s)+2), false, true); n > 0 {
- return tokens[:n]
+ ltokens := make([]C.llama_token, len(s)+2)
+ n := C.llama_tokenize(
+ llm.m,
+ cs,
+ C.int32_t(len(s)),
+ &ltokens[0],
+ C.int32_t(len(ltokens)),
+ false,
+ true,
+ )
+
+ if n < 0 {
+ return nil
}
- return nil
+ tokens := make([]int, n)
+ for i := 0; i < int(n); i++ {
+ tokens[i] = int(ltokens[i])
+ }
+
+ return tokens
}
func (llm *llamaModel) Detokenize(i32s []int) string {