Get up and running with Kimi, GLM, MiniMax, DeepSeek, gpt-oss, Qwen, Gemma and other models.
sample: add numerical stability to temperature/softmax transform (#9631)
Parth Sareen7e34f4fbfa19
diff --git a/sample/samplers.go b/sample/samplers.goindex a9d90692d..aea99b3f2 100644--- a/sample/samplers.go+++ b/sample/samplers.go@@ -90,8 +90,9 @@ func (s *Sampler) sample(tokens []token) (token, error) {sortLogits(tokens)}+ // token logit values are updated to probabilitiestokens = temperature(tokens, s.temperature)- tokens = softmax(tokens)+tokens = topP(tokens, s.topP)tokens = minP(tokens, s.minP)diff --git a/sample/transforms.go b/sample/transforms.goindex 496252975..ab62455f3 100644--- a/sample/transforms.go+++ b/sample/transforms.go@@ -5,13 +5,25 @@ import ("slices")-func softmax(ts []token) []token {+// temperature applies scaling and softmax to the logits+func temperature(ts []token, temp float32) []token {+ // Find max logit for numerical stability+ maxLogit := float32(math.Inf(-1))+ for _, t := range ts {+ if t.value > maxLogit {+ maxLogit = t.value+ }+ }++ // Apply temperature and compute exp(x - max)+ temp = max(temp, 1e-7)var sum float32for i, v := range ts {- ts[i].value = float32(math.Exp(float64(v.value)))+ ts[i].value = float32(math.Exp(float64((v.value - maxLogit) / temp)))sum += ts[i].value}+ // Normalizefor i := range ts {ts[i].value /= sum}@@ -19,27 +31,6 @@ func softmax(ts []token) []token {return ts}-func temperature(ti []token, t float32) []token {- if t == 1 {- return ti- }-- temp := max(t, 1e-7)- maxLogit := float32(math.Inf(-1))- for _, token := range ti {- if token.value > maxLogit {- maxLogit = token.value- }- }-- // subtracting max logit to avoid under/overflow- for i := range ti {- ti[i].value = (ti[i].value - maxLogit) / temp- }-- return ti-}-// siftDown maintains a min-heap property by recursively moving larger elements down the heap.//// The heap is represented as an array where for any node at index i:@@ -145,7 +136,8 @@ func minP(ts []token, p float32) []token {}// TODO(parthsareen): possibly replace with simpler implementation https://github.com/ollama/ollama/issues/9584-// Conting sort implementation to sort tokens by logits+// sortLogits sorts implementation to sort tokens by logits using counting sort+// counting sort is faster than built-in sort for this use casefunc sortLogits(tokens []token) {if len(tokens) <= 1 {returndiff --git a/sample/transforms_test.go b/sample/transforms_test.goindex 1065231dc..81e8849b7 100644--- a/sample/transforms_test.go+++ b/sample/transforms_test.go@@ -32,17 +32,9 @@ func compareLogits(t *testing.T, name string, want []float64, got []token) {}}-func TestTemperature(t *testing.T) {- input := []float64{2, -1, 4, -3, 1, -2, 0}- want := []float64{-4, -10, 0, -14, -6, -12, -8} // (logit - max logit) / temp-+func TestTemperatureAndSoftmax(t *testing.T) {+ input := []float64{1, 4, -2, 0}got := temperature(toTokens(input), 0.5)- compareLogits(t, "Temperature", want, got)-}--func TestSoftmax(t *testing.T) {- input := []float64{-3, -2, -1, 0, 1, 2, 4}- got := softmax(toTokens(input))// Check probabilities sum to 1var sum float32@@ -53,11 +45,14 @@ func TestSoftmax(t *testing.T) {t.Errorf("probabilities don't sum to 1: got %f", sum)}- // Check relative ordering is preserved- for i := 1; i < len(got); i++ {- if got[i].value < got[i-1].value {- t.Errorf("probability ordering not preserved at index %d", i)- }+ got = temperature(toTokens(input), 1)+ // Check probabilities sum to 1+ sum = 0.0+ for _, token := range got {+ sum += token.value+ }+ if math.Abs(float64(sum)-1.0) > 1e-6 {+ t.Errorf("probabilities don't sum to 1: got %f", sum)}}@@ -84,7 +79,6 @@ func TestTopP(t *testing.T) {// First apply temperature and softmax to get probabilitiestokens = temperature(tokens, 1)- tokens = softmax(tokens)sortLogits(tokens)// Then apply topP@@ -103,7 +97,6 @@ func TestMinP(t *testing.T) {// First apply temperature and softmaxtokens = temperature(tokens, 1)- tokens = softmax(tokens)// Then apply minPgot := minP(tokens, 0.2)