Skip to content

Get up and running with Kimi, GLM, MiniMax, DeepSeek, gpt-oss, Qwen, Gemma and other models.

llamarunner: update metrics

Michael Yangbbbc73d63784
diff --git a/runner/llamarunner/runner.go b/runner/llamarunner/runner.go
index ae26b52bf..7ed7ebb2b 100644
--- a/runner/llamarunner/runner.go
+++ b/runner/llamarunner/runner.go
@@ -82,10 +82,10 @@ type Sequence struct {
doneReason llm.DoneReason
// Metrics
- startProcessingTime time.Time
- startGenerationTime time.Time
- numDecoded int
- numPromptInputs int
+ processingDuration time.Duration
+ generationDuration time.Duration
+ numDecoded int
+ numPromptInputs int
}
type NewSequenceParams struct {
@@ -99,8 +99,6 @@ type NewSequenceParams struct {
func (s *Server) NewSequence(prompt string, images []llm.ImageData, params NewSequenceParams) (*Sequence, error) {
s.ready.Wait()
- startTime := time.Now()
-
inputs, err := s.inputs(prompt, images)
if err != nil {
return nil, fmt.Errorf("failed to process inputs: %w", err)
@@ -142,18 +140,17 @@ func (s *Server) NewSequence(prompt string, images []llm.ImageData, params NewSe
}
return &Sequence{
- inputs: inputs,
- numPromptInputs: len(inputs),
- startProcessingTime: startTime,
- numPredict: params.numPredict,
- pendingResponses: make([]string, 0),
- responses: make(chan string, 100),
- quit: make(chan bool, 1),
- embedding: make(chan []float32, 1),
- samplingCtx: sc,
- embeddingOnly: params.embedding,
- stop: params.stop,
- numKeep: params.numKeep,
+ inputs: inputs,
+ numPromptInputs: len(inputs),
+ numPredict: params.numPredict,
+ pendingResponses: make([]string, 0),
+ responses: make(chan string, 100),
+ quit: make(chan bool, 1),
+ embedding: make(chan []float32, 1),
+ samplingCtx: sc,
+ embeddingOnly: params.embedding,
+ stop: params.stop,
+ numKeep: params.numKeep,
}, nil
}
@@ -438,8 +435,8 @@ func (s *Server) processBatch(tokenBatch *llama.Batch, embedBatch *llama.Batch)
return nil
}
- err := s.lc.Decode(batch)
- if err != nil {
+ t := time.Now()
+ if err := s.lc.Decode(batch); err != nil {
return fmt.Errorf("failed to decode batch: %w", err)
}
@@ -459,9 +456,12 @@ func (s *Server) processBatch(tokenBatch *llama.Batch, embedBatch *llama.Batch)
continue
}
- seq.numDecoded += 1
- if seq.numDecoded == 1 {
- seq.startGenerationTime = time.Now()
+ s.lc.Synchronize()
+ seq.numDecoded++
+ if seq.numDecoded > 1 {
+ seq.generationDuration += time.Since(t)
+ } else {
+ seq.processingDuration += time.Since(t)
}
// if done processing the prompt, generate an embedding and return
@@ -646,9 +646,9 @@ func (s *Server) completion(w http.ResponseWriter, r *http.Request) {
Done: true,
DoneReason: seq.doneReason,
PromptEvalCount: seq.numPromptInputs,
- PromptEvalDuration: seq.startGenerationTime.Sub(seq.startProcessingTime),
+ PromptEvalDuration: seq.processingDuration,
EvalCount: seq.numDecoded,
- EvalDuration: time.Since(seq.startGenerationTime),
+ EvalDuration: seq.generationDuration,
}); err != nil {
http.Error(w, fmt.Sprintf("failed to encode final response: %v", err), http.StatusInternalServerError)
}