Get up and running with Kimi, GLM, MiniMax, DeepSeek, gpt-oss, Qwen, Gemma and other models.
server: fix `context`, `load_duration` and `total_duration` fields (#5676)
Jeffrey Morgan1ed0aa8feab5
diff --git a/server/routes.go b/server/routes.goindex 4059c7c52..5b6d09788 100644--- a/server/routes.go+++ b/server/routes.go@@ -102,6 +102,7 @@ func (s *Server) scheduleRunner(ctx context.Context, name string, caps []Capabil}func (s *Server) GenerateHandler(c *gin.Context) {+ checkpointStart := time.Now()var req api.GenerateRequestif err := c.ShouldBindJSON(&req); errors.Is(err, io.EOF) {c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})@@ -129,6 +130,8 @@ func (s *Server) GenerateHandler(c *gin.Context) {return}+ checkpointLoaded := time.Now()+if req.Prompt == "" {c.JSON(http.StatusOK, api.GenerateResponse{Model: req.Model,@@ -191,26 +194,48 @@ func (s *Server) GenerateHandler(c *gin.Context) {ch := make(chan any)go func() {+ // TODO (jmorganca): avoid building the response twice both here and below+ var sb strings.Builderdefer close(ch)if err := r.Completion(c.Request.Context(), llm.CompletionRequest{Prompt: prompt,Images: images,Format: req.Format,Options: opts,- }, func(r llm.CompletionResponse) {- ch <- api.GenerateResponse{+ }, func(cr llm.CompletionResponse) {+ res := api.GenerateResponse{Model: req.Model,CreatedAt: time.Now().UTC(),- Response: r.Content,- Done: r.Done,- DoneReason: r.DoneReason,+ Response: cr.Content,+ Done: cr.Done,+ DoneReason: cr.DoneReason,Metrics: api.Metrics{- PromptEvalCount: r.PromptEvalCount,- PromptEvalDuration: r.PromptEvalDuration,- EvalCount: r.EvalCount,- EvalDuration: r.EvalDuration,+ PromptEvalCount: cr.PromptEvalCount,+ PromptEvalDuration: cr.PromptEvalDuration,+ EvalCount: cr.EvalCount,+ EvalDuration: cr.EvalDuration,},}++ if _, err := sb.WriteString(cr.Content); err != nil {+ ch <- gin.H{"error": err.Error()}+ }++ if cr.Done {+ res.TotalDuration = time.Since(checkpointStart)+ res.LoadDuration = checkpointLoaded.Sub(checkpointStart)++ if !req.Raw {+ tokens, err := r.Tokenize(c.Request.Context(), prompt+sb.String())+ if err != nil {+ ch <- gin.H{"error": err.Error()}+ return+ }+ res.Context = append(req.Context, tokens...)+ }+ }++ ch <- res}); err != nil {ch <- gin.H{"error": err.Error()}}@@ -1122,6 +1147,8 @@ func (s *Server) ProcessHandler(c *gin.Context) {}func (s *Server) ChatHandler(c *gin.Context) {+ checkpointStart := time.Now()+var req api.ChatRequestif err := c.ShouldBindJSON(&req); errors.Is(err, io.EOF) {c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})@@ -1141,6 +1168,8 @@ func (s *Server) ChatHandler(c *gin.Context) {return}+ checkpointLoaded := time.Now()+if len(req.Messages) == 0 {c.JSON(http.StatusOK, api.ChatResponse{Model: req.Model,@@ -1169,7 +1198,7 @@ func (s *Server) ChatHandler(c *gin.Context) {Format: req.Format,Options: opts,}, func(r llm.CompletionResponse) {- ch <- api.ChatResponse{+ res := api.ChatResponse{Model: req.Model,CreatedAt: time.Now().UTC(),Message: api.Message{Role: "assistant", Content: r.Content},@@ -1182,6 +1211,13 @@ func (s *Server) ChatHandler(c *gin.Context) {EvalDuration: r.EvalDuration,},}++ if r.Done {+ res.TotalDuration = time.Since(checkpointStart)+ res.LoadDuration = checkpointLoaded.Sub(checkpointStart)+ }++ ch <- res}); err != nil {ch <- gin.H{"error": err.Error()}}