Get up and running with Kimi, GLM, MiniMax, DeepSeek, gpt-oss, Qwen, Gemma and other models.
mlxrunner: tokenize prompts in request handler goroutines
Jesse Grossce99f2473177
diff --git a/x/mlxrunner/client.go b/x/mlxrunner/client.goindex fa55fbd71..4730a10d0 100644--- a/x/mlxrunner/client.go+++ b/x/mlxrunner/client.go@@ -226,7 +226,7 @@ func (c *Client) Completion(ctx context.Context, req llm.CompletionRequest, fn fif resp.StatusCode != http.StatusOK {respBody, _ := io.ReadAll(resp.Body)- return fmt.Errorf("%s", strings.TrimSpace(string(respBody)))+ return api.StatusError{StatusCode: resp.StatusCode, ErrorMessage: strings.TrimSpace(string(respBody))}}scanner := bufio.NewScanner(resp.Body)diff --git a/x/mlxrunner/pipeline.go b/x/mlxrunner/pipeline.goindex b1eef58a5..6297c220b 100644--- a/x/mlxrunner/pipeline.go+++ b/x/mlxrunner/pipeline.go@@ -6,11 +6,9 @@ import ("errors""fmt""log/slog"- "net/http""sort""time"- "github.com/ollama/ollama/api""github.com/ollama/ollama/llm""github.com/ollama/ollama/logutil""github.com/ollama/ollama/x/mlxrunner/mlx"@@ -22,13 +20,37 @@ func prefillChunkSize() int {return 2 << 10}-func (r *Runner) TextGenerationPipeline(request Request) error {+// Prepare tokenizes the prompt and validates it against the model's+// context length. It is safe to call from any goroutine. On success it+// populates request.Tokens and adjusts request.Options.NumPredict.+func (r *Runner) Prepare(request *Request) error {if r.Model == nil {return errors.New("model not loaded")}+ tokens := r.Tokenizer.Encode(request.Prompt, r.Tokenizer.AddBOS())+ if len(tokens) == 0 {+ return errors.New("empty prompt")+ }++ if len(tokens) >= r.contextLength {+ return fmt.Errorf("input length (%d tokens) exceeds the model's maximum context length (%d tokens)", len(tokens), r.contextLength)+ }++ // Cap generation to stay within the model's context length+ maxGenerate := r.contextLength - len(tokens)+ if request.Options.NumPredict <= 0 {+ request.Options.NumPredict = maxGenerate+ } else {+ request.Options.NumPredict = min(request.Options.NumPredict, maxGenerate)+ }++ request.Tokens = tokens+ return nil+}++func (r *Runner) TextGenerationPipeline(ctx context.Context, request Request) error {mlx.ResetPeakMemory()- ctx := request.Ctxvar sample, nextSample sampler.Resultdefer func() {@@ -47,26 +69,7 @@ func (r *Runner) TextGenerationPipeline(request Request) error {slog.Info("peak memory", "size", mlx.PrettyBytes(mlx.PeakMemory()))}()- inputs := r.Tokenizer.Encode(request.Prompt, r.Tokenizer.AddBOS())- if len(inputs) == 0 {- return errors.New("empty prompt")- }-- if len(inputs) >= r.contextLength {- return api.StatusError{- StatusCode: http.StatusBadRequest,- ErrorMessage: fmt.Sprintf("input length (%d tokens) exceeds the model's maximum context length (%d tokens)", len(inputs), r.contextLength),- }- }-- // Cap generation to stay within the model's context length- maxGenerate := r.contextLength - len(inputs)- if request.Options.NumPredict <= 0 {- request.Options.NumPredict = maxGenerate- } else {- request.Options.NumPredict = min(request.Options.NumPredict, maxGenerate)- }-+ inputs := request.Tokensrequest.Sampler.ResetHistory(inputs)session := r.cache.begin(r.Model, inputs)diff --git a/x/mlxrunner/runner.go b/x/mlxrunner/runner.goindex 95e1f2b62..2ab5e323a 100644--- a/x/mlxrunner/runner.go+++ b/x/mlxrunner/runner.go@@ -18,13 +18,17 @@ import ("github.com/ollama/ollama/x/tokenizer")+// Request is a short-lived struct that carries a completion request through+// a channel from the HTTP handler to the runner goroutine. The ctx field+// must travel with the request so that cancellation propagates across the+// channel boundary.type Request struct {CompletionRequestResponses chan CompletionResponse- Pipeline func(Request) error-- Ctx context.Context+ Pipeline func(context.Context, Request) error+ Ctx context.Context //nolint:containedctx+ Tokens []int32Sampler *sample.Sampler}@@ -131,7 +135,7 @@ func (r *Runner) Run(host, port string, mux http.Handler) error {case <-ctx.Done():return nilcase request := <-r.Requests:- if err := request.Pipeline(request); err != nil {+ if err := request.Pipeline(request.Ctx, request); err != nil {slog.Info("Request terminated", "error", err)var statusErr api.StatusErrorif !errors.As(err, &statusErr) {diff --git a/x/mlxrunner/server.go b/x/mlxrunner/server.goindex 089e5c7f5..41d8c976f 100644--- a/x/mlxrunner/server.go+++ b/x/mlxrunner/server.go@@ -106,6 +106,11 @@ func Execute(args []string) error {TopLogprobs: request.TopLogprobs,})+ if err := runner.Prepare(&request); err != nil {+ http.Error(w, err.Error(), http.StatusBadRequest)+ return+ }+var cancel context.CancelFuncrequest.Ctx, cancel = context.WithCancel(r.Context())defer cancel()