Skip to content

Get up and running with Kimi, GLM, MiniMax, DeepSeek, gpt-oss, Qwen, Gemma and other models.

106 KB
Raw Blame
package server

import (
	"bytes"
	"cmp"
	"context"
	"encoding/base64"
	"encoding/json"
	"errors"
	"fmt"
	"image"
	"io"
	"io/fs"
	"log/slog"
	"math"
	"math/rand"
	"net"
	"net/http"
	"net/netip"
	"net/url"
	"os"
	"os/signal"
	"slices"
	"strings"
	"sync/atomic"
	"syscall"
	"time"

	"github.com/gin-contrib/cors"
	"github.com/gin-gonic/gin"
	"golang.org/x/image/webp"
	"golang.org/x/sync/errgroup"

	"github.com/ollama/ollama/api"
	"github.com/ollama/ollama/auth"
	"github.com/ollama/ollama/decision"
	"github.com/ollama/ollama/discover"
	"github.com/ollama/ollama/envconfig"
	"github.com/ollama/ollama/format"
	"github.com/ollama/ollama/fs/gguf"
	internalcloud "github.com/ollama/ollama/internal/cloud"
	"github.com/ollama/ollama/internal/proxy"
	"github.com/ollama/ollama/llm"
	"github.com/ollama/ollama/logutil"
	"github.com/ollama/ollama/manifest"
	"github.com/ollama/ollama/middleware"
	"github.com/ollama/ollama/model/parsers"
	"github.com/ollama/ollama/model/renderers"
	"github.com/ollama/ollama/template"
	thinkingparser "github.com/ollama/ollama/thinking"
	"github.com/ollama/ollama/tools"
	"github.com/ollama/ollama/types/errtypes"
	"github.com/ollama/ollama/types/model"
	"github.com/ollama/ollama/version"
)

const signinURLStr = "https://ollama.com/connect?name=%s&key=%s"

const (
	cloudErrRemoteInferenceUnavailable    = "remote model is unavailable"
	cloudErrRemoteModelDetailsUnavailable = "remote model details are unavailable"
	cloudErrWebSearchUnavailable          = "web search is unavailable"
	cloudErrWebFetchUnavailable           = "web fetch is unavailable"
	cloudErrBalanceUnavailable            = "balance is unavailable"
	cloudErrUsageUnavailable              = "usage is unavailable"
	copilotChatUserAgentPrefix            = "GitHubCopilotChat/"
)

func writeModelRefParseError(c *gin.Context, err error, fallbackStatus int, fallbackMessage string) {
	switch {
	case errors.Is(err, errConflictingModelSource):
		c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
	case errors.Is(err, model.ErrUnqualifiedName):
		c.JSON(http.StatusBadRequest, gin.H{"error": errtypes.InvalidModelNameErrMsg})
	default:
		c.JSON(fallbackStatus, gin.H{"error": fallbackMessage})
	}
}

func shouldUseHarmony(model *Model) bool {
	if slices.Contains([]string{"gptoss", "gpt-oss"}, model.Config.ModelFamily) {
		// heuristic to check whether the template expects to be parsed via harmony:
		// search for harmony tags that are nearly always used
		if model.Template.Contains("<|start|>") && model.Template.Contains("<|end|>") {
			return true
		}
	}

	return false
}

func parserNameForModel(m *Model) string {
	if m.Config.Parser != "" {
		return m.Config.Parser
	}
	if shouldUseHarmony(m) {
		return "harmony"
	}
	return ""
}

func experimentEnabled(name string) bool {
	return slices.Contains(strings.Split(os.Getenv("OLLAMA_EXPERIMENT"), ","), name)
}

var useClient2 = experimentEnabled("client2")

var mode string = gin.DebugMode

type Server struct {
	addr          net.Addr
	sched         *Scheduler
	defaultNumCtx int
	requestLogger *inferenceRequestLogger
	modelCaches   *modelCaches
}

func init() {
	switch mode {
	case gin.DebugMode:
	case gin.ReleaseMode:
	case gin.TestMode:
	default:
		mode = gin.DebugMode
	}

	gin.SetMode(mode)

	// Tell renderers to use [img] tags
	renderers.RenderImgTags = true
}

var errRequired = errors.New("is required")

func (s *Server) modelOptions(model *Model, requestOpts map[string]any) (api.Options, error) {
	return s.modelOptionsWithEmbeddingBatchDefault(model, requestOpts, shouldApplyEmbeddingBatchDefault(model, requestOpts))
}

func (s *Server) modelOptionsWithEmbeddingBatchDefault(model *Model, requestOpts map[string]any, applyEmbeddingBatchDefault bool) (api.Options, error) {
	opts := api.DefaultOptions()
	if opts.NumCtx == 0 {
		opts.NumCtx = s.defaultNumCtx
	}

	// api.Options stores defaulted values, so lower layers cannot distinguish
	// an unset draft_num_predict from the default. Track that while we still
	// have the raw model/request option maps.
	draftNumPredictSet := hasOption(requestOpts, "draft_num_predict")
	if model != nil {
		draftNumPredictSet = draftNumPredictSet || hasOption(model.Options, "draft_num_predict")
		if err := opts.FromMap(model.GenerationDefaults); err != nil {
			return api.Options{}, err
		}
		if err := opts.FromMap(model.Options); err != nil {
			return api.Options{}, err
		}
	}

	warnDeprecatedOptions(requestOpts)
	if err := opts.FromMap(requestOpts); err != nil {
		return api.Options{}, err
	}

	if applyEmbeddingBatchDefault {
		opts.NumBatch = llm.DefaultEmbeddingNumBatch
	}

	if model != nil && model.DraftPath == "" && !draftNumPredictSet {
		opts.DraftNumPredict = 0
	}

	return opts, nil
}

func shouldApplyEmbeddingBatchDefault(m *Model, requestOpts map[string]any) bool {
	if m == nil || hasOption(m.Options, "num_batch") || hasOption(requestOpts, "num_batch") {
		return false
	}
	if slices.Contains(m.Config.Capabilities, string(model.CapabilityEmbedding)) {
		return true
	}
	return m.ModelPath != "" && m.CheckCapabilities(model.CapabilityEmbedding) == nil
}

func hasOption(opts map[string]any, name string) bool {
	_, ok := opts[name]
	return ok
}

// deprecatedOptions may still be set per request, and reach the runner, but
// cannot be saved as model parameters.
var deprecatedOptions = []string{"typical_p"}

func warnDeprecatedOptions(opts map[string]any) {
	for _, name := range deprecatedOptions {
		if opts[name] != nil {
			slog.Warn("deprecated option provided", "option", name)
		}
	}
}

func usesAutomaticNumCtx(model *Model, requestOpts map[string]any) bool {
	if _, ok := requestOpts["num_ctx"]; ok {
		return false
	}
	if model != nil {
		if _, ok := model.Options["num_ctx"]; ok {
			return false
		}
	}
	return envconfig.ContextLength() == 0
}

func usesAutomaticNumBatch(model *Model, requestOpts map[string]any) bool {
	if _, ok := requestOpts["num_batch"]; ok {
		return false
	}
	if model != nil {
		if _, ok := model.Options["num_batch"]; ok {
			return false
		}
	}
	return true
}

func normalizeRunner(runner string) (string, error) {
	switch strings.ToLower(strings.TrimSpace(runner)) {
	case "":
		return "", nil
	case manifest.RunnerMLX, "mlxrunner":
		return manifest.RunnerMLX, nil
	case manifest.RunnerGGML:
		return manifest.RunnerGGML, nil
	case manifest.RunnerLlamaCPP, "llama.cpp", "llama-cpp", "llama_cpp":
		return manifest.RunnerLlamaCPP, nil
	default:
		return "", fmt.Errorf("unknown runner %q", runner)
	}
}

// scheduleRunner schedules a runner after validating inputs such as capabilities and model options.
// It returns the allocated runner, model instance, and consolidated options if successful and error otherwise.
func (s *Server) scheduleRunner(ctx context.Context, name, selectedRunner string, caps []model.Capability, requestOpts map[string]any, keepAlive *api.Duration, shift *bool) (llm.LlamaServer, *Model, *api.Options, error) {
	if name == "" {
		return nil, nil, nil, fmt.Errorf("model %w", errRequired)
	}

	model, err := s.getExistingModelForRunner(name, selectedRunner)
	if err != nil {
		return nil, nil, nil, err
	}
	return s.scheduleRunnerForModel(ctx, model, caps, requestOpts, keepAlive, shift)
}

// scheduleRunnerForModel retains the request's resolved model, including its selected
// manifest-list child, without repeating model lookup and metadata loading.
func (s *Server) scheduleRunnerForModel(ctx context.Context, m *Model, caps []model.Capability, requestOpts map[string]any, keepAlive *api.Duration, shift *bool) (llm.LlamaServer, *Model, *api.Options, error) {
	// TODO: enable compatibility migration
	// selectedName := model.ParseName(m.Name)
	// if !manifest.IsDigestReferenceName(selectedName) {
	// 	runner := m.Runner
	// 	if runner == "" && m.isGGUF() {
	// 		runner = manifest.RunnerGGML
	// 	}
	// 	if runner == manifest.RunnerGGML {
	// 		compatmigrate.StartLocalCompatibilityMigration(selectedName)
	// 	}
	// }

	if slices.Contains(m.Config.ModelFamilies, "mllama") && len(m.ProjectorPaths) > 0 {
		return nil, nil, nil, fmt.Errorf("'llama3.2-vision' is no longer compatible with your version of Ollama and has been replaced by a newer version. To re-download, run 'ollama pull llama3.2-vision'")
	}

	if err := m.CheckCapabilities(caps...); err != nil {
		return nil, nil, nil, fmt.Errorf("%s %w", m.Name, err)
	}

	numCtxAuto := usesAutomaticNumCtx(m, requestOpts)
	embeddingBatchDefault := shouldApplyEmbeddingBatchDefault(m, requestOpts)
	numBatchAuto := usesAutomaticNumBatch(m, requestOpts) && !embeddingBatchDefault
	opts, err := s.modelOptionsWithEmbeddingBatchDefault(m, requestOpts, embeddingBatchDefault)
	if err != nil {
		return nil, nil, nil, err
	}

	runnerCh, errCh := s.sched.getRunner(ctx, m, opts, keepAlive, numCtxAuto, numBatchAuto, shift)
	var runner *runnerRef
	select {
	case runner = <-runnerCh:
	case err = <-errCh:
		return nil, nil, nil, err
	}

	return runner.llama, m, &opts, nil
}

func signinURL() (string, error) {
	pubKey, err := auth.GetPublicKey()
	if err != nil {
		return "", err
	}

	encKey := base64.RawURLEncoding.EncodeToString([]byte(pubKey))
	h, _ := os.Hostname()
	return fmt.Sprintf(signinURLStr, url.PathEscape(h), encKey), nil
}

func (s *Server) GenerateHandler(c *gin.Context) {
	checkpointStart := time.Now()
	var req api.GenerateRequest
	body := struct {
		*api.GenerateRequest
		Think json.RawMessage `json:"think"`
	}{GenerateRequest: &req}
	if err := c.ShouldBindJSON(&body); errors.Is(err, io.EOF) {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
		return
	} else if err != nil {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		return
	}

	if len(body.Think) > 0 {
		if err := json.Unmarshal(body.Think, &req.Think); err != nil {
			err = s.thinkingInputError(c.Request.Context(), req.Model, err)
			c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
			return
		}
	}

	if req.TopLogprobs < 0 || req.TopLogprobs > 20 {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "top_logprobs must be between 0 and 20"})
		return
	}

	modelRef, err := parseAndValidateModelRef(req.Model)
	if err != nil {
		writeModelRefParseError(c, err, http.StatusNotFound, fmt.Sprintf("model '%s' not found", req.Model))
		return
	}

	if modelRef.Source == modelSourceCloud {
		// TODO(drifkin): evaluate an `/api/*` passthrough for cloud where the
		// original body (modulo model name normalization) is sent to cloud.
		req.Model = modelRef.Base
		req.Runner = ""
		proxyCloudJSONRequest(c, req, cloudErrRemoteInferenceUnavailable)
		return
	}

	if runner, err := normalizeRunner(req.Runner); err != nil {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		return
	} else {
		req.Runner = runner
	}

	m, err := s.getExistingModelForRunner(modelRef.Name.String(), req.Runner)
	if err != nil {
		switch {
		case errors.Is(err, fs.ErrNotExist):
			c.JSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("model '%s' not found", req.Model)})
		case errors.Is(err, manifest.ErrNoCompatibleManifest):
			c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		case err.Error() == errtypes.InvalidModelNameErrMsg:
			c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		default:
			c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		}
		return
	}

	if req.TopLogprobs < 0 || req.TopLogprobs > 20 {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "top_logprobs must be between 0 and 20"})
		return
	}

	if modelRef.Source == modelSourceLocal && m.Config.RemoteHost != "" && m.Config.RemoteModel != "" {
		c.JSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("model '%s' not found", req.Model)})
		return
	}

	if m.Config.RemoteHost != "" && m.Config.RemoteModel != "" {
		if disabled, _ := internalcloud.Status(); disabled {
			c.JSON(http.StatusForbidden, gin.H{"error": internalcloud.DisabledError(cloudErrRemoteInferenceUnavailable)})
			return
		}

		origModel := req.Model

		remoteURL, err := url.Parse(m.Config.RemoteHost)
		if err != nil {
			c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
			return
		}

		if !slices.Contains(envconfig.Remotes(), remoteURL.Hostname()) {
			slog.Info("remote model", "remotes", envconfig.Remotes(), "remoteURL", m.Config.RemoteHost, "hostname", remoteURL.Hostname())
			c.JSON(http.StatusBadRequest, gin.H{"error": "this server cannot run this remote model"})
			return
		}

		req.Model = m.Config.RemoteModel
		req.Runner = ""

		if req.Template == "" && m.Template.String() != "" {
			req.Template = m.Template.String()
		}

		if req.Options == nil {
			req.Options = map[string]any{}
		}

		for k, v := range m.Options {
			if _, ok := req.Options[k]; !ok {
				req.Options[k] = v
			}
		}

		// update the system prompt from the model if one isn't already specified
		if req.System == "" && m.System != "" {
			req.System = m.System
		}

		if len(m.Messages) > 0 {
			slog.Warn("embedded messages in the model not supported with '/api/generate'; try '/api/chat' instead")
		}

		contentType := "application/x-ndjson"
		if req.Stream != nil && !*req.Stream {
			contentType = "application/json; charset=utf-8"
		}
		c.Header("Content-Type", contentType)

		fn := func(resp api.GenerateResponse) error {
			resp.Model = origModel
			resp.RemoteModel = m.Config.RemoteModel
			resp.RemoteHost = m.Config.RemoteHost

			data, err := json.Marshal(resp)
			if err != nil {
				return err
			}

			if _, err = c.Writer.Write(append(data, '\n')); err != nil {
				return err
			}
			c.Writer.Flush()
			return nil
		}

		client := api.NewClient(remoteURL, http.DefaultClient)
		err = client.Generate(c.Request.Context(), &req, fn)
		if err != nil {
			var authError api.AuthorizationError
			if errors.As(err, &authError) {
				sURL, sErr := signinURL()
				if sErr != nil {
					slog.Error(sErr.Error())
					c.JSON(http.StatusInternalServerError, gin.H{"error": "error getting authorization details"})
					return
				}

				c.JSON(authError.StatusCode, gin.H{"error": "unauthorized", "signin_url": sURL})
				return
			}
			var apiError api.StatusError
			if errors.As(err, &apiError) {
				c.JSON(apiError.StatusCode, apiError)
				return
			}
			c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
			return
		}

		return
	}

	// expire the runner if unload is requested (empty prompt, keep alive is 0)
	if req.Prompt == "" && req.KeepAlive != nil && req.KeepAlive.Duration == 0 {
		s.sched.expireRunner(m)

		c.JSON(http.StatusOK, api.GenerateResponse{
			Model:      req.Model,
			CreatedAt:  time.Now().UTC(),
			Response:   "",
			Done:       true,
			DoneReason: "unload",
		})
		return
	}

	if slices.Contains(m.Capabilities(), model.CapabilityImage) {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "image generation models are not currently supported"})
		return
	}

	if req.Raw && (req.Template != "" || req.System != "" || len(req.Context) > 0) {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "raw mode does not support template, system, or context"})
		return
	}

	thinking := m.genericThinking()
	if thinking == nil {
		if err := api.ValidateLegacyThinking(req.Think); err != nil {
			c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
			return
		}
	}

	caps := []model.Capability{model.CapabilityCompletion}
	if req.Suffix != "" {
		caps = append(caps, model.CapabilityInsert)
	}

	requestedThink := req.Think
	think := renderers.ResolveThinking(requestedThink, thinking)
	req.Think = think
	modelCaps := m.Capabilities()
	if slices.Contains(modelCaps, model.CapabilityThinking) {
		caps = append(caps, model.CapabilityThinking)
		if req.Think == nil && thinking == nil {
			req.Think = &api.ThinkValue{Value: true}
		}
	} else {
		if req.Think != nil && req.Think.Bool() {
			c.JSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("%q does not support thinking", req.Model)})
			return
		}
	}

	parserName := parserNameForModel(m)
	var builtinParser parsers.Parser
	if shouldUseHarmony(m) {
		// harmony's Reasoning field only understands low/medium/high; map "max" to "high"
		if req.Think != nil {
			if s, ok := req.Think.Value.(string); ok && s == "max" {
				req.Think.Value = "high"
			}
		}
	}

	if !req.Raw && parserName != "" {
		builtinParser = parsers.ParserForName(parserName)
		if builtinParser != nil {
			// no tools or last message for generate endpoint
			builtinParser.Init(nil, nil, req.Think)
		}
	}

	r, m, opts, err := s.scheduleRunnerForModel(c.Request.Context(), m, caps, req.Options, req.KeepAlive, req.Shift)
	if errors.Is(err, errCapabilityCompletion) {
		c.JSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("%q does not support generate", req.Model)})
		return
	} else if err != nil {
		handleScheduleError(c, req.Model, err)
		return
	}

	checkpointLoaded := time.Now()

	// load the model
	if req.Prompt == "" {
		c.JSON(http.StatusOK, api.GenerateResponse{
			Model:      req.Model,
			CreatedAt:  time.Now().UTC(),
			Done:       true,
			DoneReason: "load",
		})
		return
	}

	if slices.Contains(m.Config.ModelFamilies, "mllama") && len(req.Images) > 1 {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "this model only supports one image while more than one image requested"})
		return
	}

	media := make([]llm.MediaData, len(req.Images))
	for i := range req.Images {
		media[i] = llm.NewMediaData(i, req.Images[i])
	}

	prompt := req.Prompt
	var leadingBOS string
	if !req.Raw {
		tmpl := m.Template
		if req.Template != "" {
			tmpl, err = template.Parse(req.Template)
			if err != nil {
				c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
				return
			}
		}

		var values template.Values
		if req.Suffix != "" {
			values.Prompt = prompt
			values.Suffix = req.Suffix
		} else {
			var msgs []api.Message
			if req.System != "" {
				msgs = append(msgs, api.Message{Role: "system", Content: req.System})
			} else if m.System != "" {
				msgs = append(msgs, api.Message{Role: "system", Content: m.System})
			}

			if req.Context == nil {
				msgs = append(msgs, m.Messages...)
			}

			userMsg := api.Message{Role: "user", Content: req.Prompt}
			for _, m := range media {
				userMsg.Images = append(userMsg.Images, m.Data)
			}
			values.Messages = append(msgs, userMsg)
		}

		values.Think = req.Think != nil && req.Think.Bool()
		values.ThinkLevel = ""
		if req.Think != nil {
			values.ThinkLevel = req.Think.String()
		}
		values.IsThinkSet = req.Think != nil

		var b bytes.Buffer
		if req.Context != nil {
			slog.Warn("the context field is deprecated and will be removed in a future version of Ollama")
			s, err := r.Detokenize(c.Request.Context(), req.Context)
			if err != nil {
				c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
				return
			}
			b.WriteString(s)
		}

		// check that we're in the `api/chat`-like flow, and if so, generate the
		// prompt the same way
		// TEMP(drifkin): we should really just detect the chat-like flow and call
		// the real chat handler, but doing this as a stopgap to get renderer
		// support for generate
		if values.Messages != nil && values.Suffix == "" && req.Template == "" {
			genTruncate := (req.Truncate == nil || *req.Truncate) && !m.IsMLX()
			if m.HasChatTemplate && chatModeForModel(m) == chatExecutionModeNative {
				nativeReq, err := prepareNativeChatRequest(c.Request.Context(), m, r, opts, llm.ChatRequest{
					Messages:    values.Messages,
					Format:      req.Format,
					Options:     opts,
					Think:       req.Think,
					Shift:       req.Shift == nil || *req.Shift,
					Logprobs:    req.Logprobs,
					TopLogprobs: req.TopLogprobs,
				}, genTruncate)
				if err != nil {
					slog.Error("chat template prompt error", "error", err)
					var serr api.StatusError
					if errors.As(err, &serr) {
						c.JSON(serr.StatusCode, gin.H{"error": serr.ErrorMessage})
					} else {
						c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
					}
					return
				}
				nativeReq.Messages, media, err = imageTaggedMessages(m, nativeReq.Messages, 0, true)
				if err != nil {
					c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
					return
				}
				prompt, err = r.ApplyChatTemplate(c.Request.Context(), nativeReq)
				if err != nil {
					var serr api.StatusError
					if errors.As(err, &serr) {
						c.JSON(serr.StatusCode, gin.H{"error": serr.ErrorMessage})
					} else {
						c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
					}
					return
				}
				// TEMP(drifkin): req.Context will be removed very soon, but we're temporarily supporting it in this flow here
				if req.Context != nil {
					b.WriteString(prompt)
					prompt = b.String()
				}
			} else {
				prompt, media, err = chatPrompt(c.Request.Context(), m, r.Tokenize, optionsForPrompt(opts, r), values.Messages, []api.Tool{}, req.Think, genTruncate)
				if err != nil {
					c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
					return
				}
				// TEMP(drifkin): req.Context will be removed very soon, but we're temporarily supporting it in this flow here
				if req.Context != nil {
					b.WriteString(prompt)
					prompt = b.String()
				}
				leadingBOS = leadingBOSForModel(m)
			}
		} else {
			// Direct template execution flow.
			if err := tmpl.Execute(&b, values); err != nil {
				c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
				return
			}

			prompt = b.String()
		}
	}

	// If debug mode is enabled, return the rendered template instead of calling the model
	if req.DebugRenderOnly {
		c.JSON(http.StatusOK, api.GenerateResponse{
			Model:     req.Model,
			CreatedAt: time.Now().UTC(),
			DebugInfo: &api.DebugInfo{
				RenderedTemplate: prompt,
				ImageCount:       len(media),
			},
		})
		return
	}

	var thinkTagParser *thinkingparser.Parser
	if builtinParser == nil {
		openingTag, closingTag := thinkingparser.InferTags(m.Template.Template)
		if req.Think != nil && req.Think.Bool() && openingTag != "" && closingTag != "" {
			thinkTagParser = &thinkingparser.Parser{
				OpeningTag: openingTag,
				ClosingTag: closingTag,
			}
			if strings.HasSuffix(strings.TrimSpace(prompt), openingTag) {
				thinkTagParser.AddContent(openingTag)
			}
		}
	}

	ch := make(chan any)
	go func() {
		// TODO (jmorganca): avoid building the response twice both here and below
		var sb strings.Builder
		defer close(ch)

		ctx, cancel := context.WithCancel(c.Request.Context())
		defer cancel()
		var parserErr error

		// A raw prompt gives no way to tell where the response starts, so the
		// format applies from its first token.
		var thinkingClose []string
		if !req.Raw {
			thinkingClose = thinkingCloseForCompletion(builtinParser, thinkTagParser)
		}

		if err := r.Completion(ctx, llm.CompletionRequest{
			Prompt:          prompt,
			Media:           media,
			Format:          req.Format,
			Options:         opts,
			Shift:           req.Shift == nil || *req.Shift,
			Truncate:        req.Truncate == nil || *req.Truncate,
			Logprobs:        req.Logprobs,
			TopLogprobs:     req.TopLogprobs,
			PreservedTokens: preservedTokensForCompletion(builtinParser),
			LeadingBOS:      leadingBOS,
			ThinkingClose:   thinkingClose,
		}, func(cr llm.CompletionResponse) {
			res := api.GenerateResponse{
				Model:     req.Model,
				CreatedAt: time.Now().UTC(),
				Response:  cr.Content,
				Done:      cr.Done,
				Metrics: api.Metrics{
					PromptEvalCount:       cr.PromptEvalCount,
					PromptEvalCachedCount: cr.PromptEvalCachedCount,
					PromptEvalDuration:    cr.PromptEvalDuration,
					EvalCount:             cr.EvalCount,
					EvalDuration:          cr.EvalDuration,
				},
				Logprobs: toAPILogprobs(cr.Logprobs),
			}

			if builtinParser != nil {
				content, thinking, toolCalls, err := builtinParser.Add(cr.Content, cr.Done)
				if err != nil {
					parserErr = err
					cancel()
					return
				}
				res.Response = content
				res.Thinking = thinking
				if cr.Done && len(toolCalls) > 0 {
					res.ToolCalls = toolCalls
				}
			} else if thinkTagParser != nil {
				thinking, content := thinkTagParser.AddContent(cr.Content)
				res.Thinking = thinking
				res.Response = content
			}

			if _, err := sb.WriteString(cr.Content); err != nil {
				ch <- gin.H{"error": err.Error()}
			}

			if cr.Done {
				res.DoneReason = cr.DoneReason.String()
				res.TotalDuration = time.Since(checkpointStart)
				res.LoadDuration = checkpointLoaded.Sub(checkpointStart)

				if !req.Raw {
					tokens, err := r.Tokenize(c.Request.Context(), prompt+sb.String())
					if err != nil {
						ch <- gin.H{"error": err.Error()}
						return
					}
					res.Context = tokens
				}
			}

			if builtinParser != nil {
				// Emit chunks that carry logprobs even if the parser is still buffering
				// visible content, otherwise generate logprobs disappear for models with
				// builtin thinking/tool parsers.
				if res.Response != "" || res.Thinking != "" || res.Done || len(res.ToolCalls) > 0 || len(res.Logprobs) > 0 {
					ch <- res
				}

				return
			}

			ch <- res
		}); err != nil {
			if parserErr == nil {
				s.sched.expireRunnersForRuntimeOOM(m, err)
				var serr api.StatusError
				if errors.As(err, &serr) {
					ch <- gin.H{"error": serr.ErrorMessage, "status": serr.StatusCode}
				} else {
					ch <- gin.H{"error": err.Error()}
				}
			}
		}
		if parserErr != nil {
			ch <- gin.H{"error": parserErr.Error()}
		}
	}()

	if req.Stream != nil && !*req.Stream {
		var r api.GenerateResponse
		var allLogprobs []api.Logprob
		var sbThinking strings.Builder
		var sbContent strings.Builder
		for rr := range ch {
			switch t := rr.(type) {
			case api.GenerateResponse:
				sbThinking.WriteString(t.Thinking)
				sbContent.WriteString(t.Response)
				r = t
				// Accumulate logprobs from all chunks for non-streaming response
				if len(t.Logprobs) > 0 {
					allLogprobs = append(allLogprobs, t.Logprobs...)
				}
			case gin.H:
				msg, ok := t["error"].(string)
				if !ok {
					msg = "unexpected error format in response"
				}

				status, ok := t["status"].(int)
				if !ok {
					status = http.StatusInternalServerError
				}

				c.JSON(status, gin.H{"error": msg})
				return
			default:
				c.JSON(http.StatusInternalServerError, gin.H{"error": "unexpected response"})
				return
			}
		}

		r.Thinking = sbThinking.String()
		r.Response = sbContent.String()
		r.Logprobs = allLogprobs

		c.JSON(http.StatusOK, r)
		return
	}

	streamResponse(c, ch)
}

func (s *Server) systemOneLlamaServer(c *gin.Context, m *Model, req decision.Request, body []byte) {
	decisionType := m.metadata.String("decision.type")
	// Old Clef GGUFs have a decision type but use qwen35; llama.cpp now
	// requires the clef architecture and its converted decision head.
	oldClef := decisionType == "clef" && m.metadata.String("general.architecture") == "qwen35"
	if decisionType == "" || oldClef {
		if oldClef || slices.Contains(m.Config.Capabilities, string(model.CapabilityDecision)) {
			c.JSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("'%s' was downloaded in a format that is no longer supported. To re-download, run 'ollama pull %s'", m.ShortName, m.ShortName)})
			return
		}
		c.JSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("model %q is not a llama.cpp decision model", req.Model)})
		return
	}
	if err := decision.ValidateLimits(req); err != nil {
		c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		return
	}
	var raw struct {
		State     json.RawMessage `json:"state"`
		Questions json.RawMessage `json:"questions"`
	}
	if err := json.Unmarshal(body, &raw); err != nil {
		c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		return
	}
	caps := []model.Capability{model.CapabilityDecision}
	if len(req.Images) > 0 {
		caps = append(caps, model.CapabilityVision)
	}
	r, _, _, err := s.scheduleRunnerForModel(c.Request.Context(), m, caps, nil, req.KeepAlive, nil)
	if err != nil {
		handleScheduleError(c, req.Model, err)
		return
	}
	runner, ok := r.(interface {
		SystemOne(ctx context.Context, state, questions json.RawMessage, images []api.ImageData) (answers, usage json.RawMessage, err error)
	})
	if !ok {
		c.JSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("model %q does not support System One scoring", req.Model)})
		return
	}
	answers, usage, err := runner.SystemOne(c.Request.Context(), raw.State, raw.Questions, req.Images)
	if err != nil {
		s.sched.expireRunnersForRuntimeOOM(m, err)
		status := http.StatusInternalServerError
		var statusErr api.StatusError
		if errors.As(err, &statusErr) {
			status = statusErr.StatusCode
		}
		c.JSON(status, gin.H{"error": err.Error()})
		return
	}
	c.JSON(http.StatusOK, struct {
		Model   string          `json:"model"`
		Answers json.RawMessage `json:"answers"`
		Usage   json.RawMessage `json:"usage"`
	}{req.Model, answers, usage})
}

// SystemOneHandler answers typed questions with probabilities.
func (s *Server) SystemOneHandler(c *gin.Context) {
	// TODO(parthsareen): Check token limits before copying state and schema into
	// each question's prompt. Non-Kev models retain the 64 KiB byte cap until then.
	c.Request.Body = http.MaxBytesReader(c.Writer, c.Request.Body, 32<<20)
	body, err := io.ReadAll(c.Request.Body)
	var req decision.Request
	if err == nil {
		err = json.Unmarshal(body, &req)
	}
	if err != nil {
		var sizeErr *http.MaxBytesError
		if errors.As(err, &sizeErr) {
			c.JSON(http.StatusRequestEntityTooLarge, gin.H{"error": "request body must not exceed 32 MiB"})
			return
		}
		c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		return
	}
	textOnly := req
	textOnly.Images = nil
	var text bytes.Buffer
	enc := json.NewEncoder(&text)
	enc.SetEscapeHTML(false)
	if err := enc.Encode(textOnly); err != nil {
		c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		return
	}
	textSize := len(bytes.TrimSpace(text.Bytes()))
	rejectText := func(limit int) {
		c.JSON(http.StatusRequestEntityTooLarge, gin.H{"error": fmt.Sprintf("text and schema must not exceed %d KiB", limit>>10)})
	}
	ref, err := parseAndValidateModelRef(req.Model)
	if err != nil {
		if errors.Is(err, errModelRequired) {
			c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
			return
		}
		writeModelRefParseError(c, err, http.StatusNotFound, fmt.Sprintf("model '%s' not found", req.Model))
		return
	}
	if ref.Source == modelSourceCloud {
		c.JSON(http.StatusBadRequest, gin.H{"error": "System One requires a local decision model"})
		return
	}
	if runner, err := normalizeRunner(req.Runner); err != nil {
		c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		return
	} else {
		req.Runner = runner
	}
	m, err := s.getExistingModelForRunner(ref.Name.String(), req.Runner)
	if err != nil {
		if errors.Is(err, os.ErrNotExist) {
			c.JSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("model '%s' not found", req.Model)})
			return
		}
		handleScheduleError(c, req.Model, err)
		return
	}
	textLimit := 64 << 10
	if m.Config.Renderer == "kev" {
		// Kev shares the state across question rows and admits up to 65,536
		// state tokens, which can exceed the generic decision prompt cap.
		textLimit = 1 << 20
	}
	if textSize > textLimit {
		rejectText(textLimit)
		return
	}
	if m.isGGUF() {
		s.systemOneLlamaServer(c, m, req, body)
		return
	}
	var encoding string
	switch m.Config.Renderer {
	case "tev1", "clef", "strands", "kev", "pplx", "d3":
		encoding = m.Config.Renderer
	}
	compiled, err := decision.Compile(req, encoding)
	if err != nil {
		c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		return
	}
	if encoding == "pplx" {
		if m.System == "" {
			c.JSON(http.StatusBadRequest, gin.H{"error": "Perplexity decision models require their publisher system prompt in the Modelfile"})
			return
		}
		compiled.Request.System = m.System
	}
	caps := []model.Capability{model.CapabilityDecision}
	if len(req.Images) > 0 {
		caps = append(caps, model.CapabilityVision)
	}
	r, _, _, err := s.scheduleRunnerForModel(c.Request.Context(), m, caps, nil, req.KeepAlive, nil)
	if err != nil {
		handleScheduleError(c, req.Model, err)
		return
	}
	scorer, ok := r.(llm.Scorer)
	if !ok {
		c.JSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("model %q does not support System One scoring; use a local decision model with a scoring-capable runner", req.Model)})
		return
	}
	if err := compiled.Render(func(messages []api.Message) (string, error) {
		if m.System != "" {
			messages = append([]api.Message{{Role: "system", Content: m.System}}, messages...)
		}
		return renderPrompt(m, messages, nil, &api.ThinkValue{Value: false})
	}); err != nil {
		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		return
	}
	compiled.Request.MaxTokens = r.ContextLength()
	result, err := scorer.Score(c.Request.Context(), compiled.Request)
	if err != nil {
		s.sched.expireRunnersForRuntimeOOM(m, err)
		status := http.StatusInternalServerError
		var statusErr api.StatusError
		if errors.As(err, &statusErr) {
			status = statusErr.StatusCode
		}
		c.JSON(status, gin.H{"error": err.Error()})
		return
	}
	response, err := compiled.Answer(req.Model, result)
	if err != nil {
		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		return
	}
	c.JSON(http.StatusOK, response)
}

func (s *Server) EmbedHandler(c *gin.Context) {
	checkpointStart := time.Now()
	var req api.EmbedRequest
	err := c.ShouldBindJSON(&req)
	switch {
	case errors.Is(err, io.EOF):
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
		return
	case err != nil:
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		return
	}

	modelRef, err := parseAndValidateModelRef(req.Model)
	if err != nil {
		writeModelRefParseError(c, err, http.StatusNotFound, fmt.Sprintf("model '%s' not found", req.Model))
		return
	}

	if modelRef.Source == modelSourceCloud {
		req.Model = modelRef.Base
		req.Runner = ""
		proxyCloudJSONRequest(c, req, cloudErrRemoteInferenceUnavailable)
		return
	}

	if runner, err := normalizeRunner(req.Runner); err != nil {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		return
	} else {
		req.Runner = runner
	}

	// embedInput is one item from the input list: its text plus whatever
	// media blobs it carries. Items without media go through the plain
	// text embed path; items with media go through the runner's media-aware
	// embed.
	type embedInput struct {
		text  string
		media [][]byte
	}

	var input []embedInput

	// parseMultimodalItem reads one map-shaped input item. Any of "text",
	// "image", "audio" are accepted (short names, base64-encoded blobs for
	// media); "video" exists in the config but the runner has no video tower
	// yet, so it's a hard error rather than silently ignored.
	parseMultimodalItem := func(m map[string]any) (embedInput, error) {
		var item embedInput
		for key, v := range m {
			str, ok := v.(string)
			if !ok {
				return item, fmt.Errorf("input.%s must be a string", key)
			}
			switch key {
			case "text":
				item.text = str
			case "image", "audio":
				raw, err := base64.StdEncoding.DecodeString(str)
				if err != nil {
					return item, fmt.Errorf("input.%s: %v", key, err)
				}
				item.media = append(item.media, raw)
			case "video":
				return item, fmt.Errorf("input.video not supported")
			default:
				return item, fmt.Errorf("unknown input field %q", key)
			}
		}
		if item.text == "" && len(item.media) == 0 {
			return item, errors.New("input item has no text, image, or audio")
		}
		return item, nil
	}

	switch i := req.Input.(type) {
	case string:
		if i != "" {
			input = append(input, embedInput{text: i})
		}
	case map[string]any:
		item, err := parseMultimodalItem(i)
		if err != nil {
			c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
			return
		}
		input = append(input, item)
	case []any:
		for _, v := range i {
			switch v := v.(type) {
			case string:
				input = append(input, embedInput{text: v})
			case map[string]any:
				item, err := parseMultimodalItem(v)
				if err != nil {
					c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
					return
				}
				input = append(input, item)
			default:
				c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "invalid input type"})
				return
			}
		}
	default:
		if req.Input != nil {
			c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "invalid input type"})
			return
		}
	}

	m, err := s.getExistingModelForRunner(modelRef.Name.String(), req.Runner)
	if err != nil {
		if errors.Is(err, os.ErrNotExist) {
			c.JSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("model '%s' not found", req.Model)})
			return
		}
		handleScheduleError(c, req.Model, err)
		return
	}

	r, m, opts, err := s.scheduleRunnerForModel(c.Request.Context(), m, []model.Capability{}, req.Options, req.KeepAlive, nil)
	if err != nil {
		handleScheduleError(c, req.Model, err)
		return
	}

	checkpointLoaded := time.Now()

	if len(input) == 0 {
		c.JSON(http.StatusOK, api.EmbedResponse{Model: req.Model, Embeddings: [][]float32{}})
		return
	}

	ctx := c.Request.Context()

	adjustTokenLimit := func(tokens []int, limit int) int {
		if bos := m.metadata.Int("tokenizer.ggml.bos_token_id"); len(tokens) > 0 && tokens[0] != int(bos) && m.metadata.Bool("add_bos_token", true) {
			limit--
		}
		if eos := m.metadata.Int("tokenizer.ggml.eos_token_id"); len(tokens) > 0 && tokens[len(tokens)-1] != int(eos) && m.metadata.Bool("add_eos_token", true) {
			limit--
		}
		return limit
	}

	inputTokensAndContext := func(text string) ([]int, int, error) {
		tokens, err := r.Tokenize(ctx, text)
		if err != nil {
			return nil, 0, err
		}

		// GGUF metadata carries the GGUF context length; safetensors (MLX)
		// models never populate ggufMetadata, so fall back to the manifest
		// config's context_length written by create/ from the HF config.
		ctxLen := int(m.metadata.Int("context_length"))
		if ctxLen == 0 && m.Config.ModelFormat == "safetensors" {
			ctxLen = m.Config.ContextLen
		}
		if opts.NumCtx > 0 {
			ctxLen = min(opts.NumCtx, ctxLen)
		}

		return tokens, adjustTokenLimit(tokens, ctxLen), nil
	}

	truncateInputToLimit := func(text string, limit int) (string, bool, error) {
		tokens, ctxLen, err := inputTokensAndContext(text)
		if err != nil {
			return "", false, err
		}
		if limit > 0 {
			ctxLen = min(ctxLen, adjustTokenLimit(tokens, limit))
		}

		if ctxLen <= 0 {
			return "", false, fmt.Errorf("input after truncation exceeds maximum context length")
		}
		if len(tokens) <= ctxLen {
			return text, false, nil
		}

		truncated, err := r.Detokenize(ctx, tokens[:ctxLen])
		if err != nil {
			return "", false, err
		}
		return truncated, true, nil
	}

	truncateInput := func(text string) (string, bool, error) {
		return truncateInputToLimit(text, 0)
	}

	embedWithRetry := func(item embedInput) ([]float32, int, error) {
		text := item.text
		if req.Truncate != nil && !*req.Truncate {
			tokens, ctxLen, err := inputTokensAndContext(text)
			if err != nil {
				return nil, 0, err
			}
			if ctxLen <= 0 {
				return nil, 0, fmt.Errorf("input after truncation exceeds maximum context length")
			}
			if len(tokens) > ctxLen {
				return nil, 0, api.StatusError{
					StatusCode:   http.StatusBadRequest,
					ErrorMessage: "the input length exceeds the context length",
				}
			}
		} else {
			var err error
			text, _, err = truncateInput(text)
			if err != nil {
				return nil, 0, err
			}
		}

		emb, tokCount, err := r.Embedding(ctx, text, item.media)
		if err == nil {
			return emb, tokCount, nil
		}

		// Only a runner 413 (input over context length) justifies a truncation
		// retry. Everything else — 400 media validation, marker mismatches,
		// malformed payloads — belongs to the caller unchanged.
		var serr api.StatusError
		if !errors.As(err, &serr) || serr.StatusCode != http.StatusRequestEntityTooLarge {
			return nil, 0, err
		}
		if req.Truncate != nil && !*req.Truncate {
			return nil, 0, err
		}

		truncated, ok, terr := truncateInputToLimit(text, opts.NumBatch)
		if terr != nil {
			return nil, 0, terr
		}
		if !ok {
			// Nothing to truncate. For plain text that means the runner
			// rejected input the server-side limit accepted (e.g. a
			// physical-batch clamp), so keep the truncation-family error the
			// API contract encodes. For media items the overflow came from
			// media expansion past the text the truncator sees, so the
			// runner's 413 is the honest error.
			if len(item.media) == 0 {
				return nil, 0, fmt.Errorf("input exceeds maximum context length and cannot be truncated further")
			}
			return nil, 0, err
		}

		return r.Embedding(ctx, truncated, item.media)
	}

	var g errgroup.Group
	embeddings := make([][]float32, len(input))
	var totalTokens uint64
	for i, item := range input {
		g.Go(func() error {
			embedding, tokenCount, err := embedWithRetry(item)
			if err != nil {
				return err
			}
			// TODO: this first normalization should be done by the model
			embedding, err = normalize(embedding)
			if err != nil {
				return err
			}
			if req.Dimensions > 0 {
				if req.Dimensions > len(embedding) {
					return fmt.Errorf("dimensions %d exceeds embedding length %d", req.Dimensions, len(embedding))
				}
				// If the runner declares a trained matryoshka set, restrict to it.
				if ed, ok := r.(interface{ EmbeddingDimensions() []int }); ok {
					if valid := ed.EmbeddingDimensions(); len(valid) > 0 && !slices.Contains(valid, req.Dimensions) {
						return api.StatusError{
							StatusCode:   http.StatusBadRequest,
							ErrorMessage: fmt.Sprintf("dimensions %d not supported by this model; valid values are %v", req.Dimensions, valid),
						}
					}
				}
				if req.Dimensions < len(embedding) {
					embedding, err = normalize(embedding[:req.Dimensions])
					if err != nil {
						return err
					}
				}
			}
			embeddings[i] = embedding
			atomic.AddUint64(&totalTokens, uint64(tokenCount))
			return nil
		})
	}

	if err := g.Wait(); err != nil {
		s.sched.expireRunnersForRuntimeOOM(m, err)
		var serr api.StatusError
		if errors.As(err, &serr) {
			c.AbortWithStatusJSON(serr.StatusCode, gin.H{
				"error": strings.TrimSpace(serr.ErrorMessage),
			})
			return
		}

		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{
			"error": strings.TrimSpace(err.Error()),
		})
		return
	}

	resp := api.EmbedResponse{
		Model:           req.Model,
		Embeddings:      embeddings,
		TotalDuration:   time.Since(checkpointStart),
		LoadDuration:    checkpointLoaded.Sub(checkpointStart),
		PromptEvalCount: int(totalTokens),
	}
	c.JSON(http.StatusOK, resp)
}

func normalize(vec []float32) ([]float32, error) {
	var sum float32
	for _, v := range vec {
		if math.IsNaN(float64(v)) || math.IsInf(float64(v), 0) {
			return nil, errors.New("embedding contains NaN or Inf values")
		}
		sum += v * v
	}

	norm := float32(1.0 / max(math.Sqrt(float64(sum)), 1e-12))
	for i := range vec {
		vec[i] *= norm
	}
	return vec, nil
}

func (s *Server) EmbeddingsHandler(c *gin.Context) {
	var req api.EmbeddingRequest
	if err := c.ShouldBindJSON(&req); errors.Is(err, io.EOF) {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
		return
	} else if err != nil {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		return
	}

	modelRef, err := parseAndValidateModelRef(req.Model)
	if err != nil {
		writeModelRefParseError(c, err, http.StatusBadRequest, "model is required")
		return
	}

	if modelRef.Source == modelSourceCloud {
		req.Model = modelRef.Base
		req.Runner = ""
		proxyCloudJSONRequest(c, req, cloudErrRemoteInferenceUnavailable)
		return
	}

	if runner, err := normalizeRunner(req.Runner); err != nil {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		return
	} else {
		req.Runner = runner
	}

	name := modelRef.Name

	r, m, _, err := s.scheduleRunner(c.Request.Context(), name.String(), req.Runner, []model.Capability{}, req.Options, req.KeepAlive, nil)
	if err != nil {
		handleScheduleError(c, req.Model, err)
		return
	}

	// an empty request loads the model
	if req.Prompt == "" {
		c.JSON(http.StatusOK, api.EmbeddingResponse{Embedding: []float64{}})
		return
	}

	embedding, _, err := r.Embedding(c.Request.Context(), req.Prompt, nil)
	if err != nil {
		s.sched.expireRunnersForRuntimeOOM(m, err)
		c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{"error": strings.TrimSpace(err.Error())})
		return
	}

	var e []float64
	for _, v := range embedding {
		e = append(e, float64(v))
	}

	resp := api.EmbeddingResponse{
		Embedding: e,
	}
	c.JSON(http.StatusOK, resp)
}

func (s *Server) PullHandler(c *gin.Context) {
	var req api.PullRequest
	err := c.ShouldBindJSON(&req)
	switch {
	case errors.Is(err, io.EOF):
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
		return
	case err != nil:
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		return
	}

	// TEMP(drifkin): we're temporarily allowing to continue pulling cloud model
	// stub-files until we integrate cloud models into `/api/tags` (in which case
	// this roundabout way of "adding" cloud models won't be needed anymore). So
	// right here normalize any `:cloud` models into the legacy-style suffixes
	// `:<tag>-cloud` and `:cloud`
	modelRef, err := parseNormalizePullModelRef(cmp.Or(req.Model, req.Name))
	if err != nil {
		writeModelRefParseError(c, err, http.StatusBadRequest, errtypes.InvalidModelNameErrMsg)
		return
	}

	name := modelRef.Name

	name, err = getExistingName(name)
	if err != nil {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		return
	}

	runner, err := normalizeRunner(req.Runner)
	if err != nil {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		return
	}

	ch := make(chan any)
	go func() {
		defer close(ch)
		fn := func(r api.ProgressResponse) {
			ch <- r
		}

		regOpts := &registryOptions{
			Insecure: req.Insecure,
		}

		ctx, cancel := context.WithCancel(c.Request.Context())
		defer cancel()

		if err := PullModel(ctx, name.DisplayShortest(), runner, regOpts, fn); err != nil {
			ch <- gin.H{"error": err.Error()}
			return
		}

		if s.modelCaches != nil && s.modelCaches.show != nil {
			s.modelCaches.show.invalidateLocal(name)
		}
	}()

	if req.Stream != nil && !*req.Stream {
		waitForStream(c, ch)
		return
	}

	streamResponse(c, ch)
}

func (s *Server) PushHandler(c *gin.Context) {
	var req api.PushRequest
	err := c.ShouldBindJSON(&req)
	switch {
	case errors.Is(err, io.EOF):
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
		return
	case err != nil:
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		return
	}

	var mname string
	if req.Model != "" {
		mname = req.Model
	} else if req.Name != "" {
		mname = req.Name
	} else {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "model is required"})
		return
	}

	ch := make(chan any)
	go func() {
		defer close(ch)
		fn := func(r api.ProgressResponse) {
			ch <- r
		}

		regOpts := &registryOptions{
			Insecure: req.Insecure,
		}

		ctx, cancel := context.WithCancel(c.Request.Context())
		defer cancel()

		name, err := getExistingName(model.ParseName(mname))
		if err != nil {
			ch <- gin.H{"error": err.Error()}
			return
		}

		if err := PushModel(ctx, name.DisplayShortest(), regOpts, fn); err != nil {
			ch <- gin.H{"error": err.Error()}
		}
	}()

	if req.Stream != nil && !*req.Stream {
		waitForStream(c, ch)
		return
	}

	streamResponse(c, ch)
}

// getExistingModelForRunner first tries the model retained by a loaded runner.
// That model was built from checked manifest bytes; the link check only confirms
// the name still selects its parent digest. Legacy manifests and cache misses
// use a full manifest read and selection below.
func (s *Server) getExistingModelForRunner(name, runner string) (*Model, error) {
	n := model.ParseName(name)
	if s.sched != nil && n.IsValid() && !manifest.IsDigestReferenceName(n) {
		for _, loaded := range s.sched.loadedModelsForName(n, runner) {
			if manifest.NamedManifestPointsToDigest(model.ParseName(loaded.Name), loaded.Digest) {
				return loaded, nil
			}
		}
	}

	m, err := loadExistingModelForRunner(name, runner)
	if err == nil {
		m.runnerSelection = runner
	}
	return m, err
}

// loadExistingModelForRunner uses the manifest parsed while resolving the
// stored spelling of a model.
func loadExistingModelForRunner(name, runner string) (*Model, error) {
	n := model.ParseName(name)
	if !n.IsValid() || manifest.IsDigestReferenceName(n) {
		return GetModelForRunner(name, runner)
	}

	selected, err := manifest.FindNamedManifestForRunner(n, runner)
	if err != nil {
		return nil, err
	}
	return modelFromManifest(selected.Name(), selected)
}

// getExistingName searches the models directory for the longest prefix match of
// the input name and returns the input name with all existing parts replaced
// with each part found. If no parts are found, the input name is returned as
// is.
func getExistingName(n model.Name) (model.Name, error) {
	var zero model.Name
	if manifest.IsDigestReferenceName(n) {
		return n, nil
	}

	// The usual request names an existing model. Avoid decoding every installed
	// manifest just to resolve its spelling; partial matches below are for new names.
	if name, err := manifest.FindName(n); err == nil {
		return name, nil
	} else if !errors.Is(err, os.ErrNotExist) {
		return zero, err
	}

	existing, err := manifest.Manifests(true)
	if err != nil {
		return zero, err
	}
	// Find the single manifest with the longest consecutive
	// case-insensitive prefix match (host -> namespace -> model) and copy
	// only the matching prefix parts from that manifest. The tag is left
	// as-is so that an unrelated manifest with a matching tag cannot
	// influence the casing of a different model's tag (e.g. pulling
	// "myorg/mymodel:q8" when "MyOrg/MyModel:q4" and
	// "OtherOrg/OtherModel:Q8" exist must not produce "MyOrg/MyModel:Q8").
	var best model.Name
	bestLen := 0
	for e := range existing {
		length := 0
		if strings.EqualFold(e.Host, n.Host) {
			length = 1
			if strings.EqualFold(e.Namespace, n.Namespace) {
				length = 2
				if strings.EqualFold(e.Model, n.Model) {
					length = 3
				}
			}
		}
		if length > bestLen {
			bestLen = length
			best = e
		}
	}

	switch bestLen {
	case 3:
		n.Host = best.Host
		n.Namespace = best.Namespace
		n.Model = best.Model
	case 2:
		n.Host = best.Host
		n.Namespace = best.Namespace
	case 1:
		n.Host = best.Host
	}

	return n, nil
}

func (s *Server) DeleteHandler(c *gin.Context) {
	var r api.DeleteRequest
	if err := c.ShouldBindJSON(&r); errors.Is(err, io.EOF) {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
		return
	} else if err != nil {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		return
	}

	modelRef, err := parseDeleteModelRef(cmp.Or(r.Model, r.Name))
	if err != nil {
		switch {
		case errors.Is(err, errConflictingModelSource):
			c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		case errors.Is(err, model.ErrUnqualifiedName):
			c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("name %q is invalid", cmp.Or(r.Model, r.Name))})
		default:
			c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		}
		return
	}

	n, err := getExistingName(modelRef.Name)
	if err != nil {
		c.JSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("model '%s' not found", cmp.Or(r.Model, r.Name))})
		return
	}

	removed, err := manifest.RemoveNamed(n)
	if err != nil {
		switch {
		case os.IsNotExist(err):
			c.JSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("model '%s' not found", cmp.Or(r.Model, r.Name))})
		default:
			c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		}
		return
	}
	removeGGUFMetadata(removed...)
}

func (s *Server) ShowHandler(c *gin.Context) {
	var req api.ShowRequest
	err := c.ShouldBindJSON(&req)
	switch {
	case errors.Is(err, io.EOF):
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
		return
	case err != nil:
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		return
	}

	if req.Model != "" {
		// noop
	} else if req.Name != "" {
		req.Model = req.Name
	} else {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "model is required"})
		return
	}
	requestedModel := req.Model

	modelRef, err := parseAndValidateModelRef(req.Model)
	if err != nil {
		writeModelRefParseError(c, err, http.StatusBadRequest, err.Error())
		return
	}

	if modelRef.Source == modelSourceCloud {
		req.Model = modelRef.Base
		req.Runner = ""
		req.AllManifests = false
		if modelShowCacheable(req) && s.modelCaches != nil && s.modelCaches.show != nil {
			if disabled, _ := internalcloud.Status(); disabled {
				c.JSON(http.StatusForbidden, gin.H{"error": internalcloud.DisabledError(cloudErrRemoteModelDetailsUnavailable)})
				return
			}

			ctx := context.Background()
			if c.Request != nil {
				ctx = c.Request.Context()
			}
			if resp, ok := s.modelCaches.show.GetCloudSWR(ctx, req); ok {
				c.JSON(http.StatusOK, resp)
				return
			}
		}
		proxyCloudJSONRequest(c, req, cloudErrRemoteModelDetailsUnavailable)
		return
	}

	if req.Runner, err = normalizeRunner(req.Runner); err != nil {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		return
	}
	if req.AllManifests && req.Runner != "" {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "runner cannot be used with all_manifests"})
		return
	}

	name := modelRef.Name
	name, err = getExistingName(name)
	if err != nil {
		c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		return
	}
	req.Model = name.DisplayShortest()

	var resp *api.ShowResponse
	if req.AllManifests {
		resp, err := GetAllManifestsInfo(req)
		if err != nil {
			writeShowError(c, req.Model, err)
			return
		}

		c.JSON(http.StatusOK, resp)
		return
	}

	if modelShowCacheable(req) && s.modelCaches != nil && s.modelCaches.show != nil {
		resp, err = s.modelCaches.show.GetLocal(req)
	} else {
		resp, err = GetModelInfo(req)
	}
	if err != nil {
		writeShowError(c, req.Model, err)
		return
	}

	if modelRef.Source == modelSourceLocal && resp.RemoteHost != "" {
		c.JSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("model '%s' not found", modelRef.Original)})
		return
	}

	userAgent := c.Request.UserAgent()
	if strings.HasPrefix(userAgent, copilotChatUserAgentPrefix) {
		if resp.ModelInfo == nil {
			resp.ModelInfo = map[string]any{}
		}
		// Copilot Chat prefers `general.basename`, but this is usually not what
		// users are familiar with, so echo back the requested model name.
		resp.ModelInfo["general.basename"] = requestedModel
	}

	c.JSON(http.StatusOK, resp)
}

func GetModelInfo(req api.ShowRequest) (*api.ShowResponse, error) {
	runner, err := normalizeRunner(req.Runner)
	if err != nil {
		return nil, api.StatusError{
			StatusCode:   http.StatusBadRequest,
			ErrorMessage: err.Error(),
		}
	}
	req.Runner = runner

	name := model.ParseName(req.Model)
	if !name.IsValid() {
		return nil, model.Unqualified(name)
	}
	name, err = getExistingName(name)
	if err != nil {
		return nil, err
	}

	m, err := GetModelForRunner(name.String(), req.Runner)
	if err != nil {
		return nil, err
	}

	if m.Config.RemoteHost != "" {
		if disabled, _ := internalcloud.Status(); disabled {
			return nil, api.StatusError{
				StatusCode:   http.StatusForbidden,
				ErrorMessage: internalcloud.DisabledError(cloudErrRemoteModelDetailsUnavailable),
			}
		}
	}

	modelDetails := api.ModelDetails{
		ParentModel:       m.ParentModel,
		Format:            m.Config.ModelFormat,
		Family:            m.Config.ModelFamily,
		Families:          m.Config.ModelFamilies,
		ParameterSize:     m.Config.ModelType,
		QuantizationLevel: m.Config.FileType,
		Runner:            m.Runner,
	}

	// For safetensors completion, decision and speech models, populate details from config.json.
	isSafetensorsLLM := m.IsMLX() && (slices.Contains(m.Config.Capabilities, "completion") || slices.Contains(m.Config.Capabilities, "decision") || slices.Contains(m.Config.Capabilities, "speech"))
	if isSafetensorsLLM {
		if info, err := getSafetensorsLLMInfoForRunner(name, req.Runner); err == nil {
			if arch, ok := info["general.architecture"].(string); ok && arch != "" {
				modelDetails.Family = arch
			}
			if paramCount, ok := info["general.parameter_count"].(int64); ok && paramCount > 0 {
				modelDetails.ParameterSize = format.HumanNumber(uint64(paramCount))
			}
		}
		// Older manifests may not have file_type populated for safetensors models.
		if modelDetails.QuantizationLevel == "" {
			if dtype, err := getSafetensorsDtypeForRunner(name, req.Runner); err == nil && dtype != "" {
				modelDetails.QuantizationLevel = dtype
			}
		}
	}

	if req.System != "" {
		m.System = req.System
	}

	msgs := make([]api.Message, len(m.Messages))
	for i, msg := range m.Messages {
		msgs[i] = api.Message{Role: msg.Role, Content: msg.Content}
	}

	mf, err := manifest.ParseNamedManifestForRunner(name, req.Runner)
	if err != nil {
		return nil, err
	}

	resp := &api.ShowResponse{
		License:      strings.Join(m.License, "\n"),
		System:       m.System,
		Template:     m.Template.String(),
		Details:      modelDetails,
		Messages:     msgs,
		Capabilities: m.publicCapabilities(),
		ModifiedAt:   mf.FileInfo().ModTime(),
		Requires:     m.Config.Requires,
		// Several integrations crash on a nil/omitempty+empty ModelInfo, so by
		// default we return an empty map.
		ModelInfo: make(map[string]any),
	}
	resp.Renderer = resolveRendererName(m)
	resp.Parser = m.Config.Parser
	resp.TemplateSource = m.templateSource()
	if !slices.Contains(resp.Capabilities, model.CapabilityDecision) {
		resp.Thinking = m.Thinking()
	}

	if summaries, err := manifestSummariesForShow(name, mf.SelectedDigest()); err != nil {
		return nil, err
	} else {
		resp.Manifests = summaries
	}

	if m.Config.RemoteHost != "" {
		resp.RemoteHost = m.Config.RemoteHost
		resp.RemoteModel = m.Config.RemoteModel

		if m.Config.ModelFamily != "" {
			resp.ModelInfo = make(map[string]any)
			resp.ModelInfo["general.architecture"] = m.Config.ModelFamily

			if m.Config.BaseName != "" {
				resp.ModelInfo["general.basename"] = m.Config.BaseName
			}

			if m.Config.ContextLen > 0 {
				resp.ModelInfo[fmt.Sprintf("%s.context_length", m.Config.ModelFamily)] = m.Config.ContextLen
			}

			if m.Config.EmbedLen > 0 {
				resp.ModelInfo[fmt.Sprintf("%s.embedding_length", m.Config.ModelFamily)] = m.Config.EmbedLen
			}
		}
	}

	var params []string
	cs := 30
	for k, v := range m.Options {
		switch val := v.(type) {
		case []any:
			for _, nv := range val {
				params = append(params, fmt.Sprintf("%-*s %#v", cs, k, nv))
			}
		default:
			params = append(params, fmt.Sprintf("%-*s %#v", cs, k, v))
		}
	}
	resp.Parameters = strings.Join(params, "\n")

	if len(req.Options) > 0 {
		if m.Options == nil {
			m.Options = make(map[string]any)
		}
		for k, v := range req.Options {
			m.Options[k] = v
		}
	}

	var sb strings.Builder
	fmt.Fprintln(&sb, "# Modelfile generated by \"ollama show\"")
	modelfile := m.String()
	if m.IsMLX() {
		fmt.Fprintf(&sb, "FROM %s\n", m.ShortName)
		if _, rest, ok := strings.Cut(modelfile, "\n"); ok {
			fmt.Fprint(&sb, rest)
		}
	} else {
		fmt.Fprintln(&sb, "# To build a new Modelfile based on this, replace FROM with:")
		fmt.Fprintf(&sb, "# FROM %s\n\n", m.ShortName)
		fmt.Fprint(&sb, modelfile)
	}
	resp.Modelfile = sb.String()

	// skip loading tensor information if this is a remote model
	if m.Config.RemoteHost != "" && m.Config.RemoteModel != "" {
		return resp, nil
	}

	// For safetensors completion and decision models, populate ModelInfo from config.json.
	if isSafetensorsLLM {
		if info, err := getSafetensorsLLMInfoForRunner(name, req.Runner); err == nil {
			resp.ModelInfo = info
		}
		// Populate tensor info if verbose
		if req.Verbose {
			if tensors, err := getSafetensorsTensorInfoForRunner(name, req.Runner); err == nil {
				resp.Tensors = tensors
			}
		}
		return resp, nil
	}

	// Image generation models are stored as safetensors tensor layers with no
	// GGUF model layer to introspect; return what we have (incl. the "image"
	// capability) rather than failing to load a non-existent model file.
	if slices.Contains(m.Capabilities(), model.CapabilityImage) {
		// Populate tensor info if verbose
		if req.Verbose {
			if tensors, err := getSafetensorsTensorInfoForRunner(name, req.Runner); err == nil {
				resp.Tensors = tensors
			}
		}
		return resp, nil
	}

	// Safetensors embedding models (e.g. Gemma4Embedding) likewise have no
	// GGUF model layer to introspect; withoutCompletion=false and the image
	// fallback above don't cover them, so mirror that branch rather than
	// falling through to a GGUF read that returns os.ErrNotExist on an empty
	// path list.
	if m.Config.ModelFormat == "safetensors" && slices.Contains(m.Capabilities(), model.CapabilityEmbedding) {
		if info, err := getSafetensorsLLMInfoForRunner(name, req.Runner); err == nil {
			resp.ModelInfo = info
		}
		if req.Verbose {
			if tensors, err := getSafetensorsTensorInfoForRunner(name, req.Runner); err == nil {
				resp.Tensors = tensors
			}
		}
		return resp, nil
	}

	kvData, tensors, err := getModelData(m.modelPaths(), req.Verbose)
	if err != nil {
		return nil, err
	}

	resp.Template = selectedModelTemplate(m, kvData)
	if isUnknownQuantization(resp.Details.QuantizationLevel) {
		if value := kvData.FileType().String(); !isUnknownQuantization(value) {
			resp.Details.QuantizationLevel = value
		}
	}

	modelInfo := jsonSafeValues(kvData)
	delete(modelInfo, "general.name")
	delete(modelInfo, "tokenizer.chat_template")
	resp.ModelInfo = modelInfo

	if req.Verbose {
		tensorItems := tensors.Items()
		resp.Tensors = make([]api.Tensor, 0, len(tensorItems))
		for _, tensor := range tensorItems {
			tensorType := tensor.Type.String()
			if tensorType != "unknown" {
				tensorType = strings.ToUpper(tensorType)
			}
			resp.Tensors = append(resp.Tensors, api.Tensor{Name: tensor.Name, Type: tensorType, Shape: tensor.Shape})
		}
	}

	if len(m.ProjectorPaths) > 0 {
		projectorData, _, err := getModelData(m.ProjectorPaths[:1], req.Verbose)
		if err != nil {
			return nil, err
		}
		projectorInfo := jsonSafeValues(projectorData)
		resp.ProjectorInfo = projectorInfo
	}

	return resp, nil
}

func getModelData(paths []string, verbose bool) (*gguf.Metadata, gguf.Tensors, error) {
	if len(paths) == 0 {
		return nil, gguf.Tensors{}, os.ErrNotExist
	}

	maxArraySize := 0
	if verbose {
		maxArraySize = -1
	}
	data, err := llm.LoadModel(paths[0], maxArraySize, paths[1:]...)
	if err != nil {
		return nil, gguf.Tensors{}, err
	}

	return data.KV(), data.Tensors(), nil
}

func selectedModelTemplate(m *Model, kv *gguf.Metadata) string {
	if m.HasChatTemplate && chatModeForModel(m) == chatExecutionModeNative {
		if chatTemplate := kv.ChatTemplate(); chatTemplate != "" {
			return chatTemplate
		}
	}
	return m.Template.String()
}

func (s *Server) ListHandler(c *gin.Context) {
	models, err := listModels(c.Request.Context())
	if err != nil {
		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		return
	}

	c.JSON(http.StatusOK, api.ListResponse{Models: models})
}

func (s *Server) CopyHandler(c *gin.Context) {
	var r api.CopyRequest
	if err := c.ShouldBindJSON(&r); errors.Is(err, io.EOF) {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
		return
	} else if err != nil {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		return
	}

	srcRef, err := parseAndValidateModelRef(r.Source)
	if err != nil {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("source %q is invalid", r.Source)})
		return
	}
	if srcRef.Source == modelSourceCloud {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("source %q is invalid", r.Source)})
		return
	}
	src := srcRef.Name
	src, err = getExistingName(src)
	if err != nil {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		return
	}

	dst := model.ParseName(r.Destination)
	if !dst.IsValid() {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("destination %q is invalid", r.Destination)})
		return
	}
	dst, err = getExistingName(dst)
	if err != nil {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		return
	}

	warning, err := CopyModel(src, dst)
	switch {
	case errors.Is(err, os.ErrNotExist):
		c.JSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("model %q not found", r.Source)})
	case errors.Is(err, manifest.ErrNoCompatibleManifest):
		c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
	case err != nil:
		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
	default:
		c.JSON(http.StatusOK, api.ProgressResponse{Status: cmp.Or(warning, "success")})
	}
}

func (s *Server) HeadBlobHandler(c *gin.Context) {
	path, err := manifest.BlobsPath(c.Param("digest"))
	if err != nil {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		return
	}

	if _, err := os.Stat(path); errors.Is(err, os.ErrNotExist) {
		c.AbortWithStatusJSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("blob %q not found", c.Param("digest"))})
		return
	} else if err != nil {
		c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		return
	}

	c.Status(http.StatusOK)
}

func (s *Server) CreateBlobHandler(c *gin.Context) {
	path, err := manifest.BlobsPath(c.Param("digest"))
	if err != nil {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		return
	}

	_, err = os.Stat(path)
	switch {
	case errors.Is(err, os.ErrNotExist):
		// noop
	case err != nil:
		c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		return
	default:
		c.Status(http.StatusOK)
		return
	}

	layer, err := manifest.NewLayer(c.Request.Body, "")
	if err != nil {
		c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		return
	}

	if layer.Digest != c.Param("digest") {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("digest mismatch, expected %q, got %q", c.Param("digest"), layer.Digest)})
		return
	}

	c.Status(http.StatusCreated)
}

func isLocalIP(ip netip.Addr) bool {
	if interfaces, err := net.Interfaces(); err == nil {
		for _, iface := range interfaces {
			addrs, err := iface.Addrs()
			if err != nil {
				continue
			}

			for _, a := range addrs {
				if parsed, _, err := net.ParseCIDR(a.String()); err == nil {
					if parsed.String() == ip.String() {
						return true
					}
				}
			}
		}
	}

	return false
}

func allowedHost(host string) bool {
	host = strings.ToLower(host)

	if host == "" || host == "localhost" {
		return true
	}

	if hostname, err := os.Hostname(); err == nil && host == strings.ToLower(hostname) {
		return true
	}

	tlds := []string{
		"localhost",
		"local",
		"internal",
	}

	// check if the host is a local TLD
	for _, tld := range tlds {
		if strings.HasSuffix(host, "."+tld) {
			return true
		}
	}

	return false
}

func allowedHostsMiddleware(addr net.Addr) gin.HandlerFunc {
	return func(c *gin.Context) {
		if addr == nil {
			c.Next()
			return
		}

		if addr, err := netip.ParseAddrPort(addr.String()); err == nil && !addr.Addr().IsLoopback() {
			c.Next()
			return
		}

		host, _, err := net.SplitHostPort(c.Request.Host)
		if err != nil {
			host = c.Request.Host
		}

		if addr, err := netip.ParseAddr(host); err == nil {
			if addr.IsLoopback() || addr.IsPrivate() || addr.IsUnspecified() || isLocalIP(addr) {
				c.Next()
				return
			}
		}

		if allowedHost(host) {
			if c.Request.Method == http.MethodOptions {
				c.AbortWithStatus(http.StatusNoContent)
				return
			}

			c.Next()
			return
		}

		c.AbortWithStatus(http.StatusForbidden)
	}
}

func (s *Server) GenerateRoutes() (http.Handler, error) {
	codexDesktopProxy, err := newCodexDesktopProxy()
	if err != nil {
		return nil, err
	}

	corsConfig := cors.DefaultConfig()
	corsConfig.AllowWildcard = true
	corsConfig.AllowBrowserExtensions = true
	corsConfig.AllowHeaders = []string{
		"Authorization",
		"Content-Type",
		"User-Agent",
		"Accept",
		"X-Requested-With",

		// OpenAI compatibility headers
		"OpenAI-Beta",
		"x-stainless-arch",
		"x-stainless-async",
		"x-stainless-custom-poll-interval",
		"x-stainless-helper-method",
		"x-stainless-lang",
		"x-stainless-os",
		"x-stainless-package-version",
		"x-stainless-poll-helper",
		"x-stainless-retry-count",
		"x-stainless-runtime",
		"x-stainless-runtime-version",
		"x-stainless-timeout",
	}
	corsConfig.AllowOrigins = envconfig.AllowedOrigins()

	r := gin.Default()
	r.HandleMethodNotAllowed = true
	r.Use(
		cors.New(corsConfig),
		allowedHostsMiddleware(s.addr),
	)

	// General
	r.HEAD("/", func(c *gin.Context) { c.String(http.StatusOK, "Ollama is running") })
	r.GET("/", func(c *gin.Context) { c.String(http.StatusOK, "Ollama is running") })
	r.HEAD("/api/version", func(c *gin.Context) { c.JSON(http.StatusOK, gin.H{"version": version.Version}) })
	r.GET("/api/version", func(c *gin.Context) { c.JSON(http.StatusOK, gin.H{"version": version.Version}) })
	r.GET("/api/status", s.StatusHandler)
	// Codex uses this existing Ollama listener for both native and Ollama
	// models. The proxy selects the upstream per request.
	r.Any(proxy.CodexDesktopPathPrefix+"/*path", gin.WrapH(codexDesktopProxy))

	// Local model cache management (new implementation is at end of function)
	r.POST("/api/pull", s.PullHandler)
	r.POST("/api/push", s.PushHandler)
	r.HEAD("/api/tags", s.ListHandler)
	r.GET("/api/tags", s.ListHandler)
	r.POST("/api/show", s.ShowHandler)
	r.DELETE("/api/delete", s.DeleteHandler)

	r.POST("/api/me", s.WhoamiHandler)
	r.GET("/api/balance", s.BalanceHandler)
	r.GET("/api/usage", s.UsageHandler)

	r.POST("/api/signout", s.SignoutHandler)
	// deprecated
	r.DELETE("/api/user/keys/:encodedKey", s.SignoutHandler)

	// Create
	r.POST("/api/create", s.CreateHandler)
	r.POST("/api/blobs/:digest", s.CreateBlobHandler)
	r.HEAD("/api/blobs/:digest", s.HeadBlobHandler)
	r.POST("/api/copy", s.CopyHandler)
	r.POST("/api/experimental/web_search", s.WebSearchExperimentalHandler)
	r.POST("/api/experimental/web_fetch", s.WebFetchExperimentalHandler)
	r.GET("/api/experimental/model-recommendations", s.ModelRecommendationsExperimentalHandler)

	// Inference
	r.GET("/api/ps", s.PsHandler)
	r.POST("/api/generate", s.withInferenceRequestLogging("/api/generate", s.GenerateHandler)...)
	r.POST("/api/chat", s.withInferenceRequestLogging("/api/chat", s.ChatHandler)...)
	r.POST("/api/embed", s.EmbedHandler)
	r.POST("/api/embeddings", s.EmbeddingsHandler)
	r.POST("/v1/systemone", cloudPassthroughMiddleware(cloudErrRemoteInferenceUnavailable), s.SystemOneHandler)

	// Inference (OpenAI compatibility)
	// TODO(cloud-stage-a): apply Modelfile overlay deltas for local models with cloud
	// parents on v1 request families while preserving this explicit :cloud passthrough.
	r.POST("/v1/chat/completions", s.withInferenceRequestLogging("/v1/chat/completions", cloudPassthroughMiddleware(cloudErrRemoteInferenceUnavailable), middleware.ChatMiddleware(lookupThinking), s.ChatHandler)...)
	r.POST("/v1/completions", s.withInferenceRequestLogging("/v1/completions", cloudPassthroughMiddleware(cloudErrRemoteInferenceUnavailable), middleware.CompletionsMiddleware(), s.GenerateHandler)...)
	r.POST("/v1/embeddings", cloudPassthroughMiddleware(cloudErrRemoteInferenceUnavailable), middleware.EmbeddingsMiddleware(), s.EmbedHandler)
	r.GET("/v1/models", middleware.ListMiddleware(), s.ListHandler)
	r.GET("/v1/models/:model", cloudModelPathPassthroughMiddleware(cloudErrRemoteModelDetailsUnavailable), middleware.RetrieveMiddleware(), s.ShowHandler)
	r.POST("/v1/responses", s.withInferenceRequestLogging("/v1/responses", s.responsesCompactionMiddleware(), cloudPassthroughMiddleware(cloudErrRemoteInferenceUnavailable), middleware.ResponsesMiddleware(lookupThinking), s.ChatHandler)...)
	r.POST("/v1/responses/compact", s.ResponsesCompactHandler)
	// OpenAI-compatible audio endpoints
	r.POST("/v1/audio/transcriptions", middleware.TranscriptionMiddleware(), s.ChatHandler)
	r.POST("/v1/audio/speech", s.SpeechHandler)

	// Inference (Anthropic compatibility)
	r.POST("/v1/messages", s.withInferenceRequestLogging("/v1/messages", cloudPassthroughMiddleware(cloudErrRemoteInferenceUnavailable), middleware.AnthropicMessagesMiddleware(lookupThinking), s.ChatHandler)...)

	return r, nil
}

func (s *Server) ModelRecommendationsExperimentalHandler(c *gin.Context) {
	recs := defaultModelRecommendations
	source := "default"
	if s.modelCaches != nil && s.modelCaches.recommendations != nil {
		ctx := context.Background()
		if c.Request != nil {
			ctx = c.Request.Context()
		}
		recs = s.modelCaches.recommendations.GetSWR(ctx)
		source = "cache"
	}

	slog.Debug("serving model recommendations", "recommendation_source", source, "count", len(recs))
	c.JSON(http.StatusOK, api.ModelRecommendationsResponse{
		Recommendations: recs,
	})
}

func Serve(ln net.Listener) error {
	slog.SetDefault(logutil.NewLogger(os.Stderr, envconfig.LogLevel()))
	slog.Info("server config", "env", envconfig.Values())
	cloudDisabled, _ := internalcloud.Status()
	slog.Info(fmt.Sprintf("Ollama cloud disabled: %t", cloudDisabled))

	blobsDir, err := manifest.BlobsPath("")
	if err != nil {
		return err
	}
	if err := llm.PruneSplitModelDirs(blobsDir, time.Now().Add(-layerPruneGracePeriod)); err != nil {
		return err
	}
	if err := fixBlobs(blobsDir); err != nil {
		return err
	}

	if !envconfig.NoPrune() {
		if _, err := manifest.Manifests(false); err != nil {
			slog.Warn("corrupt manifests detected, skipping prune operation.  Re-pull or delete to clear", "error", err)
		} else {
			// clean up unused layers and manifests
			if err := PruneLayers(); err != nil {
				return err
			}

			for _, rootFn := range []func() (string, error){manifest.Path, manifest.V2Path} {
				manifestsPath, err := rootFn()
				if err != nil {
					return err
				}

				if err := manifest.PruneDirectory(manifestsPath); err != nil && !os.IsNotExist(err) {
					return err
				}
			}
		}
	}

	s := &Server{
		addr:        ln.Addr(),
		modelCaches: newModelCaches(),
	}
	if err := s.initRequestLogging(); err != nil {
		return err
	}

	if useClient2 {
		slog.Warn("OLLAMA_EXPERIMENT=client2 is no longer available. Please remove this environment.")
	}

	h, err := s.GenerateRoutes()
	if err != nil {
		return err
	}

	http.Handle("/", h)

	ctx, done := context.WithCancel(context.Background())
	schedCtx, schedDone := context.WithCancel(ctx)
	sched := InitScheduler(schedCtx)
	s.sched = sched
	s.modelCaches.Start(ctx)

	slog.Info(fmt.Sprintf("Listening on %s (version %s)", ln.Addr(), version.Version))
	srvr := &http.Server{
		// Use http.DefaultServeMux so we get net/http/pprof for
		// free.
		//
		// TODO(bmizerany): Decide if we want to make this
		// configurable so it is not exposed by default, or allow
		// users to bind it to a different port. This was a quick
		// and easy way to get pprof, but it may not be the best
		// way.
		Handler: nil,
	}

	// listen for a ctrl+c and stop any loaded llm
	signals := make(chan os.Signal, 1)
	signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM)
	go func() {
		<-signals
		srvr.Close()
		schedDone()
		sched.unloadAllRunners()
		done()
	}()

	s.sched.Run(schedCtx)

	// register the experimental webp decoder
	// so webp images can be used in multimodal inputs
	image.RegisterFormat("webp", "RIFF????WEBP", webp.Decode, webp.DecodeConfig)

	// At startup we retrieve GPU information so we can get log messages before loading a model
	// This will log warnings to the log in case we have problems with detected GPUs
	gpus := discover.GPUDevices(ctx, nil)
	discover.LogDetails(gpus)

	var totalVRAM uint64
	for _, gpu := range gpus {
		totalVRAM += gpu.TotalMemory - envconfig.GpuOverhead()
	}

	// Set default context based on VRAM tier
	// Use slightly lower thresholds (47/23 GiB vs. 48/24 GiB) to account for small differences in the exact value
	switch {
	case totalVRAM >= 47*format.GibiByte:
		s.defaultNumCtx = 262144
	case totalVRAM >= 23*format.GibiByte:
		s.defaultNumCtx = 32768
	default:
		s.defaultNumCtx = 4096
	}
	slog.Info("vram-based default context", "total_vram", format.HumanBytes2(totalVRAM), "default_num_ctx", s.defaultNumCtx)

	err = srvr.Serve(ln)
	// If server is closed from the signal handler, wait for the ctx to be done
	// otherwise error out quickly
	if !errors.Is(err, http.ErrServerClosed) {
		return err
	}
	<-ctx.Done()
	return nil
}

func waitForStream(c *gin.Context, ch chan any) {
	c.Header("Content-Type", "application/json")
	var latest api.ProgressResponse
	for resp := range ch {
		switch r := resp.(type) {
		case api.ProgressResponse:
			latest = r
		case gin.H:
			status, ok := r["status"].(int)
			if !ok {
				status = http.StatusInternalServerError
			}
			errorMsg, ok := r["error"].(string)
			if !ok {
				errorMsg = "unknown error"
			}
			c.JSON(status, gin.H{"error": errorMsg})
			return
		default:
			c.JSON(http.StatusInternalServerError, gin.H{"error": "unknown message type"})
			return
		}
	}

	c.JSON(http.StatusOK, latest)
}

func streamResponse(c *gin.Context, ch chan any) {
	c.Header("Content-Type", "application/x-ndjson")
	c.Stream(func(w io.Writer) bool {
		val, ok := <-ch
		if !ok {
			return false
		}

		// errors are provided as a gin.H with an "error" field and
		// an optional "status" field.  For errors that are streamed
		// before any content, we need to set the status code and
		// content type for the error.
		if h, ok := val.(gin.H); ok {
			if e, ok := h["error"].(string); ok {
				status, ok := h["status"].(int)
				if !ok {
					status = http.StatusInternalServerError
				}

				if !c.Writer.Written() {
					c.Header("Content-Type", "application/json")
					c.JSON(status, gin.H{"error": e})
				} else {
					if err := json.NewEncoder(c.Writer).Encode(gin.H{"error": e}); err != nil {
						slog.Error("streamResponse failed to encode json error", "error", err)
					}
				}

				return false
			}
		}

		bts, err := json.Marshal(val)
		if err != nil {
			slog.Info(fmt.Sprintf("streamResponse: json.Marshal failed with %s", err))
			return false
		}

		// Delineate chunks with new-line delimiter
		bts = append(bts, '\n')
		if _, err := w.Write(bts); err != nil {
			slog.Info(fmt.Sprintf("streamResponse: w.Write failed with %s", err))
			return false
		}

		return true
	})
}

func (s *Server) StatusHandler(c *gin.Context) {
	disabled, source := internalcloud.Status()
	c.JSON(http.StatusOK, api.StatusResponse{
		Cloud: api.CloudStatus{
			Disabled: disabled,
			Source:   source,
		},
	})
}

func (s *Server) BalanceHandler(c *gin.Context) {
	c.Header("Cache-Control", "private, no-store")
	// Use the account signed into this server, regardless of caller credentials.
	c.Request.Header.Del("Authorization")
	proxyCloudRequest(c, nil, cloudErrBalanceUnavailable)
}

func (s *Server) UsageHandler(c *gin.Context) {
	c.Header("Cache-Control", "private, no-store")
	// Use the account signed into this server, regardless of caller credentials.
	c.Request.Header.Del("Authorization")
	proxyCloudRequest(c, nil, cloudErrUsageUnavailable)
}

func (s *Server) WebSearchExperimentalHandler(c *gin.Context) {
	s.webExperimentalProxyHandler(c, "/api/web_search", cloudErrWebSearchUnavailable)
}

func (s *Server) WebFetchExperimentalHandler(c *gin.Context) {
	s.webExperimentalProxyHandler(c, "/api/web_fetch", cloudErrWebFetchUnavailable)
}

func (s *Server) webExperimentalProxyHandler(c *gin.Context, proxyPath, disabledOperation string) {
	// This endpoint is authenticated by the server's cloud signature. A client
	// may have supplied an unrelated provider credential (for example, Codex's
	// Responses API key); it must not be sent to the web-search service.
	c.Request.Header.Del("Authorization")

	body, err := readRequestBody(c.Request)
	if err != nil {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		return
	}

	if len(bytes.TrimSpace(body)) == 0 {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
		return
	}

	proxyCloudRequestWithPath(c, body, proxyPath, disabledOperation)
}

func (s *Server) WhoamiHandler(c *gin.Context) {
	// todo allow other hosts
	u, err := url.Parse("https://ollama.com")
	if err != nil {
		slog.Error(err.Error())
		c.JSON(http.StatusInternalServerError, gin.H{"error": "URL parse error"})
		return
	}

	client := api.NewClient(u, http.DefaultClient)
	user, err := client.Whoami(c)
	if err != nil {
		var authErr api.AuthorizationError
		if errors.As(err, &authErr) && authErr.StatusCode == http.StatusUnauthorized {
			// Preserve an actionable sign-in response for launch; other failures
			// below mean account or plan verification is temporarily unavailable.
			sURL := authErr.SigninURL
			if sURL == "" {
				var sErr error
				sURL, sErr = signinURL()
				if sErr != nil {
					slog.Error(sErr.Error())
					c.JSON(http.StatusInternalServerError, gin.H{"error": "error getting authorization details"})
					return
				}
			}
			c.JSON(http.StatusUnauthorized, gin.H{"error": "unauthorized", "signin_url": sURL})
			return
		}

		slog.Error(err.Error())
		c.JSON(http.StatusServiceUnavailable, gin.H{"error": "account unavailable"})
		return
	}

	if user == nil || user.Name == "" {
		sURL, sErr := signinURL()
		if sErr != nil {
			slog.Error(sErr.Error())
			c.JSON(http.StatusInternalServerError, gin.H{"error": "error getting authorization details"})
			return
		}

		c.JSON(http.StatusUnauthorized, gin.H{"error": "unauthorized", "signin_url": sURL})
		return
	}

	if strings.TrimSpace(user.Plan) == "" {
		slog.Warn("account plan was not set; defaulting to free")
		user.Plan = "free"
	}
	c.JSON(http.StatusOK, user)
}

func (s *Server) SignoutHandler(c *gin.Context) {
	pubKey, err := auth.GetPublicKey()
	if err != nil {
		slog.Error("couldn't get public key", "error", err)
		c.JSON(http.StatusInternalServerError, gin.H{"error": "there was an error signing out"})
		return
	}

	encKey := base64.RawURLEncoding.EncodeToString([]byte(pubKey))

	// todo allow other hosts
	u, err := url.Parse("https://ollama.com")
	if err != nil {
		slog.Error(err.Error())
		c.JSON(http.StatusInternalServerError, gin.H{"error": "URL parse error"})
		return
	}

	client := api.NewClient(u, http.DefaultClient)
	err = client.Disconnect(c, encKey)
	if err != nil {
		var authError api.AuthorizationError
		if errors.As(err, &authError) {
			c.JSON(http.StatusUnauthorized, gin.H{"error": "you are not currently signed in"})
			return
		}
		c.JSON(http.StatusInternalServerError, gin.H{"error": "there was an error signing out"})
		return
	}

	c.JSON(http.StatusOK, nil)
}

func (s *Server) PsHandler(c *gin.Context) {
	models := []api.ProcessModelResponse{}

	for _, v := range s.sched.loadedModels() {
		m := v.model
		displayName := model.ParseName(m.ShortName).DisplayShortest()
		digest := m.ManifestDigest
		if digest == "" {
			digest = m.Digest
		}
		runner := v.runner
		if runner == "" {
			runner = m.Runner
		}
		if runner == "" {
			// Legacy manifests predate runner metadata; report the default
			// the scheduler applies for the config's weight format.
			runner, _ = manifest.MetadataForConfig(m.Config)
		}
		if normalized, err := normalizeRunner(runner); err == nil && normalized != "" {
			runner = normalized
		}
		modelDetails := api.ModelDetails{
			Format:            m.Config.ModelFormat,
			Family:            m.Config.ModelFamily,
			Families:          m.Config.ModelFamilies,
			ParameterSize:     m.Config.ModelType,
			QuantizationLevel: m.Config.FileType,
		}

		contextLength := v.contextLength
		if slices.Contains(m.Config.Capabilities, "speech") {
			// Speech models have no text context.
			contextLength = 0
		}

		mr := api.ProcessModelResponse{
			Model:         displayName,
			Name:          displayName,
			Size:          v.size,
			SizeVRAM:      v.sizeVRAM,
			Digest:        digest,
			Details:       modelDetails,
			ExpiresAt:     v.expiresAt,
			ContextLength: contextLength,
			Runner:        runner,
		}

		models = append(models, mr)
	}

	slices.SortStableFunc(models, func(i, j api.ProcessModelResponse) int {
		// longest duration remaining listed first
		return cmp.Compare(j.ExpiresAt.Unix(), i.ExpiresAt.Unix())
	})

	c.JSON(http.StatusOK, api.ProcessResponse{Models: models})
}

func toolCallId() string {
	const letterBytes = "abcdefghijklmnopqrstuvwxyz0123456789"
	b := make([]byte, 8)
	for i := range b {
		b[i] = letterBytes[rand.Intn(len(letterBytes))]
	}
	return "call_" + strings.ToLower(string(b))
}

func preservedTokensForCompletion(builtinParser parsers.Parser) []string {
	if builtinParser != nil {
		return builtinParser.PreservedTokens()
	}
	return nil
}

func toolCallTagForCompletion(toolParser *tools.Parser) string {
	if toolParser == nil {
		return ""
	}
	return toolParser.Tag()
}

func thinkingCloseForCompletion(builtinParser parsers.Parser, thinkTagParser *thinkingparser.Parser) []string {
	if builtinParser != nil {
		return builtinParser.ThinkingClose()
	}
	if thinkTagParser != nil {
		return []string{thinkTagParser.ClosingTag}
	}
	return nil
}

func leadingBOSForModel(m *Model) string {
	if m == nil || m.Config.Renderer == "" {
		return ""
	}

	return renderers.LeadingBOSForRenderer(resolveRendererName(m))
}

func optionsForPrompt(opts *api.Options, runner llm.LlamaServer) *api.Options {
	if opts == nil || runner == nil {
		return opts
	}

	if ctxLen := runner.ContextLength(); ctxLen > 0 && opts.NumCtx > ctxLen {
		copied := *opts
		copied.NumCtx = ctxLen
		return &copied
	}

	return opts
}

type chatExecutionMode int

const (
	chatExecutionModeNative chatExecutionMode = iota
	chatExecutionModeRendered
)

func chatModeForModel(m *Model) chatExecutionMode {
	if m.IsMLX() || usesOllamaRenderedChat(m) {
		return chatExecutionModeRendered
	}

	return chatExecutionModeNative
}

func llamaServerConfigForModel(m *Model) llm.LlamaServerConfig {
	return llm.LlamaServerConfig{
		DisableJinja:         usesOllamaRenderedChat(m),
		ManifestDigest:       m.Digest,
		DraftModelPath:       m.DraftPath,
		DraftModelShardPaths: slices.Clone(m.DraftShardPaths),
	}
}

func usesOllamaRenderedChat(m *Model) bool {
	return m != nil && (m.Config.Renderer != "" || m.Config.Parser != "" || shouldUseHarmony(m) || shouldUseGoTemplate(m))
}

func shouldUseGoTemplate(m *Model) bool {
	if !m.HasGoTemplate {
		return false
	}
	if goTemplateEnvSet() {
		return envconfig.GoTemplate(true)
	}

	return !m.PreferChatTemplate && envconfig.GoTemplate(true)
}

func writeChatResponse(c *gin.Context, req api.ChatRequest, ch chan any) {
	if req.Stream != nil && !*req.Stream {
		var resp api.ChatResponse
		var toolCalls []api.ToolCall
		var allLogprobs []api.Logprob
		var sbThinking strings.Builder
		var sbContent strings.Builder
		for rr := range ch {
			switch t := rr.(type) {
			case api.ChatResponse:
				sbThinking.WriteString(t.Message.Thinking)
				sbContent.WriteString(t.Message.Content)
				resp = t
				if len(req.Tools) > 0 {
					toolCalls = append(toolCalls, t.Message.ToolCalls...)
				}
				if len(t.Logprobs) > 0 {
					allLogprobs = append(allLogprobs, t.Logprobs...)
				}
			case gin.H:
				msg, ok := t["error"].(string)
				if !ok {
					msg = "unexpected error format in response"
				}

				status, ok := t["status"].(int)
				if !ok {
					status = http.StatusInternalServerError
				}

				c.JSON(status, gin.H{"error": msg})
				return
			default:
				c.JSON(http.StatusInternalServerError, gin.H{"error": "unexpected response"})
				return
			}
		}

		resp.Message.Content = sbContent.String()
		resp.Message.Thinking = sbThinking.String()
		resp.Logprobs = allLogprobs

		if len(toolCalls) > 0 {
			resp.Message.ToolCalls = toolCalls
		}

		c.JSON(http.StatusOK, resp)
		return
	}

	streamResponse(c, ch)
}

func (s *Server) ChatHandler(c *gin.Context) {
	checkpointStart := time.Now()

	var req api.ChatRequest
	body := struct {
		*api.ChatRequest
		Think json.RawMessage `json:"think"`
	}{ChatRequest: &req}
	if err := c.ShouldBindJSON(&body); errors.Is(err, io.EOF) {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
		return
	} else if err != nil {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		return
	}

	if len(body.Think) > 0 {
		if err := json.Unmarshal(body.Think, &req.Think); err != nil {
			err = s.thinkingInputError(c.Request.Context(), req.Model, err)
			c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
			return
		}
	}

	if req.TopLogprobs < 0 || req.TopLogprobs > 20 {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "top_logprobs must be between 0 and 20"})
		return
	}

	modelRef, err := parseAndValidateModelRef(req.Model)
	if err != nil {
		writeModelRefParseError(c, err, http.StatusBadRequest, "model is required")
		return
	}

	if modelRef.Source == modelSourceCloud {
		req.Model = modelRef.Base
		req.Runner = ""
		if c.GetBool(cloudWebSearchOrchestrationKey) {
			proxyCloudJSONRequestWithPath(c, req, "/api/chat", cloudErrRemoteInferenceUnavailable)
			return
		}
		proxyCloudJSONRequest(c, req, cloudErrRemoteInferenceUnavailable)
		return
	}

	if runner, err := normalizeRunner(req.Runner); err != nil {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		return
	} else {
		req.Runner = runner
	}

	m, err := s.getExistingModelForRunner(modelRef.Name.String(), req.Runner)
	if err != nil {
		switch {
		case errors.Is(err, os.ErrNotExist):
			c.JSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("model '%s' not found", req.Model)})
		case errors.Is(err, manifest.ErrNoCompatibleManifest):
			c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		case err.Error() == errtypes.InvalidModelNameErrMsg:
			c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		default:
			c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		}
		return
	}

	if req.TopLogprobs < 0 || req.TopLogprobs > 20 {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "top_logprobs must be between 0 and 20"})
		return
	}

	if modelRef.Source == modelSourceLocal && m.Config.RemoteHost != "" && m.Config.RemoteModel != "" {
		c.JSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("model '%s' not found", req.Model)})
		return
	}

	// expire the runner
	if len(req.Messages) == 0 && req.KeepAlive != nil && req.KeepAlive.Duration == 0 {
		s.sched.expireRunner(m)

		c.JSON(http.StatusOK, api.ChatResponse{
			Model:      req.Model,
			CreatedAt:  time.Now().UTC(),
			Message:    api.Message{Role: "assistant"},
			Done:       true,
			DoneReason: "unload",
		})
		return
	}

	if m.Config.RemoteHost != "" && m.Config.RemoteModel != "" {
		if disabled, _ := internalcloud.Status(); disabled {
			c.JSON(http.StatusForbidden, gin.H{"error": internalcloud.DisabledError(cloudErrRemoteInferenceUnavailable)})
			return
		}

		origModel := req.Model

		remoteURL, err := url.Parse(m.Config.RemoteHost)
		if err != nil {
			c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
			return
		}

		if !slices.Contains(envconfig.Remotes(), remoteURL.Hostname()) {
			slog.Info("remote model", "remotes", envconfig.Remotes(), "remoteURL", m.Config.RemoteHost, "hostname", remoteURL.Hostname())
			c.JSON(http.StatusBadRequest, gin.H{"error": "this server cannot run this remote model"})
			return
		}

		req.Model = m.Config.RemoteModel
		req.Runner = ""
		if req.Options == nil {
			req.Options = map[string]any{}
		}

		req.Messages = chatMessagesForRequest(m, req.Messages)

		for k, v := range m.Options {
			if _, ok := req.Options[k]; !ok {
				req.Options[k] = v
			}
		}

		contentType := "application/x-ndjson"
		if req.Stream != nil && !*req.Stream {
			contentType = "application/json; charset=utf-8"
		}
		c.Header("Content-Type", contentType)

		fn := func(resp api.ChatResponse) error {
			resp.Model = origModel
			resp.RemoteModel = m.Config.RemoteModel
			resp.RemoteHost = m.Config.RemoteHost

			data, err := json.Marshal(resp)
			if err != nil {
				return err
			}

			if _, err = c.Writer.Write(append(data, '\n')); err != nil {
				return err
			}
			c.Writer.Flush()
			return nil
		}

		client := api.NewClient(remoteURL, http.DefaultClient)
		err = client.Chat(c.Request.Context(), &req, fn)
		if err != nil {
			var authError api.AuthorizationError
			if errors.As(err, &authError) {
				sURL, sErr := signinURL()
				if sErr != nil {
					slog.Error(sErr.Error())
					c.JSON(http.StatusInternalServerError, gin.H{"error": "error getting authorization details"})
					return
				}

				c.JSON(authError.StatusCode, gin.H{"error": "unauthorized", "signin_url": sURL})
				return
			}
			var apiError api.StatusError
			if errors.As(err, &apiError) {
				c.JSON(apiError.StatusCode, apiError)
				return
			}
			c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
			return
		}

		return
	}

	thinking := m.genericThinking()
	if thinking == nil {
		if err := api.ValidateLegacyThinking(req.Think); err != nil {
			c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
			return
		}
	}

	caps := []model.Capability{model.CapabilityCompletion}
	if len(req.Tools) > 0 {
		caps = append(caps, model.CapabilityTools)
	}

	requestedThink := req.Think
	think := renderers.ResolveThinking(requestedThink, thinking)
	req.Think = think
	modelCaps := m.Capabilities()
	if slices.Contains(modelCaps, model.CapabilityThinking) {
		caps = append(caps, model.CapabilityThinking)
		if req.Think == nil && thinking == nil {
			req.Think = &api.ThinkValue{Value: true}
		}
	} else {
		if req.Think != nil && req.Think.Bool() {
			// Clients may retain thinking settings when switching models.
			if _, ok := c.Get("relax_thinking"); ok {
				slog.Debug("model does not support thinking, relaxing thinking to nil", "model", req.Model)
				req.Think = nil
			} else {
				c.JSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("%q does not support thinking", req.Model)})
				return
			}
		}
	}

	r, m, opts, err := s.scheduleRunnerForModel(c.Request.Context(), m, caps, req.Options, req.KeepAlive, req.Shift)
	if errors.Is(err, errCapabilityCompletion) {
		c.JSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("%q does not support chat", req.Model)})
		return
	} else if err != nil {
		handleScheduleError(c, req.Model, err)
		return
	}

	checkpointLoaded := time.Now()

	if len(req.Messages) == 0 {
		c.JSON(http.StatusOK, api.ChatResponse{
			Model:      req.Model,
			CreatedAt:  time.Now().UTC(),
			Message:    api.Message{Role: "assistant"},
			Done:       true,
			DoneReason: "load",
		})
		return
	}

	msgs := chatMessagesForRequest(m, req.Messages)

	parserName := parserNameForModel(m)
	if shouldUseHarmony(m) {
		// harmony's Reasoning field only understands low/medium/high; map "max" to "high"
		if req.Think != nil {
			if s, ok := req.Think.Value.(string); ok && s == "max" {
				req.Think.Value = "high"
			}
		}
	}

	// Reuse resolved thinking for server-side tool follow-ups.
	if w, ok := c.Writer.(interface{ SetThinking(*api.ThinkValue) }); ok {
		w.SetThinking(req.Think)
	}

	if chatModeForModel(m) == chatExecutionModeNative {
		s.handleNativeChat(c, req, m, r, opts, msgs, checkpointStart, checkpointLoaded)
		return
	}

	var builtinParser parsers.Parser
	processedTools := req.Tools

	if parserName != "" {
		builtinParser = parsers.ParserForName(parserName)
		if builtinParser != nil {
			// Determine last message for chat prefill
			var lastMessage *api.Message
			if len(msgs) > 0 {
				lastMessage = &msgs[len(msgs)-1]
			}
			// Initialize parser and get processed tools
			processedTools = builtinParser.Init(req.Tools, lastMessage, req.Think)
		}
	}

	truncate := req.Truncate == nil || *req.Truncate
	if m.IsMLX() {
		truncate = false
	}
	promptOpts := optionsForPrompt(opts, r)
	prompt, media, err := chatPrompt(c.Request.Context(), m, r.Tokenize, promptOpts, msgs, processedTools, req.Think, truncate)
	if err != nil {
		slog.Error("chat prompt error", "error", err)
		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		return
	}

	// If debug mode is enabled, return the rendered template instead of calling the model
	if req.DebugRenderOnly {
		c.JSON(http.StatusOK, api.ChatResponse{
			Model:     req.Model,
			CreatedAt: time.Now().UTC(),
			DebugInfo: &api.DebugInfo{
				RenderedTemplate: prompt,
				ImageCount:       len(media),
			},
		})
		return
	}

	var thinkTagParser *thinkingparser.Parser
	openingTag, closingTag := thinkingparser.InferTags(m.Template.Template)
	if req.Think != nil && req.Think.Bool() && openingTag != "" && closingTag != "" {
		thinkTagParser = &thinkingparser.Parser{
			OpeningTag: openingTag,
			ClosingTag: closingTag,
		}

		if strings.HasSuffix(strings.TrimSpace(prompt), openingTag) {
			thinkTagParser.AddContent(openingTag)
		}
	}

	var toolParser *tools.Parser
	if len(req.Tools) > 0 && (builtinParser == nil || !builtinParser.HasToolSupport()) {
		toolParser = tools.NewParser(m.Template.Template, req.Tools)
	}

	ch := make(chan any)
	go func() {
		defer close(ch)

		ctx, cancel := context.WithCancel(c.Request.Context())
		defer cancel()

		var parserErr error

		err := r.Completion(ctx, llm.CompletionRequest{
			Prompt:          prompt,
			Media:           media,
			Format:          req.Format,
			Options:         opts,
			Shift:           req.Shift == nil || *req.Shift,
			Truncate:        truncate,
			Logprobs:        req.Logprobs,
			TopLogprobs:     req.TopLogprobs,
			PreservedTokens: preservedTokensForCompletion(builtinParser),
			ToolCallTag:     toolCallTagForCompletion(toolParser),
			LeadingBOS:      leadingBOSForModel(m),
			ThinkingClose:   thinkingCloseForCompletion(builtinParser, thinkTagParser),
		}, func(r llm.CompletionResponse) {
			res := api.ChatResponse{
				Model:     req.Model,
				CreatedAt: time.Now().UTC(),
				Message:   api.Message{Role: "assistant", Content: r.Content},
				Done:      r.Done,
				Metrics: api.Metrics{
					PromptEvalCount:       r.PromptEvalCount,
					PromptEvalCachedCount: r.PromptEvalCachedCount,
					PromptEvalDuration:    r.PromptEvalDuration,
					EvalCount:             r.EvalCount,
					EvalDuration:          r.EvalDuration,
				},
				Logprobs: toAPILogprobs(r.Logprobs),
			}

			if r.Done {
				res.DoneReason = r.DoneReason.String()
				res.TotalDuration = time.Since(checkpointStart)
				res.LoadDuration = checkpointLoaded.Sub(checkpointStart)
			}

			if builtinParser != nil {
				slog.Log(ctx, logutil.LevelTrace, "builtin parser input", "parser", parserName, "content", r.Content)

				content, thinking, toolCalls, err := builtinParser.Add(r.Content, r.Done)
				if err != nil {
					parserErr = err
					cancel()
					return
				}

				res.Message.Content = content
				res.Message.Thinking = thinking
				for i := range toolCalls {
					toolCalls[i].ID = toolCallId()
				}
				res.Message.ToolCalls = toolCalls

				if res.Message.Content != "" || res.Message.Thinking != "" || len(res.Message.ToolCalls) > 0 || r.Done || len(res.Logprobs) > 0 {
					slog.Log(ctx, logutil.LevelTrace, "builtin parser output", "parser", parserName, "content", content, "thinking", thinking, "toolCalls", toolCalls, "done", r.Done)
					ch <- res
				} else {
					slog.Log(ctx, logutil.LevelTrace, "builtin parser empty output", "parser", parserName)
				}
				return
			}

			if thinkTagParser != nil {
				thinkingContent, remainingContent := thinkTagParser.AddContent(res.Message.Content)
				if thinkingContent == "" && remainingContent == "" && !r.Done {
					// need to accumulate more to decide what to send
					return
				}
				res.Message.Thinking = thinkingContent
				res.Message.Content = remainingContent
			}

			if len(req.Tools) > 0 {
				toolCalls, content := toolParser.Add(res.Message.Content)
				if len(content) > 0 {
					res.Message.Content = content
				} else if len(toolCalls) > 0 {
					for i := range toolCalls {
						toolCalls[i].ID = toolCallId()
					}
					res.Message.ToolCalls = toolCalls
					res.Message.Content = ""
				} else if res.Message.Thinking != "" {
					// don't return, fall through to send
				} else {
					//  Send logprobs while content is being buffered by the parser for tool calls
					if len(res.Logprobs) > 0 && !r.Done {
						logprobRes := res
						logprobRes.Message.Content = ""
						logprobRes.Message.ToolCalls = nil
						ch <- logprobRes
					}

					if r.Done {
						res.Message.Content = toolParser.Content()
						ch <- res
					}
					return
				}
			}

			ch <- res
		})
		if parserErr != nil {
			ch <- gin.H{"error": parserErr.Error()}
			return
		}
		if err != nil {
			s.sched.expireRunnersForRuntimeOOM(m, err)
			var serr api.StatusError
			if errors.As(err, &serr) {
				ch <- gin.H{"error": serr.ErrorMessage, "status": serr.StatusCode}
			} else {
				ch <- gin.H{"error": err.Error()}
			}
		}
	}()

	writeChatResponse(c, req, ch)
}

func prepareNativeChatRequest(ctx context.Context, m *Model, r llm.LlamaServer, opts *api.Options, nativeReq llm.ChatRequest, truncate bool) (llm.ChatRequest, error) {
	var err error
	nativeReq.Messages, err = truncateNativeChatMessages(ctx, m, r, optionsForPrompt(opts, r), nativeReq, truncate)
	return nativeReq, err
}

func (s *Server) handleNativeChat(c *gin.Context, req api.ChatRequest, m *Model, r llm.LlamaServer, opts *api.Options, msgs []api.Message, checkpointStart, checkpointLoaded time.Time) {
	truncate := req.Truncate == nil || *req.Truncate
	nativeReq, err := prepareNativeChatRequest(c.Request.Context(), m, r, opts, llm.ChatRequest{
		Messages:    msgs,
		Tools:       req.Tools,
		Format:      req.Format,
		Options:     opts,
		Think:       req.Think,
		Shift:       req.Shift == nil || *req.Shift,
		Logprobs:    req.Logprobs,
		TopLogprobs: req.TopLogprobs,
	}, truncate)
	if err != nil {
		slog.Error("chat template prompt error", "error", err)
		var serr api.StatusError
		if errors.As(err, &serr) {
			c.JSON(serr.StatusCode, gin.H{"error": serr.ErrorMessage})
		} else {
			c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		}
		return
	}

	if req.DebugRenderOnly {
		prompt, err := r.ApplyChatTemplate(c.Request.Context(), nativeReq)
		if err != nil {
			var serr api.StatusError
			if errors.As(err, &serr) {
				c.JSON(serr.StatusCode, gin.H{"error": serr.ErrorMessage})
			} else {
				c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
			}
			return
		}

		c.JSON(http.StatusOK, api.ChatResponse{
			Model:     req.Model,
			CreatedAt: time.Now().UTC(),
			DebugInfo: &api.DebugInfo{
				RenderedTemplate: prompt,
				ImageCount:       countChatImages(msgs),
			},
		})
		return
	}

	ch := make(chan any)
	go func() {
		defer close(ch)

		err := r.Chat(c.Request.Context(), nativeReq, func(r llm.ChatResponse) {
			res := api.ChatResponse{
				Model:     req.Model,
				CreatedAt: time.Now().UTC(),
				Message:   r.Message,
				Done:      r.Done,
				Metrics: api.Metrics{
					PromptEvalCount:       r.PromptEvalCount,
					PromptEvalCachedCount: r.PromptEvalCachedCount,
					PromptEvalDuration:    r.PromptEvalDuration,
					EvalCount:             r.EvalCount,
					EvalDuration:          r.EvalDuration,
				},
				Logprobs: toAPILogprobs(r.Logprobs),
			}

			if res.Message.Role == "" {
				res.Message.Role = "assistant"
			}

			if r.Done {
				res.DoneReason = r.DoneReason.String()
				res.TotalDuration = time.Since(checkpointStart)
				res.LoadDuration = checkpointLoaded.Sub(checkpointStart)
			}

			ch <- res
		})
		if err != nil {
			s.sched.expireRunnersForRuntimeOOM(m, err)
			var serr api.StatusError
			if errors.As(err, &serr) {
				ch <- gin.H{"error": serr.ErrorMessage, "status": serr.StatusCode}
			} else {
				ch <- gin.H{"error": err.Error()}
			}
		}
	}()

	writeChatResponse(c, req, ch)
}

func truncateNativeChatMessages(ctx context.Context, m *Model, r llm.LlamaServer, opts *api.Options, req llm.ChatRequest, truncate bool) ([]api.Message, error) {
	if !truncate || opts == nil || opts.NumCtx <= 0 || len(req.Messages) <= 1 {
		return req.Messages, nil
	}

	lastMsgIdx := len(req.Messages) - 1
	currMsgIdx := 0
	var system []api.Message

	for i := 0; i <= lastMsgIdx; i++ {
		system = system[:0]
		for j := range i {
			if req.Messages[j].Role == "system" {
				system = append(system, req.Messages[j])
			}
		}

		renderReq := req
		renderReq.Messages = append(slices.Clone(system), req.Messages[i:]...)
		prompt, err := r.ApplyChatTemplate(ctx, renderReq)
		if err != nil {
			return nil, err
		}

		tokens, err := r.Tokenize(ctx, prompt)
		if err != nil {
			return nil, err
		}

		ctxLen := len(tokens)
		if m != nil && m.ProjectorPaths != nil {
			for _, msg := range renderReq.Messages {
				ctxLen += 768 * len(msg.Images)
			}
		}

		if ctxLen <= opts.NumCtx {
			currMsgIdx = i
			break
		}
		if i == lastMsgIdx {
			currMsgIdx = lastMsgIdx
			break
		}
	}

	if currMsgIdx > 0 {
		slog.Debug("truncating native chat messages which exceed context length", "truncated", currMsgIdx)
	}

	system = system[:0]
	for j := range currMsgIdx {
		if req.Messages[j].Role == "system" {
			system = append(system, req.Messages[j])
		}
	}
	return append(slices.Clone(system), req.Messages[currMsgIdx:]...), nil
}

func countChatImages(msgs []api.Message) int {
	var count int
	for _, msg := range msgs {
		count += len(msg.Images)
	}
	return count
}

func handleScheduleError(c *gin.Context, name string, err error) {
	status, message := scheduleError(name, err)
	c.JSON(status, gin.H{"error": message})
}

// scheduleError keeps native and OpenAI-compatible endpoints consistent while
// allowing each to use its own error response format.
func scheduleError(name string, err error) (int, string) {
	switch {
	case errors.Is(err, errCapabilities), errors.Is(err, errRequired), errors.Is(err, manifest.ErrNoCompatibleManifest):
		return http.StatusBadRequest, err.Error()
	case errors.Is(err, context.Canceled):
		return 499, "request canceled"
	case errors.Is(err, ErrMaxQueue):
		return http.StatusServiceUnavailable, err.Error()
	case errors.Is(err, os.ErrNotExist):
		return http.StatusNotFound, fmt.Sprintf("model %q not found, try pulling it first", name)
	default:
		return http.StatusInternalServerError, err.Error()
	}
}

func chatMessagesForRequest(m *Model, requestMessages []api.Message) []api.Message {
	if len(requestMessages) == 0 {
		return nil
	}

	// Loaded runners retain m, so request messages must not share its backing slice.
	msgs := append(slices.Clone(m.Messages), requestMessages...)
	if requestMessages[0].Role != "system" && m.System != "" {
		msgs = append([]api.Message{{Role: "system", Content: m.System}}, msgs...)
	}
	return filterThinkTags(msgs, m)
}

func filterThinkTags(msgs []api.Message, m *Model) []api.Message {
	if m.Config.ModelFamily == "qwen3" || model.ParseName(m.Name).Model == "deepseek-r1" {
		finalUserIndex := -1
		for i, msg := range msgs {
			if msg.Role == "user" {
				finalUserIndex = i
			}
		}

		for i, msg := range msgs {
			if msg.Role == "assistant" && i < finalUserIndex {
				// TODO(drifkin): this is from before we added proper thinking support.
				// However, even if thinking is not enabled (and therefore we shouldn't
				// change the user output), we should probably perform this filtering
				// for all thinking models (not just qwen3 & deepseek-r1) since it tends
				// to save tokens and improve quality.
				thinkTagParser := &thinkingparser.Parser{
					OpeningTag: "<think>",
					ClosingTag: "</think>",
				}
				_, content := thinkTagParser.AddContent(msg.Content)
				msgs[i].Content = content
			}
		}
	}
	return msgs
}