Skip to content

Router that virtually distributes inference across connected devices in the home.

Go
9.7 KB
Raw Blame
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

package main

// This is the only file in the proxy that carries table-driven engine values.
// Everything else reads them from the profile it is handed.
//
// Cross-process identity — Name, DisplayName, DiscoveryService, Namespace,
// FacadePort, EnginePortBase — lives in nvpair-shared/engines, because the
// broker, the scheduler and the TUI need the same values and a second
// declaration here could drift from theirs without failing to compile. What
// remains below is what only the proxy needs to know.
//
// Adding an engine is one entry here plus one in the broker's table. The
// values that are derivable are derived: the applog component and the error-ID
// prefix both come from Name via ComponentName, so they cannot disagree with
// each other. The persisted-port filename is declared instead, because
// Ollama's predates the multi-engine world — see PortFile in
// nvpair-shared/engines.

import (
	"slices"
	"strings"

	"nvpair-shared/engines"
)

// routeRole classifies an inbound request path. It carries the HTTP method
// because the method is part of the classification, not a separate axis: a
// POST to /v1/models is not a model list, and a GET to /api/chat is not
// inference. Folding them into one constant keeps the two from disagreeing.
//
// The dialect distinction is meaningful only for model-list roles. All of
// Ollama's inference paths are handled identically — no envelope, identity
// field or response shape is selected — so there is deliberately no
// per-dialect inference role.
type routeRole int

const (
	// roleInferencePOST is a POST that counts as a cluster workload: it is
	// routed to an owner of the requested model, emits workload lifecycle
	// notifications, and retries a 404 on another owner.
	roleInferencePOST routeRole = iota

	// roleModelListNativeGET is Ollama's /api/tags dialect: the upstream
	// envelope carries a "models" array, each record's identity is its
	// "model" field falling back to "name", and the federated response is
	// re-emitted as {"models": [...]}.
	roleModelListNativeGET

	// roleModelListOpenAIGET is the OpenAI /v1/models dialect: the upstream
	// envelope carries a "data" array, each record's identity is its "id",
	// and the federated response is re-emitted as
	// {"object": "list", "data": [...]}.
	roleModelListOpenAIGET
)

// modelNaming is how an engine spells a model identifier. It governs the
// native model list's dedupe key and nodeAdvertisesModel's match against a
// node's advertised inventory; those two must agree there, or the proxy can
// list a model it then refuses to route.
//
// The OpenAI list path (roleModelListOpenAIGET) is deliberately outside that
// pairing: it dedupes on the upstream identity verbatim, as the pre-unification
// proxies both did, while nodeAdvertisesModel still normalizes. Widening the
// claim to "both list paths" would describe code that does not exist.
type modelNaming int

const (
	// impliedLatestTag is Ollama's convention: an untagged name means the
	// ":latest" tag, so "llama3" and "llama3:latest" are one model.
	impliedLatestTag modelNaming = iota

	// exactID compares identifiers byte for byte.
	exactID
)

// route is one classified request path.
type route struct {
	// Path is the client-facing path matched on the facade.
	Path string

	// UpstreamPath optionally rewrites Path for the engine; empty preserves it.
	UpstreamPath string
	Role         routeRole
}

// engineProfile is everything the proxy needs to front one engine.
type engineProfile struct {
	engines.Engine

	// StandalonePort is the port the proxy binds when nothing passes --port
	// and no port has been persisted. It must be neither FacadePort (which
	// the engine itself owns) nor a port an engine is relocated onto.
	//
	// Ollama's 11435 currently equals its EnginePortBase, which violates that
	// second half. It is left as-is because it is the value on disk in every
	// existing install; see the plan's port-field note.
	StandalonePort int

	// Routes classifies the paths this engine's clients call. It is NOT an
	// allowlist: an unlisted path is forwarded verbatim, which is how
	// /api/show, /api/pull, /api/ps, /api/version and OPTIONS preflights keep
	// working.
	Routes []route

	// ModelNaming is how this engine spells model identifiers.
	ModelNaming modelNaming

	// ReservedPersistedPort is a port that must never be restored from the
	// persisted-port file even if it is stored there, because it belongs to
	// the engine rather than the proxy. Zero means no port is reserved.
	ReservedPersistedPort int

	// SupportsHostAlias reports whether facade/enable's aliasAddresses apply to
	// this engine. Only Ollama has an inherited host variable (OLLAMA_HOST) for the
	// alias to stand in for, and the warning the proxy raises when it cannot
	// claim one names that variable. Gating on this makes the scoping
	// enforced rather than left to the broker's restraint in passing the flag.
	SupportsHostAlias bool
}

// ollamaBaseRoutes is the engine-specific surface that Ollama exposes before
// the shared compatibility routes are added.
var ollamaBaseRoutes = []route{
	{Path: "/api/generate", Role: roleInferencePOST},
	{Path: "/api/chat", Role: roleInferencePOST},
	{Path: "/api/embeddings", Role: roleInferencePOST},
	{Path: "/api/embed", Role: roleInferencePOST},
	{Path: "/api/tags", Role: roleModelListNativeGET},
	{Path: "/v1/models", Role: roleModelListOpenAIGET},
}

// lmStudioBaseRoutes is the engine-specific surface that LM Studio exposes
// before the shared compatibility routes are added.
var lmStudioBaseRoutes = []route{
	{Path: "/v1/models", Role: roleModelListOpenAIGET},
}

// llamaCPPBaseRoutes serves both model-list paths through the fleet inventory,
// querying llama.cpp's router endpoint. Its response uses the OpenAI envelope.
var llamaCPPBaseRoutes = []route{
	{Path: "/models", Role: roleModelListOpenAIGET},
	{Path: "/v1/models", UpstreamPath: "/models", Role: roleModelListOpenAIGET},
}

// openAIInferenceRoutes is the OpenAI-compatible inference surface.
var openAIInferenceRoutes = []route{
	{Path: "/v1/chat/completions", Role: roleInferencePOST},
	{Path: "/v1/completions", Role: roleInferencePOST},
	{Path: "/v1/embeddings", Role: roleInferencePOST},
}

// anthropicInferenceRoutes is the Anthropic-compatible inference surface.
var anthropicInferenceRoutes = []route{
	{Path: "/v1/messages", Role: roleInferencePOST},
}

var profiles = buildProfiles()

func buildProfiles() []engineProfile {
	ollama, _ := engines.ByName("ollama")
	lmstudio, _ := engines.ByName("lmstudio")
	llamacpp, _ := engines.ByName("llamacpp")
	ollamaRoutes := slices.Concat(ollamaBaseRoutes, openAIInferenceRoutes, anthropicInferenceRoutes)
	lmStudioRoutes := slices.Concat(lmStudioBaseRoutes, openAIInferenceRoutes, anthropicInferenceRoutes)
	llamaCPPRoutes := slices.Concat(llamaCPPBaseRoutes, openAIInferenceRoutes)

	return []engineProfile{
		{
			Engine:                ollama,
			StandalonePort:        11435,
			Routes:                ollamaRoutes,
			ModelNaming:           impliedLatestTag,
			ReservedPersistedPort: 0,
			SupportsHostAlias:     true,
		},
		{
			Engine:         lmstudio,
			StandalonePort: 1234,
			Routes:         lmStudioRoutes,
			ModelNaming:    exactID,
			// 1235 is where engine-manager runs a managed LM Studio, so a
			// proxy that restored it would sit on the engine's own port. The
			// stored value predates the current default of 1234.
			ReservedPersistedPort: 1235,
		},
		{
			Engine:                llamacpp,
			StandalonePort:        8080,
			Routes:                llamaCPPRoutes,
			ModelNaming:           exactID,
			ReservedPersistedPort: 8081,
		},
	}
}

// profileFor resolves the engine named in a facade/enable request.
func profileFor(name string) (engineProfile, bool) {
	for _, p := range profiles {
		if p.Name == name {
			return p, true
		}
	}
	return engineProfile{}, false
}

// engineNames lists the accepted engine ids for a facade/enable rejection.
func engineNames() string {
	names := make([]string, 0, len(profiles))
	for _, p := range profiles {
		names = append(names, p.Name)
	}
	return strings.Join(names, ", ")
}

// routeFor classifies a request. The bool reports whether the path is one this
// engine handles specially; false means forward it verbatim.
func (p engineProfile) routeFor(method, path string) (route, bool) {
	for _, r := range p.Routes {
		// Keep scanning on a method mismatch rather than bailing: a path may
		// legitimately appear twice under different methods, and returning
		// here would make the second entry permanently unreachable — silently
		// forwarded verbatim instead of classified.
		if r.Path != path || r.Role.method() != method {
			continue
		}
		return r, true
	}
	return route{}, false
}

func (p engineProfile) roleFor(method, path string) (routeRole, bool) {
	r, ok := p.routeFor(method, path)
	return r.Role, ok
}

func (r route) upstreamPath() string {
	if r.UpstreamPath != "" {
		return r.UpstreamPath
	}
	return r.Path
}

// method is the HTTP method a role applies to.
func (r routeRole) method() string {
	if r == roleInferencePOST {
		return "POST"
	}
	return "GET"
}

// isModelList reports whether a role serves the federated model list.
func (r routeRole) isModelList() bool {
	return r == roleModelListNativeGET || r == roleModelListOpenAIGET
}

// normalizeModel applies the engine's naming convention to a model
// identifier, so the federated list's dedupe key and the routing gate agree.
func (p engineProfile) normalizeModel(model string) string {
	model = strings.TrimSpace(model)
	if p.ModelNaming != impliedLatestTag || model == "" {
		return model
	}
	name := model[strings.LastIndex(model, "/")+1:]
	if name != "" && !strings.ContainsAny(name, ":@") {
		return model + ":latest"
	}
	return model
}