Skip to content

Router that virtually distributes inference across connected devices in the home.

Go
8.5 KB
Raw Blame
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

package ui

import (
	"fmt"
	"log/slog"
	"strings"

	"nvpair-shared/engines"
	"nvpair-tui/rpc"

	tea "github.com/charmbracelet/bubbletea"
)

// proxyEngine is one engine's facade on the proxy the broker fronts. Every
// facade speaks the same control contract; only the JSON-RPC prefix and the
// label differ.
//
// A facade, not a process: one nvpair-proxy serves them all, so readiness and
// port are per-engine here while a crash is reported once against the process
// (see serviceWorkers). The prefix is the per-engine ComponentName, because
// addressing "ollama-proxy:nodes/list" means that engine regardless of which
// process happens to serve it.
type proxyEngine struct {
	label  string // "Ollama" / "LM Studio"
	prefix string // "ollama-proxy" / "lmstudio-proxy"
	// engine is the engine-manager engine name this facade fronts, used to line
	// a facade up with the engine it serves.
	engine string
	ready  bool
	port   int
}

// proxyTracker keeps every facade's readiness and listen port current from the
// broker. It is a shared component rather than a tab: where a request is served
// belongs with the jobs, while which port to listen on belongs with the rest of
// the service configuration.
//
// It deliberately does not track upstreams. The proxies' node lists are a
// second, staler view of the machines the Nodes tab already owns — and one that
// kept showing peers from a cluster this node had left.
type proxyTracker struct {
	engines []*proxyEngine
}

type proxyStatusMsg struct {
	idx   int
	ready bool
	port  int
	err   error
}

// engineDisplayName is an engine's wire id rendered the way the operator sees
// it elsewhere.
//
// The Jobs and Errors tabs name engines too, and neither has an engine list to
// resolve a label from — they receive the id on the wire and nothing else. The
// pairing is already fixed here, in the proxy inventory, so this reads it from
// the same place rather than repeating it. An unknown id passes through, which
// is what a new engine would produce until this list caught up.
func engineDisplayName(engine string) string {
	want := strings.ToLower(strings.TrimSpace(engine))
	for _, e := range newProxyTracker().engines {
		if e.engine == want {
			return e.label
		}
	}
	return engine
}

// newProxyTracker builds one entry per engine, in the shared table's order, so
// an engine added there appears here rather than being silently absent from
// every screen that reads a facade's port.
func newProxyTracker() *proxyTracker {
	all := engines.All()
	out := make([]*proxyEngine, 0, len(all))
	for _, e := range all {
		out = append(out, &proxyEngine{
			label:  e.DisplayName,
			prefix: e.ComponentName(),
			engine: e.Name,
		})
	}
	return &proxyTracker{engines: out}
}

// indexForEngine finds the proxy fronting an engine-manager engine, or -1. Each
// proxy serves exactly one engine type, which is what lets a node's engine list
// show the client-facing port beside each engine's own.
func (p *proxyTracker) indexForEngine(engine string) int {
	want := strings.ToLower(strings.TrimSpace(engine))
	for i, e := range p.engines {
		if e.engine == want {
			return i
		}
	}
	return -1
}

// portForEngine is the listen port of the proxy fronting an engine, and whether
// that proxy is up. A port with the proxy down is not an endpoint a client can
// use, so callers distinguish the two.
func (p *proxyTracker) portForEngine(engine string) (int, bool) {
	idx := p.indexForEngine(engine)
	if idx < 0 {
		return 0, false
	}
	return p.engines[idx].port, p.engines[idx].ready
}

// init subscribes to every engine's facade and fetches its current status.
// Subscribing is idempotent at the broker, so several consumers may each hold a
// tracker.
func (p *proxyTracker) init(client *rpc.Client) tea.Cmd {
	cmds := make([]tea.Cmd, 0, len(p.engines)*2)
	for i, e := range p.engines {
		method := e.prefix + ":subscribe"
		cmds = append(cmds,
			// Logged rather than shown: without the subscription the strip
			// still follows the broker through the periodic status read, only
			// more slowly, so there is nothing for the operator to do about it.
			call(client, method, nil, func(_ *rpc.Message, err error) tea.Msg {
				if err != nil {
					slog.Warn("proxy status subscription failed; relying on polling",
						"method", method, "err", err)
				}
				return nil
			}),
			p.statusCmd(client, i),
		)
	}
	return tea.Batch(cmds...)
}

func (p *proxyTracker) statusCmd(client *rpc.Client, idx int) tea.Cmd {
	e := p.engines[idx]
	return call(client, e.prefix+":get-status", nil, func(msg *rpc.Message, err error) tea.Msg {
		if err != nil {
			return proxyStatusMsg{idx: idx, err: err}
		}
		var r struct {
			Ready bool `json:"ready"`
			Port  int  `json:"port"`
		}
		decodeOrLog(e.prefix+":get-status", msg.Result, &r)
		return proxyStatusMsg{idx: idx, ready: r.Ready, port: r.Port}
	})
}

// A proxy's listen port is not changed from here. It is one of the three
// fields engine:apply-settings writes together against a revision, so it goes
// through the node detail's settings editor with the server port and the launch
// arguments. This tracker only reads: readiness, and the port in force.

// apply folds a status reply into the tracker.
//
// A failed read marks the facade down. It is not known to be listening, and
// the strip exists to tell clients where they can connect.
//
// The last known port is kept whenever the facade is down, from a failed read
// or from the broker's {ready:false, port:0} for a facade it is not running,
// as the error push keeps it: the configured port, shown as down, says more
// than a blank.
func (p *proxyTracker) apply(msg proxyStatusMsg) {
	if msg.idx < 0 || msg.idx >= len(p.engines) {
		return
	}
	e := p.engines[msg.idx]
	if msg.err != nil {
		e.ready = false
		return
	}
	e.ready = msg.ready
	if msg.port != 0 {
		e.port = msg.port
	}
}

// handleNotification consumes a proxy push. A ready frame carries the port
// directly; an error frame takes the proxy down.
//
// Both directions are handled deliberately. Readiness only ever being set meant
// a proxy that died stayed green with its old port on screen, pointing clients
// at an endpoint that had stopped listening — the one thing this strip exists to
// tell them.
//
// Pushes and the periodic status read describe the same thing: the broker's
// record of each facade, which it updates from the facade's own ready
// announcement. The pushes make a change visible at once; the read is there
// because a crashed proxy announces nothing. Whichever arrives later wins.
//
// The broker forwards every facade's pushes under that facade's own prefix, so
// the facade is found by prefix rather than by a position in the table.
func (p *proxyTracker) handleNotification(msg *rpc.Message) {
	idx := -1
	for i, e := range p.engines {
		if strings.HasPrefix(msg.Method, e.prefix+":") {
			idx = i
			break
		}
	}
	if idx < 0 {
		return
	}
	switch {
	case strings.HasSuffix(msg.Method, ":ready"):
		var r struct {
			Port int `json:"port"`
		}
		decodeOrLog(msg.Method, msg.Params, &r)
		p.engines[idx].ready = true
		if r.Port != 0 {
			p.engines[idx].port = r.Port
		}
	case strings.HasSuffix(msg.Method, ":error"):
		// The port is kept: it is still the configured value, and showing the
		// proxy as down at a known port is more useful than blanking it.
		p.engines[idx].ready = false
	}
}

// refreshCmd re-reads every facade's status.
//
// Polled on the view's tick as well as pushed, because a proxy going away does
// not always announce it — a crash, or a broker restart, produces no error frame
// — and a readiness strip that can only be corrected by a push stays wrong
// indefinitely.
func (p *proxyTracker) refreshCmd(client *rpc.Client) tea.Cmd {
	cmds := make([]tea.Cmd, 0, len(p.engines))
	for i := range p.engines {
		cmds = append(cmds, p.statusCmd(client, i))
	}
	return tea.Batch(cmds...)
}

// strip is the one-line summary of where local clients should point, and whether
// anything is listening. Routing mode is stated because it is not adjustable:
// the scheduler and proxies own placement.
func (p *proxyTracker) strip() string {
	parts := make([]string, 0, len(p.engines))
	for _, e := range p.engines {
		if e.ready {
			parts = append(parts, fmt.Sprintf("%s %s", e.label, statusOKStyle.Render(fmt.Sprintf(":%d", e.port))))
			continue
		}
		parts = append(parts, fmt.Sprintf("%s %s", e.label, statusErrStyle.Render("down")))
	}
	return strings.Join(parts, "   ") + footerStyle.Render("   routing=automatic")
}