Skip to content

Agent Mesh

feat(node): add OpenAI-compatible facade and inference engine connector

Antonio Ojead9524a7a57b3
diff --git a/internal/node/inference_engine.go b/internal/node/inference_engine.go
new file mode 100644
index 00000000..4a25d795
--- /dev/null
+++ b/internal/node/inference_engine.go
@@ -0,0 +1,83 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package node
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "io"
+ "net/http"
+ "net/url"
+)
+
+// InferenceEngine is the connector between SAM and an inference backend.
+// Implementations adapt one backend family; the service and facade layers
+// stay backend-agnostic.
+type InferenceEngine interface {
+ // Models returns the model IDs the backend currently serves.
+ Models(ctx context.Context) ([]string, error)
+}
+
+// openAIEngine adapts any OpenAI-compatible HTTP backend
+// (vLLM, ollama, llama.cpp, MaaS providers).
+type openAIEngine struct {
+ baseURL *url.URL
+ client *http.Client
+}
+
+func newOpenAIEngine(baseURL *url.URL, client *http.Client) *openAIEngine {
+ if client == nil {
+ client = http.DefaultClient
+ }
+ return &openAIEngine{baseURL: baseURL, client: client}
+}
+
+func (e *openAIEngine) Models(ctx context.Context) ([]string, error) {
+ u := *e.baseURL
+ u.Path = singleJoiningSlash(e.baseURL.Path, "/v1/models")
+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil)
+ if err != nil {
+ return nil, err
+ }
+ resp, err := e.client.Do(req)
+ if err != nil {
+ return nil, err
+ }
+ defer func() { _ = resp.Body.Close() }()
+ if resp.StatusCode != http.StatusOK {
+ return nil, fmt.Errorf("backend %s returned status %d for /v1/models", e.baseURL.Host, resp.StatusCode)
+ }
+ return decodeModelIDs(resp.Body)
+}
+
+// decodeModelIDs parses an OpenAI-style /v1/models list response.
+func decodeModelIDs(r io.Reader) ([]string, error) {
+ var list struct {
+ Data []struct {
+ ID string `json:"id"`
+ } `json:"data"`
+ }
+ if err := json.NewDecoder(r).Decode(&list); err != nil {
+ return nil, fmt.Errorf("invalid models response: %w", err)
+ }
+ ids := make([]string, 0, len(list.Data))
+ for _, m := range list.Data {
+ if m.ID != "" {
+ ids = append(ids, m.ID)
+ }
+ }
+ return ids, nil
+}
diff --git a/internal/node/inference_engine_test.go b/internal/node/inference_engine_test.go
new file mode 100644
index 00000000..c26a0543
--- /dev/null
+++ b/internal/node/inference_engine_test.go
@@ -0,0 +1,140 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package node
+
+import (
+ "context"
+ "net/http"
+ "net/http/httptest"
+ "net/url"
+ "reflect"
+ "testing"
+
+ "github.com/google/sam/api"
+)
+
+func TestOpenAIEngine_Models(t *testing.T) {
+ tests := []struct {
+ name string
+ basePath string
+ status int
+ body string
+ want []string
+ wantErr bool
+ wantedPath string
+ }{
+ {
+ name: "plain backend",
+ status: http.StatusOK,
+ body: `{"object":"list","data":[{"id":"llama-3"},{"id":"qwen-2.5"}]}`,
+ want: []string{"llama-3", "qwen-2.5"},
+ wantedPath: "/v1/models",
+ },
+ {
+ name: "backend with path prefix",
+ basePath: "/proxy",
+ status: http.StatusOK,
+ body: `{"data":[{"id":"m1"}]}`,
+ want: []string{"m1"},
+ wantedPath: "/proxy/v1/models",
+ },
+ {
+ name: "backend error status",
+ status: http.StatusInternalServerError,
+ body: `boom`,
+ wantErr: true,
+ },
+ {
+ name: "invalid JSON",
+ status: http.StatusOK,
+ body: `not-json`,
+ wantErr: true,
+ },
+ {
+ name: "empty and missing IDs skipped",
+ status: http.StatusOK,
+ body: `{"data":[{"id":""},{"id":"m2"},{}]}`,
+ want: []string{"m2"},
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ var gotPath string
+ backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ gotPath = r.URL.Path
+ w.WriteHeader(tt.status)
+ _, _ = w.Write([]byte(tt.body))
+ }))
+ defer backend.Close()
+
+ u, err := url.Parse(backend.URL + tt.basePath)
+ if err != nil {
+ t.Fatalf("parse URL: %v", err)
+ }
+ engine := newOpenAIEngine(u, nil)
+ got, err := engine.Models(context.Background())
+ if tt.wantErr {
+ if err == nil {
+ t.Fatal("Models: expected error, got nil")
+ }
+ return
+ }
+ if err != nil {
+ t.Fatalf("Models: %v", err)
+ }
+ if !reflect.DeepEqual(got, tt.want) {
+ t.Errorf("Models: got %v, want %v", got, tt.want)
+ }
+ if tt.wantedPath != "" && gotPath != tt.wantedPath {
+ t.Errorf("backend path: got %q, want %q", gotPath, tt.wantedPath)
+ }
+ })
+ }
+}
+
+func TestInferenceService_Models_ProbesBackend(t *testing.T) {
+ backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.URL.Path != "/v1/models" {
+ http.NotFound(w, r)
+ return
+ }
+ _, _ = w.Write([]byte(`{"data":[{"id":"gemma-3"}]}`))
+ }))
+ defer backend.Close()
+
+ svc := &InferenceService{baseService: baseService{
+ info: &api.ServiceInfo{Type: api.ServiceType_SERVICE_TYPE_INFERENCE, Name: "llm"},
+ backend: &api.RegisterServiceRequest_TargetUrl{TargetUrl: backend.URL},
+ }}
+ if err := svc.Init(context.Background()); err != nil {
+ t.Fatalf("Init: %v", err)
+ }
+ got, err := svc.Models(context.Background())
+ if err != nil {
+ t.Fatalf("Models: %v", err)
+ }
+ if want := []string{"gemma-3"}; !reflect.DeepEqual(got, want) {
+ t.Errorf("Models: got %v, want %v", got, want)
+ }
+}
+
+func TestInferenceService_Models_Uninitialized(t *testing.T) {
+ svc := &InferenceService{baseService: baseService{
+ info: &api.ServiceInfo{Type: api.ServiceType_SERVICE_TYPE_INFERENCE, Name: "llm"},
+ }}
+ if _, err := svc.Models(context.Background()); err == nil {
+ t.Fatal("Models: expected error for uninitialized service, got nil")
+ }
+}
diff --git a/internal/node/inference_service.go b/internal/node/inference_service.go
index 6d0a359a..4da3f8a6 100644
--- a/internal/node/inference_service.go
+++ b/internal/node/inference_service.go
@@ -36,6 +36,7 @@ import (
type InferenceService struct {
baseService
backendURL *url.URL
+ engine InferenceEngine
}
func (s *InferenceService) Init(ctx context.Context) error {
@@ -46,6 +47,7 @@ func (s *InferenceService) Init(ctx context.Context) error {
return fmt.Errorf("invalid inference backend URL %q: %w", x.TargetUrl, err)
}
s.backendURL = u
+ s.engine = newOpenAIEngine(u, nil)
s.handler = s.newInferenceProxy()
case *api.RegisterServiceRequest_Command:
return fmt.Errorf("command-based backends are not supported for InferenceService")
@@ -55,6 +57,14 @@ func (s *InferenceService) Init(ctx context.Context) error {
return nil
}
+// Models probes the backend connector for the model IDs it serves.
+func (s *InferenceService) Models(ctx context.Context) ([]string, error) {
+ if s.engine == nil {
+ return nil, fmt.Errorf("inference service %q not initialized", s.info.GetName())
+ }
+ return s.engine.Models(ctx)
+}
+
func (s *InferenceService) newInferenceProxy() http.Handler {
return &httputil.ReverseProxy{
Director: func(req *http.Request) {
diff --git a/internal/node/openai_facade.go b/internal/node/openai_facade.go
new file mode 100644
index 00000000..d583f4e8
--- /dev/null
+++ b/internal/node/openai_facade.go
@@ -0,0 +1,342 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package node
+
+import (
+ "bytes"
+ "context"
+ "encoding/base64"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io"
+ "net/http"
+ "sort"
+ "sync"
+ "sync/atomic"
+ "time"
+
+ "github.com/google/sam/api"
+ libp2phttp "github.com/libp2p/go-libp2p-http"
+ "github.com/libp2p/go-libp2p/core/network"
+ "github.com/libp2p/go-libp2p/core/peer"
+)
+
+const (
+ // modelRegistryTTL bounds how stale the mesh-wide model view can be.
+ modelRegistryTTL = 30 * time.Second
+ // remoteModelProbeTimeout bounds a single provider /v1/models probe.
+ remoteModelProbeTimeout = 10 * time.Second
+ // remoteProbeConcurrency bounds parallel provider probes during refresh.
+ remoteProbeConcurrency = 10
+ // maxInferenceBodyBytes caps buffered completion request bodies.
+ maxInferenceBodyBytes = 16 << 20 // 16 MiB
+)
+
+// modelLister is implemented by local services that can report served models.
+type modelLister interface {
+ Models(ctx context.Context) ([]string, error)
+}
+
+// modelProvider locates one service able to serve a model.
+// An empty peerID means the service runs on this node.
+type modelProvider struct {
+ peerID string
+ service string
+}
+
+// openAIFacade exposes an OpenAI-compatible surface on the sidecar
+// (/v1/models, /v1/chat/completions, /v1/completions): point any OpenAI SDK
+// at the sidecar and the facade resolves the requested model to a local or
+// mesh provider, preferring local. Function fields are seams so unit tests
+// can fake discovery and probing.
+type openAIFacade struct {
+ forward http.Handler // egress proxy handling /sam/<peer>/<type>/<name>/...
+ localPeerID func() string
+ localServices func() []Service
+ discover func(ctx context.Context) ([]*api.DiscoveredProvider, error)
+ remoteModels func(ctx context.Context, peerID, srvName string) ([]string, error)
+
+ ttl time.Duration
+ mu sync.Mutex
+ models map[string][]modelProvider
+ expires time.Time
+}
+
+func newOpenAIFacade(node *SamNode, egress http.Handler) *openAIFacade {
+ client := &http.Client{Transport: libp2phttp.NewTransport(node.Host)}
+ return &openAIFacade{
+ forward: egress,
+ ttl: modelRegistryTTL,
+ localPeerID: func() string { return node.Host.ID().String() },
+ localServices: func() []Service {
+ var out []Service
+ for _, info := range node.ListLocalServices(api.ServiceType_SERVICE_TYPE_INFERENCE) {
+ if svc, ok := node.services.Get(info.GetName()); ok {
+ out = append(out, svc)
+ }
+ }
+ return out
+ },
+ discover: func(ctx context.Context) ([]*api.DiscoveredProvider, error) {
+ return node.DiscoverRemoteServices(ctx, api.ServiceType_SERVICE_TYPE_INFERENCE, "")
+ },
+ remoteModels: func(ctx context.Context, peerID, srvName string) ([]string, error) {
+ return fetchRemoteModels(ctx, node, client, peerID, srvName)
+ },
+ }
+}
+
+// fetchRemoteModels probes a remote provider's backend /v1/models through its
+// libp2p ingress, authenticating with this node's biscuit.
+func fetchRemoteModels(ctx context.Context, node *SamNode, client *http.Client, peerID, srvName string) ([]string, error) {
+ pid, err := peer.Decode(peerID)
+ if err != nil {
+ return nil, fmt.Errorf("invalid peer ID %q: %w", peerID, err)
+ }
+ identity := node.GetIdentity()
+ if identity == nil {
+ return nil, fmt.Errorf("missing node identity")
+ }
+ ctx, cancel := context.WithTimeout(network.WithAllowLimitedConn(ctx, "openai-facade"), remoteModelProbeTimeout)
+ defer cancel()
+ if cond := node.Host.Network().Connectedness(pid); cond != network.Connected && cond != network.Limited {
+ node.preparePeerAddrs(ctx, pid)
+ }
+ u := fmt.Sprintf("libp2p://%s/%s/%s/v1/models", peerID, api.ServiceTypeStringInference, srvName)
+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil)
+ if err != nil {
+ return nil, err
+ }
+ req.Header.Set(api.HeaderSamBiscuit, base64.StdEncoding.EncodeToString(identity))
+ resp, err := client.Do(req)
+ if err != nil {
+ return nil, err
+ }
+ defer func() { _ = resp.Body.Close() }()
+ if resp.StatusCode != http.StatusOK {
+ return nil, fmt.Errorf("provider %s/%s returned status %d for /v1/models", peerID, srvName, resp.StatusCode)
+ }
+ return decodeModelIDs(resp.Body)
+}
+
+// providersFor resolves a model to its providers, force-refreshing once on a
+// miss so newly appeared providers are usable before the TTL lapses.
+func (f *openAIFacade) providersFor(ctx context.Context, model string) []modelProvider {
+ f.mu.Lock()
+ defer f.mu.Unlock()
+ refreshed := false
+ if f.models == nil || time.Now().After(f.expires) {
+ f.refreshLocked(ctx)
+ refreshed = true
+ }
+ if providers := f.models[model]; len(providers) > 0 || refreshed {
+ return providers
+ }
+ f.refreshLocked(ctx)
+ return f.models[model]
+}
+
+// modelsView returns the current model registry, refreshing when stale. The
+// returned map is a snapshot: refreshes swap the pointer, never mutate.
+func (f *openAIFacade) modelsView(ctx context.Context) map[string][]modelProvider {
+ f.mu.Lock()
+ defer f.mu.Unlock()
+ if f.models == nil || time.Now().After(f.expires) {
+ f.refreshLocked(ctx)
+ }
+ return f.models
+}
+
+func (f *openAIFacade) refreshLocked(ctx context.Context) {
+ f.models = f.collectModels(ctx)
+ f.expires = time.Now().Add(f.ttl)
+}
+
+// collectModels aggregates model IDs from local services (first, so they are
+// preferred) and mesh providers. Failed probes drop the provider with a log
+// line rather than failing the whole view.
+func (f *openAIFacade) collectModels(ctx context.Context) map[string][]modelProvider {
+ models := map[string][]modelProvider{}
+ for _, svc := range f.localServices() {
+ lister, ok := svc.(modelLister)
+ if !ok {
+ continue
+ }
+ ids, err := lister.Models(ctx)
+ if err != nil {
+ logger.Warnf("[OpenAIFacade] local model probe for %q failed: %v", svc.Info().GetName(), err)
+ continue
+ }
+ for _, id := range ids {
+ models[id] = append(models[id], modelProvider{service: svc.Info().GetName()})
+ }
+ }
+
+ providers, err := f.discover(ctx)
+ if err != nil {
+ logger.Warnf("[OpenAIFacade] mesh discovery failed: %v", err)
+ return models
+ }
+ var mu sync.Mutex
+ var wg sync.WaitGroup
+ sem := make(chan struct{}, remoteProbeConcurrency)
+ for _, p := range providers {
+ wg.Add(1)
+ go func(peerID, srvName string) {
+ defer wg.Done()
+ select {
+ case sem <- struct{}{}:
+ defer func() { <-sem }()
+ case <-ctx.Done():
+ return
+ }
+ ids, err := f.remoteModels(ctx, peerID, srvName)
+ if err != nil {
+ logger.Warnf("[OpenAIFacade] model probe for %s/%s failed: %v", peerID, srvName, err)
+ return
+ }
+ mu.Lock()
+ for _, id := range ids {
+ models[id] = append(models[id], modelProvider{peerID: peerID, service: srvName})
+ }
+ mu.Unlock()
+ }(p.GetPeerId(), p.GetSrvName())
+ }
+ wg.Wait()
+ return models
+}
+
+// facadeRR spreads load across remote providers of the same model.
+var facadeRR atomic.Uint64
+
+// pickProvider prefers a local provider (locals sort first in collectModels),
+// otherwise round-robins across remotes.
+func pickProvider(providers []modelProvider) modelProvider {
+ if providers[0].peerID == "" {
+ return providers[0]
+ }
+ return providers[facadeRR.Add(1)%uint64(len(providers))]
+}
+
+func (f *openAIFacade) handleModels(w http.ResponseWriter, r *http.Request) {
+ if r.Method != http.MethodGet {
+ writeOpenAIError(w, http.StatusMethodNotAllowed, "method_not_allowed", "only GET is supported")
+ return
+ }
+ type openAIModel struct {
+ ID string `json:"id"`
+ Object string `json:"object"`
+ Created int64 `json:"created"`
+ OwnedBy string `json:"owned_by"`
+ }
+ view := f.modelsView(r.Context())
+ data := make([]openAIModel, 0, len(view))
+ for id, providers := range view {
+ ownedBy := "local"
+ if providers[0].peerID != "" {
+ ownedBy = providers[0].peerID
+ }
+ data = append(data, openAIModel{ID: id, Object: "model", OwnedBy: ownedBy})
+ }
+ sort.Slice(data, func(i, j int) bool { return data[i].ID < data[j].ID })
+
+ w.Header().Set("Content-Type", "application/json")
+ if err := json.NewEncoder(w).Encode(map[string]any{"object": "list", "data": data}); err != nil {
+ logger.Errorf("[OpenAIFacade] failed to encode models response: %v", err)
+ }
+}
+
+func (f *openAIFacade) handleCompletions(w http.ResponseWriter, r *http.Request) {
+ if r.Method != http.MethodPost {
+ writeOpenAIError(w, http.StatusMethodNotAllowed, "method_not_allowed", "only POST is supported")
+ return
+ }
+ body, err := io.ReadAll(http.MaxBytesReader(w, r.Body, maxInferenceBodyBytes))
+ if err != nil {
+ var maxErr *http.MaxBytesError
+ if errors.As(err, &maxErr) {
+ writeOpenAIError(w, http.StatusRequestEntityTooLarge, "request_too_large", "request body exceeds limit")
+ return
+ }
+ writeOpenAIError(w, http.StatusBadRequest, "invalid_request", "failed to read request body")
+ return
+ }
+ var req struct {
+ Model string `json:"model"`
+ }
+ if err := json.Unmarshal(body, &req); err != nil {
+ writeOpenAIError(w, http.StatusBadRequest, "invalid_request", "request body is not valid JSON")
+ return
+ }
+ if req.Model == "" {
+ writeOpenAIError(w, http.StatusBadRequest, "invalid_request", "missing required field: model")
+ return
+ }
+
+ providers := f.providersFor(r.Context(), req.Model)
+ if len(providers) == 0 {
+ writeOpenAIError(w, http.StatusNotFound, "model_not_found",
+ fmt.Sprintf("model %q is not served by any provider on the mesh", req.Model))
+ return
+ }
+ p := pickProvider(providers)
+ logger.Debugf("[OpenAIFacade] model %q -> provider peer=%q service=%q (%d candidates)",
+ req.Model, p.peerID, p.service, len(providers))
+
+ // The gate already stripped the credential it consumed; a surviving
+ // Authorization header is the destination backend's own credential and
+ // passes through, mirroring the egress path. X-Sam-Authentication is
+ // local-only and must never travel off-node.
+ r.Header.Del(api.HeaderSamAuthentication)
+ r.Body = io.NopCloser(bytes.NewReader(body))
+ r.ContentLength = int64(len(body))
+
+ if p.peerID == "" {
+ f.serveLocal(w, r, p.service)
+ return
+ }
+ r.URL.Path = fmt.Sprintf("/sam/%s/%s/%s%s", p.peerID, api.ServiceTypeStringInference, p.service, r.URL.Path)
+ r.URL.RawPath = ""
+ f.forward.ServeHTTP(w, r)
+}
+
+func (f *openAIFacade) serveLocal(w http.ResponseWriter, r *http.Request, serviceName string) {
+ for _, svc := range f.localServices() {
+ if svc.Info().GetName() != serviceName || svc.Handler() == nil {
+ continue
+ }
+ // Attribute locally served tokens to this node in usage metrics.
+ if id := f.localPeerID(); id != "" {
+ r.Header.Set("X-Peer-Id", id)
+ }
+ svc.Handler().ServeHTTP(w, r)
+ return
+ }
+ writeOpenAIError(w, http.StatusServiceUnavailable, "service_unavailable",
+ fmt.Sprintf("local service %q is no longer available", serviceName))
+}
+
+// writeOpenAIError emits an OpenAI-style error body so SDKs surface it cleanly.
+func writeOpenAIError(w http.ResponseWriter, status int, code, msg string) {
+ w.Header().Set("Content-Type", "application/json")
+ w.WriteHeader(status)
+ err := json.NewEncoder(w).Encode(map[string]any{
+ "error": map[string]any{"message": msg, "type": "invalid_request_error", "code": code},
+ })
+ if err != nil {
+ logger.Errorf("[OpenAIFacade] failed to encode error response: %v", err)
+ }
+}
diff --git a/internal/node/openai_facade_test.go b/internal/node/openai_facade_test.go
new file mode 100644
index 00000000..c4b559dc
--- /dev/null
+++ b/internal/node/openai_facade_test.go
@@ -0,0 +1,321 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package node
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "io"
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/google/sam/api"
+)
+
+// fakeModelService is a local Service that also reports served models.
+type fakeModelService struct {
+ testService
+ models []string
+ err error
+}
+
+func (f *fakeModelService) Models(_ context.Context) ([]string, error) {
+ return f.models, f.err
+}
+
+func newFakeModelService(name string, handler http.Handler, models ...string) *fakeModelService {
+ return &fakeModelService{
+ testService: testService{
+ info: &api.ServiceInfo{Type: api.ServiceType_SERVICE_TYPE_INFERENCE, Name: name},
+ handler: handler,
+ },
+ models: models,
+ }
+}
+
+// newTestFacade builds a facade with inert seams; tests override as needed.
+func newTestFacade() *openAIFacade {
+ return &openAIFacade{
+ ttl: time.Minute,
+ localPeerID: func() string { return "" },
+ localServices: func() []Service { return nil },
+ discover: func(_ context.Context) ([]*api.DiscoveredProvider, error) {
+ return nil, nil
+ },
+ remoteModels: func(_ context.Context, _, _ string) ([]string, error) {
+ return nil, nil
+ },
+ }
+}
+
+func decodeModelsResponse(t *testing.T, body io.Reader) map[string]string {
+ t.Helper()
+ var resp struct {
+ Object string `json:"object"`
+ Data []struct {
+ ID string `json:"id"`
+ OwnedBy string `json:"owned_by"`
+ } `json:"data"`
+ }
+ if err := json.NewDecoder(body).Decode(&resp); err != nil {
+ t.Fatalf("decode models response: %v", err)
+ }
+ if resp.Object != "list" {
+ t.Errorf("object: got %q, want %q", resp.Object, "list")
+ }
+ out := map[string]string{}
+ for _, m := range resp.Data {
+ out[m.ID] = m.OwnedBy
+ }
+ return out
+}
+
+func TestFacade_HandleModels_AggregatesLocalAndRemote(t *testing.T) {
+ f := newTestFacade()
+ f.localServices = func() []Service {
+ return []Service{newFakeModelService("local-llm", nil, "local-model", "shared-model")}
+ }
+ f.discover = func(_ context.Context) ([]*api.DiscoveredProvider, error) {
+ return []*api.DiscoveredProvider{
+ {PeerId: "peerA", SrvName: "srvA"},
+ {PeerId: "peerB", SrvName: "srvB"},
+ }, nil
+ }
+ f.remoteModels = func(_ context.Context, peerID, _ string) ([]string, error) {
+ if peerID == "peerA" {
+ return []string{"remote-model", "shared-model"}, nil
+ }
+ return nil, fmt.Errorf("unreachable")
+ }
+
+ rec := httptest.NewRecorder()
+ f.handleModels(rec, httptest.NewRequest(http.MethodGet, "/v1/models", nil))
+
+ if rec.Code != http.StatusOK {
+ t.Fatalf("status: got %d, want %d", rec.Code, http.StatusOK)
+ }
+ got := decodeModelsResponse(t, rec.Body)
+ if len(got) != 3 {
+ t.Fatalf("models: got %v, want 3 entries", got)
+ }
+ if got["local-model"] != "local" {
+ t.Errorf("local-model owned_by: got %q, want %q", got["local-model"], "local")
+ }
+ // Local providers sort first, so a model served both locally and remotely
+ // is reported as local.
+ if got["shared-model"] != "local" {
+ t.Errorf("shared-model owned_by: got %q, want %q", got["shared-model"], "local")
+ }
+ if got["remote-model"] != "peerA" {
+ t.Errorf("remote-model owned_by: got %q, want %q", got["remote-model"], "peerA")
+ }
+}
+
+func TestFacade_HandleModels_MethodNotAllowed(t *testing.T) {
+ f := newTestFacade()
+ rec := httptest.NewRecorder()
+ f.handleModels(rec, httptest.NewRequest(http.MethodPost, "/v1/models", nil))
+ if rec.Code != http.StatusMethodNotAllowed {
+ t.Fatalf("status: got %d, want %d", rec.Code, http.StatusMethodNotAllowed)
+ }
+}
+
+func TestFacade_Completions_ForwardsToRemote(t *testing.T) {
+ f := newTestFacade()
+ f.discover = func(_ context.Context) ([]*api.DiscoveredProvider, error) {
+ return []*api.DiscoveredProvider{{PeerId: "peerA", SrvName: "srvA"}}, nil
+ }
+ f.remoteModels = func(_ context.Context, _, _ string) ([]string, error) {
+ return []string{"m1"}, nil
+ }
+ var gotPath, gotBody, gotAuth, gotSamAuth string
+ f.forward = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ gotPath = r.URL.Path
+ b, _ := io.ReadAll(r.Body)
+ gotBody = string(b)
+ gotAuth = r.Header.Get("Authorization")
+ gotSamAuth = r.Header.Get(api.HeaderSamAuthentication)
+ w.WriteHeader(http.StatusOK)
+ })
+
+ body := `{"model":"m1","messages":[{"role":"user","content":"hi"}]}`
+ req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(body))
+ // Post-gate state: the gate stripped the credential it consumed; a
+ // surviving Authorization header is the backend's own credential.
+ req.Header.Set("Authorization", "Bearer backend-credential")
+ req.Header.Set(api.HeaderSamAuthentication, "Bearer sidecar-token")
+ rec := httptest.NewRecorder()
+ f.handleCompletions(rec, req)
+
+ if rec.Code != http.StatusOK {
+ t.Fatalf("status: got %d, want %d", rec.Code, http.StatusOK)
+ }
+ if want := "/sam/peerA/inference/srvA/v1/chat/completions"; gotPath != want {
+ t.Errorf("forwarded path: got %q, want %q", gotPath, want)
+ }
+ if gotBody != body {
+ t.Errorf("forwarded body: got %q, want %q", gotBody, body)
+ }
+ if gotSamAuth != "" {
+ t.Errorf("%s must never travel off-node, got %q", api.HeaderSamAuthentication, gotSamAuth)
+ }
+ if gotAuth != "Bearer backend-credential" {
+ t.Errorf("backend Authorization must pass through, got %q", gotAuth)
+ }
+}
+
+func TestFacade_Completions_LegacyCompletionsPath(t *testing.T) {
+ f := newTestFacade()
+ f.discover = func(_ context.Context) ([]*api.DiscoveredProvider, error) {
+ return []*api.DiscoveredProvider{{PeerId: "peerA", SrvName: "srvA"}}, nil
+ }
+ f.remoteModels = func(_ context.Context, _, _ string) ([]string, error) {
+ return []string{"m1"}, nil
+ }
+ var gotPath string
+ f.forward = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ gotPath = r.URL.Path
+ })
+
+ req := httptest.NewRequest(http.MethodPost, "/v1/completions", strings.NewReader(`{"model":"m1","prompt":"hi"}`))
+ f.handleCompletions(httptest.NewRecorder(), req)
+
+ if want := "/sam/peerA/inference/srvA/v1/completions"; gotPath != want {
+ t.Errorf("forwarded path: got %q, want %q", gotPath, want)
+ }
+}
+
+func TestFacade_Completions_PrefersLocal(t *testing.T) {
+ f := newTestFacade()
+ localHit := false
+ var gotPeerHeader string
+ local := newFakeModelService("local-llm", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ localHit = true
+ gotPeerHeader = r.Header.Get("X-Peer-Id")
+ w.WriteHeader(http.StatusOK)
+ }), "m1")
+ f.localPeerID = func() string { return "selfPeer" }
+ f.localServices = func() []Service { return []Service{local} }
+ f.discover = func(_ context.Context) ([]*api.DiscoveredProvider, error) {
+ return []*api.DiscoveredProvider{{PeerId: "peerA", SrvName: "srvA"}}, nil
+ }
+ f.remoteModels = func(_ context.Context, _, _ string) ([]string, error) {
+ return []string{"m1"}, nil
+ }
+ f.forward = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ t.Error("remote forward must not be used when a local provider exists")
+ })
+
+ req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{"model":"m1"}`))
+ rec := httptest.NewRecorder()
+ f.handleCompletions(rec, req)
+
+ if !localHit {
+ t.Fatal("local service was not invoked")
+ }
+ if gotPeerHeader != "selfPeer" {
+ t.Errorf("X-Peer-Id: got %q, want %q", gotPeerHeader, "selfPeer")
+ }
+}
+
+func TestFacade_Completions_UnknownModel(t *testing.T) {
+ f := newTestFacade()
+ req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{"model":"nope"}`))
+ rec := httptest.NewRecorder()
+ f.handleCompletions(rec, req)
+ if rec.Code != http.StatusNotFound {
+ t.Fatalf("status: got %d, want %d", rec.Code, http.StatusNotFound)
+ }
+ if !strings.Contains(rec.Body.String(), "model_not_found") {
+ t.Errorf("body missing model_not_found code: %s", rec.Body.String())
+ }
+}
+
+func TestFacade_Completions_BadRequests(t *testing.T) {
+ tests := []struct {
+ name string
+ method string
+ body string
+ want int
+ }{
+ {"missing model", http.MethodPost, `{"messages":[]}`, http.StatusBadRequest},
+ {"invalid JSON", http.MethodPost, `{`, http.StatusBadRequest},
+ {"wrong method", http.MethodGet, ``, http.StatusMethodNotAllowed},
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ f := newTestFacade()
+ req := httptest.NewRequest(tt.method, "/v1/chat/completions", strings.NewReader(tt.body))
+ rec := httptest.NewRecorder()
+ f.handleCompletions(rec, req)
+ if rec.Code != tt.want {
+ t.Fatalf("status: got %d, want %d", rec.Code, tt.want)
+ }
+ })
+ }
+}
+
+func TestFacade_RegistryCacheAndForcedRefresh(t *testing.T) {
+ f := newTestFacade()
+ discoverCalls := 0
+ f.discover = func(_ context.Context) ([]*api.DiscoveredProvider, error) {
+ discoverCalls++
+ return []*api.DiscoveredProvider{{PeerId: "peerA", SrvName: "srvA"}}, nil
+ }
+ f.remoteModels = func(_ context.Context, _, _ string) ([]string, error) {
+ return []string{"m1"}, nil
+ }
+
+ ctx := context.Background()
+ // First resolution populates the cache.
+ if got := f.providersFor(ctx, "m1"); len(got) != 1 {
+ t.Fatalf("providersFor(m1): got %v, want 1 provider", got)
+ }
+ if discoverCalls != 1 {
+ t.Fatalf("discover calls after first resolve: got %d, want 1", discoverCalls)
+ }
+ // Cached hit within TTL: no new discovery.
+ f.providersFor(ctx, "m1")
+ if discoverCalls != 1 {
+ t.Fatalf("discover calls after cached resolve: got %d, want 1", discoverCalls)
+ }
+ // Unknown model on a fresh cache forces one refresh.
+ f.providersFor(ctx, "m2")
+ if discoverCalls != 2 {
+ t.Fatalf("discover calls after unknown-model resolve: got %d, want 2", discoverCalls)
+ }
+}
+
+func TestPickProvider(t *testing.T) {
+ local := modelProvider{service: "local-llm"}
+ remoteA := modelProvider{peerID: "peerA", service: "srvA"}
+ remoteB := modelProvider{peerID: "peerB", service: "srvB"}
+
+ if got := pickProvider([]modelProvider{local, remoteA}); got != local {
+ t.Errorf("pickProvider with local first: got %+v, want local", got)
+ }
+
+ seen := map[string]bool{}
+ for range 10 {
+ seen[pickProvider([]modelProvider{remoteA, remoteB}).peerID] = true
+ }
+ if !seen["peerA"] || !seen["peerB"] {
+ t.Errorf("round-robin should hit both remotes, got %v", seen)
+ }
+}
diff --git a/internal/node/sidecar.go b/internal/node/sidecar.go
index 43ceec6a..6416a2b6 100644
--- a/internal/node/sidecar.go
+++ b/internal/node/sidecar.go
@@ -61,7 +61,18 @@ func StartSidecarServer(node *SamNode, addr, token, certFile, keyFile, caFile st
// Mount Egress Proxy. allowAuthorizationFallback=false is required here: this
// handler forwards Authorization to the destination service, so it must never
// also accept it as the local gate credential (would leak the sidecar token off-node).
- mux.Handle("/sam/", withAuth(token, false, withMeshConnection(node, createEgressProxy(node))))
+ egress := createEgressProxy(node)
+ mux.Handle("/sam/", withAuth(token, false, withMeshConnection(node, egress)))
+
+ // OpenAI-compatible facade: point any OpenAI SDK at the sidecar.
+ // allowAuthorizationFallback=true lets SDKs send the sidecar token as their
+ // api_key; withAuth strips whichever header carried it, so an Authorization
+ // header that survives the gate is the backend's own credential and is
+ // forwarded like on the egress path.
+ facade := newOpenAIFacade(node, egress)
+ mux.Handle("/v1/models", withAuth(token, true, withMeshConnection(node, http.HandlerFunc(facade.handleModels))))
+ mux.Handle("/v1/chat/completions", withAuth(token, true, withMeshConnection(node, http.HandlerFunc(facade.handleCompletions))))
+ mux.Handle("/v1/completions", withAuth(token, true, withMeshConnection(node, http.HandlerFunc(facade.handleCompletions))))
// Mount MCP handler
mcpHandler := NewMCPHandler(node)
@@ -242,6 +253,12 @@ func withAuth(token string, allowAuthorizationFallback bool, next http.Handler)
return
}
+ // The gate credential is local-only: strip exactly the header it came in
+ // on so it can never flow past the gate. Anything left (e.g. Authorization
+ // when the gate was passed via X-Sam-Authentication) is the destination
+ // service's own credential and passes through untouched.
+ r.Header.Del(headerName)
+
next.ServeHTTP(w, r)
})
}
diff --git a/internal/node/sidecar_test.go b/internal/node/sidecar_test.go
index d3e17a19..3ad787c7 100644
--- a/internal/node/sidecar_test.go
+++ b/internal/node/sidecar_test.go
@@ -93,6 +93,45 @@ func TestWithAuth(t *testing.T) {
}
})
+ t.Run("Consumed gate header is stripped, other headers survive", func(t *testing.T) {
+ var gotSamAuth, gotAuth string
+ inspect := withAuth(token, true, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ gotSamAuth = r.Header.Get(api.HeaderSamAuthentication)
+ gotAuth = r.Header.Get("Authorization")
+ w.WriteHeader(http.StatusOK)
+ }))
+
+ req := httptest.NewRequest("GET", "/any", nil)
+ req.Header.Set(api.HeaderSamAuthentication, "Bearer "+token)
+ req.Header.Set("Authorization", "Bearer backend-credential")
+ rr := httptest.NewRecorder()
+ inspect.ServeHTTP(rr, req)
+
+ if gotSamAuth != "" {
+ t.Errorf("consumed %s header must be stripped, got %q", api.HeaderSamAuthentication, gotSamAuth)
+ }
+ if gotAuth != "Bearer backend-credential" {
+ t.Errorf("Authorization must survive when not consumed, got %q", gotAuth)
+ }
+ })
+
+ t.Run("Consumed Authorization fallback is stripped", func(t *testing.T) {
+ var gotAuth string
+ inspect := withAuth(token, true, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ gotAuth = r.Header.Get("Authorization")
+ w.WriteHeader(http.StatusOK)
+ }))
+
+ req := httptest.NewRequest("GET", "/any", nil)
+ req.Header.Set("Authorization", "Bearer "+token)
+ rr := httptest.NewRecorder()
+ inspect.ServeHTTP(rr, req)
+
+ if gotAuth != "" {
+ t.Errorf("consumed Authorization header must be stripped, got %q", gotAuth)
+ }
+ })
+
t.Run("Authorization fallback rejected in strict mode", func(t *testing.T) {
strictHandler := withAuth(token, false, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)