Agent Mesh
Go a2aa2a-protocolagent-meshagentic-aiinferencemcpmeshmesh-networknetwork-securitynetworking-aip2p-network
feat(node): add OpenAI-compatible facade and inference engine connector
Antonio Ojead9524a7a57b3
diff --git a/internal/node/inference_engine.go b/internal/node/inference_engine.gonew file mode 100644index 00000000..4a25d795--- /dev/null+++ b/internal/node/inference_engine.go@@ -0,0 +1,83 @@+// Copyright 2026 Google LLC+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package node++import (+ "context"+ "encoding/json"+ "fmt"+ "io"+ "net/http"+ "net/url"+)++// InferenceEngine is the connector between SAM and an inference backend.+// Implementations adapt one backend family; the service and facade layers+// stay backend-agnostic.+type InferenceEngine interface {+ // Models returns the model IDs the backend currently serves.+ Models(ctx context.Context) ([]string, error)+}++// openAIEngine adapts any OpenAI-compatible HTTP backend+// (vLLM, ollama, llama.cpp, MaaS providers).+type openAIEngine struct {+ baseURL *url.URL+ client *http.Client+}++func newOpenAIEngine(baseURL *url.URL, client *http.Client) *openAIEngine {+ if client == nil {+ client = http.DefaultClient+ }+ return &openAIEngine{baseURL: baseURL, client: client}+}++func (e *openAIEngine) Models(ctx context.Context) ([]string, error) {+ u := *e.baseURL+ u.Path = singleJoiningSlash(e.baseURL.Path, "/v1/models")+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil)+ if err != nil {+ return nil, err+ }+ resp, err := e.client.Do(req)+ if err != nil {+ return nil, err+ }+ defer func() { _ = resp.Body.Close() }()+ if resp.StatusCode != http.StatusOK {+ return nil, fmt.Errorf("backend %s returned status %d for /v1/models", e.baseURL.Host, resp.StatusCode)+ }+ return decodeModelIDs(resp.Body)+}++// decodeModelIDs parses an OpenAI-style /v1/models list response.+func decodeModelIDs(r io.Reader) ([]string, error) {+ var list struct {+ Data []struct {+ ID string `json:"id"`+ } `json:"data"`+ }+ if err := json.NewDecoder(r).Decode(&list); err != nil {+ return nil, fmt.Errorf("invalid models response: %w", err)+ }+ ids := make([]string, 0, len(list.Data))+ for _, m := range list.Data {+ if m.ID != "" {+ ids = append(ids, m.ID)+ }+ }+ return ids, nil+}diff --git a/internal/node/inference_engine_test.go b/internal/node/inference_engine_test.gonew file mode 100644index 00000000..c26a0543--- /dev/null+++ b/internal/node/inference_engine_test.go@@ -0,0 +1,140 @@+// Copyright 2026 Google LLC+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package node++import (+ "context"+ "net/http"+ "net/http/httptest"+ "net/url"+ "reflect"+ "testing"++ "github.com/google/sam/api"+)++func TestOpenAIEngine_Models(t *testing.T) {+ tests := []struct {+ name string+ basePath string+ status int+ body string+ want []string+ wantErr bool+ wantedPath string+ }{+ {+ name: "plain backend",+ status: http.StatusOK,+ body: `{"object":"list","data":[{"id":"llama-3"},{"id":"qwen-2.5"}]}`,+ want: []string{"llama-3", "qwen-2.5"},+ wantedPath: "/v1/models",+ },+ {+ name: "backend with path prefix",+ basePath: "/proxy",+ status: http.StatusOK,+ body: `{"data":[{"id":"m1"}]}`,+ want: []string{"m1"},+ wantedPath: "/proxy/v1/models",+ },+ {+ name: "backend error status",+ status: http.StatusInternalServerError,+ body: `boom`,+ wantErr: true,+ },+ {+ name: "invalid JSON",+ status: http.StatusOK,+ body: `not-json`,+ wantErr: true,+ },+ {+ name: "empty and missing IDs skipped",+ status: http.StatusOK,+ body: `{"data":[{"id":""},{"id":"m2"},{}]}`,+ want: []string{"m2"},+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ var gotPath string+ backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ gotPath = r.URL.Path+ w.WriteHeader(tt.status)+ _, _ = w.Write([]byte(tt.body))+ }))+ defer backend.Close()++ u, err := url.Parse(backend.URL + tt.basePath)+ if err != nil {+ t.Fatalf("parse URL: %v", err)+ }+ engine := newOpenAIEngine(u, nil)+ got, err := engine.Models(context.Background())+ if tt.wantErr {+ if err == nil {+ t.Fatal("Models: expected error, got nil")+ }+ return+ }+ if err != nil {+ t.Fatalf("Models: %v", err)+ }+ if !reflect.DeepEqual(got, tt.want) {+ t.Errorf("Models: got %v, want %v", got, tt.want)+ }+ if tt.wantedPath != "" && gotPath != tt.wantedPath {+ t.Errorf("backend path: got %q, want %q", gotPath, tt.wantedPath)+ }+ })+ }+}++func TestInferenceService_Models_ProbesBackend(t *testing.T) {+ backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ if r.URL.Path != "/v1/models" {+ http.NotFound(w, r)+ return+ }+ _, _ = w.Write([]byte(`{"data":[{"id":"gemma-3"}]}`))+ }))+ defer backend.Close()++ svc := &InferenceService{baseService: baseService{+ info: &api.ServiceInfo{Type: api.ServiceType_SERVICE_TYPE_INFERENCE, Name: "llm"},+ backend: &api.RegisterServiceRequest_TargetUrl{TargetUrl: backend.URL},+ }}+ if err := svc.Init(context.Background()); err != nil {+ t.Fatalf("Init: %v", err)+ }+ got, err := svc.Models(context.Background())+ if err != nil {+ t.Fatalf("Models: %v", err)+ }+ if want := []string{"gemma-3"}; !reflect.DeepEqual(got, want) {+ t.Errorf("Models: got %v, want %v", got, want)+ }+}++func TestInferenceService_Models_Uninitialized(t *testing.T) {+ svc := &InferenceService{baseService: baseService{+ info: &api.ServiceInfo{Type: api.ServiceType_SERVICE_TYPE_INFERENCE, Name: "llm"},+ }}+ if _, err := svc.Models(context.Background()); err == nil {+ t.Fatal("Models: expected error for uninitialized service, got nil")+ }+}diff --git a/internal/node/inference_service.go b/internal/node/inference_service.goindex 6d0a359a..4da3f8a6 100644--- a/internal/node/inference_service.go+++ b/internal/node/inference_service.go@@ -36,6 +36,7 @@ import (type InferenceService struct {baseServicebackendURL *url.URL+ engine InferenceEngine}func (s *InferenceService) Init(ctx context.Context) error {@@ -46,6 +47,7 @@ func (s *InferenceService) Init(ctx context.Context) error {return fmt.Errorf("invalid inference backend URL %q: %w", x.TargetUrl, err)}s.backendURL = u+ s.engine = newOpenAIEngine(u, nil)s.handler = s.newInferenceProxy()case *api.RegisterServiceRequest_Command:return fmt.Errorf("command-based backends are not supported for InferenceService")@@ -55,6 +57,14 @@ func (s *InferenceService) Init(ctx context.Context) error {return nil}+// Models probes the backend connector for the model IDs it serves.+func (s *InferenceService) Models(ctx context.Context) ([]string, error) {+ if s.engine == nil {+ return nil, fmt.Errorf("inference service %q not initialized", s.info.GetName())+ }+ return s.engine.Models(ctx)+}+func (s *InferenceService) newInferenceProxy() http.Handler {return &httputil.ReverseProxy{Director: func(req *http.Request) {diff --git a/internal/node/openai_facade.go b/internal/node/openai_facade.gonew file mode 100644index 00000000..d583f4e8--- /dev/null+++ b/internal/node/openai_facade.go@@ -0,0 +1,342 @@+// Copyright 2026 Google LLC+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package node++import (+ "bytes"+ "context"+ "encoding/base64"+ "encoding/json"+ "errors"+ "fmt"+ "io"+ "net/http"+ "sort"+ "sync"+ "sync/atomic"+ "time"++ "github.com/google/sam/api"+ libp2phttp "github.com/libp2p/go-libp2p-http"+ "github.com/libp2p/go-libp2p/core/network"+ "github.com/libp2p/go-libp2p/core/peer"+)++const (+ // modelRegistryTTL bounds how stale the mesh-wide model view can be.+ modelRegistryTTL = 30 * time.Second+ // remoteModelProbeTimeout bounds a single provider /v1/models probe.+ remoteModelProbeTimeout = 10 * time.Second+ // remoteProbeConcurrency bounds parallel provider probes during refresh.+ remoteProbeConcurrency = 10+ // maxInferenceBodyBytes caps buffered completion request bodies.+ maxInferenceBodyBytes = 16 << 20 // 16 MiB+)++// modelLister is implemented by local services that can report served models.+type modelLister interface {+ Models(ctx context.Context) ([]string, error)+}++// modelProvider locates one service able to serve a model.+// An empty peerID means the service runs on this node.+type modelProvider struct {+ peerID string+ service string+}++// openAIFacade exposes an OpenAI-compatible surface on the sidecar+// (/v1/models, /v1/chat/completions, /v1/completions): point any OpenAI SDK+// at the sidecar and the facade resolves the requested model to a local or+// mesh provider, preferring local. Function fields are seams so unit tests+// can fake discovery and probing.+type openAIFacade struct {+ forward http.Handler // egress proxy handling /sam/<peer>/<type>/<name>/...+ localPeerID func() string+ localServices func() []Service+ discover func(ctx context.Context) ([]*api.DiscoveredProvider, error)+ remoteModels func(ctx context.Context, peerID, srvName string) ([]string, error)++ ttl time.Duration+ mu sync.Mutex+ models map[string][]modelProvider+ expires time.Time+}++func newOpenAIFacade(node *SamNode, egress http.Handler) *openAIFacade {+ client := &http.Client{Transport: libp2phttp.NewTransport(node.Host)}+ return &openAIFacade{+ forward: egress,+ ttl: modelRegistryTTL,+ localPeerID: func() string { return node.Host.ID().String() },+ localServices: func() []Service {+ var out []Service+ for _, info := range node.ListLocalServices(api.ServiceType_SERVICE_TYPE_INFERENCE) {+ if svc, ok := node.services.Get(info.GetName()); ok {+ out = append(out, svc)+ }+ }+ return out+ },+ discover: func(ctx context.Context) ([]*api.DiscoveredProvider, error) {+ return node.DiscoverRemoteServices(ctx, api.ServiceType_SERVICE_TYPE_INFERENCE, "")+ },+ remoteModels: func(ctx context.Context, peerID, srvName string) ([]string, error) {+ return fetchRemoteModels(ctx, node, client, peerID, srvName)+ },+ }+}++// fetchRemoteModels probes a remote provider's backend /v1/models through its+// libp2p ingress, authenticating with this node's biscuit.+func fetchRemoteModels(ctx context.Context, node *SamNode, client *http.Client, peerID, srvName string) ([]string, error) {+ pid, err := peer.Decode(peerID)+ if err != nil {+ return nil, fmt.Errorf("invalid peer ID %q: %w", peerID, err)+ }+ identity := node.GetIdentity()+ if identity == nil {+ return nil, fmt.Errorf("missing node identity")+ }+ ctx, cancel := context.WithTimeout(network.WithAllowLimitedConn(ctx, "openai-facade"), remoteModelProbeTimeout)+ defer cancel()+ if cond := node.Host.Network().Connectedness(pid); cond != network.Connected && cond != network.Limited {+ node.preparePeerAddrs(ctx, pid)+ }+ u := fmt.Sprintf("libp2p://%s/%s/%s/v1/models", peerID, api.ServiceTypeStringInference, srvName)+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil)+ if err != nil {+ return nil, err+ }+ req.Header.Set(api.HeaderSamBiscuit, base64.StdEncoding.EncodeToString(identity))+ resp, err := client.Do(req)+ if err != nil {+ return nil, err+ }+ defer func() { _ = resp.Body.Close() }()+ if resp.StatusCode != http.StatusOK {+ return nil, fmt.Errorf("provider %s/%s returned status %d for /v1/models", peerID, srvName, resp.StatusCode)+ }+ return decodeModelIDs(resp.Body)+}++// providersFor resolves a model to its providers, force-refreshing once on a+// miss so newly appeared providers are usable before the TTL lapses.+func (f *openAIFacade) providersFor(ctx context.Context, model string) []modelProvider {+ f.mu.Lock()+ defer f.mu.Unlock()+ refreshed := false+ if f.models == nil || time.Now().After(f.expires) {+ f.refreshLocked(ctx)+ refreshed = true+ }+ if providers := f.models[model]; len(providers) > 0 || refreshed {+ return providers+ }+ f.refreshLocked(ctx)+ return f.models[model]+}++// modelsView returns the current model registry, refreshing when stale. The+// returned map is a snapshot: refreshes swap the pointer, never mutate.+func (f *openAIFacade) modelsView(ctx context.Context) map[string][]modelProvider {+ f.mu.Lock()+ defer f.mu.Unlock()+ if f.models == nil || time.Now().After(f.expires) {+ f.refreshLocked(ctx)+ }+ return f.models+}++func (f *openAIFacade) refreshLocked(ctx context.Context) {+ f.models = f.collectModels(ctx)+ f.expires = time.Now().Add(f.ttl)+}++// collectModels aggregates model IDs from local services (first, so they are+// preferred) and mesh providers. Failed probes drop the provider with a log+// line rather than failing the whole view.+func (f *openAIFacade) collectModels(ctx context.Context) map[string][]modelProvider {+ models := map[string][]modelProvider{}+ for _, svc := range f.localServices() {+ lister, ok := svc.(modelLister)+ if !ok {+ continue+ }+ ids, err := lister.Models(ctx)+ if err != nil {+ logger.Warnf("[OpenAIFacade] local model probe for %q failed: %v", svc.Info().GetName(), err)+ continue+ }+ for _, id := range ids {+ models[id] = append(models[id], modelProvider{service: svc.Info().GetName()})+ }+ }++ providers, err := f.discover(ctx)+ if err != nil {+ logger.Warnf("[OpenAIFacade] mesh discovery failed: %v", err)+ return models+ }+ var mu sync.Mutex+ var wg sync.WaitGroup+ sem := make(chan struct{}, remoteProbeConcurrency)+ for _, p := range providers {+ wg.Add(1)+ go func(peerID, srvName string) {+ defer wg.Done()+ select {+ case sem <- struct{}{}:+ defer func() { <-sem }()+ case <-ctx.Done():+ return+ }+ ids, err := f.remoteModels(ctx, peerID, srvName)+ if err != nil {+ logger.Warnf("[OpenAIFacade] model probe for %s/%s failed: %v", peerID, srvName, err)+ return+ }+ mu.Lock()+ for _, id := range ids {+ models[id] = append(models[id], modelProvider{peerID: peerID, service: srvName})+ }+ mu.Unlock()+ }(p.GetPeerId(), p.GetSrvName())+ }+ wg.Wait()+ return models+}++// facadeRR spreads load across remote providers of the same model.+var facadeRR atomic.Uint64++// pickProvider prefers a local provider (locals sort first in collectModels),+// otherwise round-robins across remotes.+func pickProvider(providers []modelProvider) modelProvider {+ if providers[0].peerID == "" {+ return providers[0]+ }+ return providers[facadeRR.Add(1)%uint64(len(providers))]+}++func (f *openAIFacade) handleModels(w http.ResponseWriter, r *http.Request) {+ if r.Method != http.MethodGet {+ writeOpenAIError(w, http.StatusMethodNotAllowed, "method_not_allowed", "only GET is supported")+ return+ }+ type openAIModel struct {+ ID string `json:"id"`+ Object string `json:"object"`+ Created int64 `json:"created"`+ OwnedBy string `json:"owned_by"`+ }+ view := f.modelsView(r.Context())+ data := make([]openAIModel, 0, len(view))+ for id, providers := range view {+ ownedBy := "local"+ if providers[0].peerID != "" {+ ownedBy = providers[0].peerID+ }+ data = append(data, openAIModel{ID: id, Object: "model", OwnedBy: ownedBy})+ }+ sort.Slice(data, func(i, j int) bool { return data[i].ID < data[j].ID })++ w.Header().Set("Content-Type", "application/json")+ if err := json.NewEncoder(w).Encode(map[string]any{"object": "list", "data": data}); err != nil {+ logger.Errorf("[OpenAIFacade] failed to encode models response: %v", err)+ }+}++func (f *openAIFacade) handleCompletions(w http.ResponseWriter, r *http.Request) {+ if r.Method != http.MethodPost {+ writeOpenAIError(w, http.StatusMethodNotAllowed, "method_not_allowed", "only POST is supported")+ return+ }+ body, err := io.ReadAll(http.MaxBytesReader(w, r.Body, maxInferenceBodyBytes))+ if err != nil {+ var maxErr *http.MaxBytesError+ if errors.As(err, &maxErr) {+ writeOpenAIError(w, http.StatusRequestEntityTooLarge, "request_too_large", "request body exceeds limit")+ return+ }+ writeOpenAIError(w, http.StatusBadRequest, "invalid_request", "failed to read request body")+ return+ }+ var req struct {+ Model string `json:"model"`+ }+ if err := json.Unmarshal(body, &req); err != nil {+ writeOpenAIError(w, http.StatusBadRequest, "invalid_request", "request body is not valid JSON")+ return+ }+ if req.Model == "" {+ writeOpenAIError(w, http.StatusBadRequest, "invalid_request", "missing required field: model")+ return+ }++ providers := f.providersFor(r.Context(), req.Model)+ if len(providers) == 0 {+ writeOpenAIError(w, http.StatusNotFound, "model_not_found",+ fmt.Sprintf("model %q is not served by any provider on the mesh", req.Model))+ return+ }+ p := pickProvider(providers)+ logger.Debugf("[OpenAIFacade] model %q -> provider peer=%q service=%q (%d candidates)",+ req.Model, p.peerID, p.service, len(providers))++ // The gate already stripped the credential it consumed; a surviving+ // Authorization header is the destination backend's own credential and+ // passes through, mirroring the egress path. X-Sam-Authentication is+ // local-only and must never travel off-node.+ r.Header.Del(api.HeaderSamAuthentication)+ r.Body = io.NopCloser(bytes.NewReader(body))+ r.ContentLength = int64(len(body))++ if p.peerID == "" {+ f.serveLocal(w, r, p.service)+ return+ }+ r.URL.Path = fmt.Sprintf("/sam/%s/%s/%s%s", p.peerID, api.ServiceTypeStringInference, p.service, r.URL.Path)+ r.URL.RawPath = ""+ f.forward.ServeHTTP(w, r)+}++func (f *openAIFacade) serveLocal(w http.ResponseWriter, r *http.Request, serviceName string) {+ for _, svc := range f.localServices() {+ if svc.Info().GetName() != serviceName || svc.Handler() == nil {+ continue+ }+ // Attribute locally served tokens to this node in usage metrics.+ if id := f.localPeerID(); id != "" {+ r.Header.Set("X-Peer-Id", id)+ }+ svc.Handler().ServeHTTP(w, r)+ return+ }+ writeOpenAIError(w, http.StatusServiceUnavailable, "service_unavailable",+ fmt.Sprintf("local service %q is no longer available", serviceName))+}++// writeOpenAIError emits an OpenAI-style error body so SDKs surface it cleanly.+func writeOpenAIError(w http.ResponseWriter, status int, code, msg string) {+ w.Header().Set("Content-Type", "application/json")+ w.WriteHeader(status)+ err := json.NewEncoder(w).Encode(map[string]any{+ "error": map[string]any{"message": msg, "type": "invalid_request_error", "code": code},+ })+ if err != nil {+ logger.Errorf("[OpenAIFacade] failed to encode error response: %v", err)+ }+}diff --git a/internal/node/openai_facade_test.go b/internal/node/openai_facade_test.gonew file mode 100644index 00000000..c4b559dc--- /dev/null+++ b/internal/node/openai_facade_test.go@@ -0,0 +1,321 @@+// Copyright 2026 Google LLC+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package node++import (+ "context"+ "encoding/json"+ "fmt"+ "io"+ "net/http"+ "net/http/httptest"+ "strings"+ "testing"+ "time"++ "github.com/google/sam/api"+)++// fakeModelService is a local Service that also reports served models.+type fakeModelService struct {+ testService+ models []string+ err error+}++func (f *fakeModelService) Models(_ context.Context) ([]string, error) {+ return f.models, f.err+}++func newFakeModelService(name string, handler http.Handler, models ...string) *fakeModelService {+ return &fakeModelService{+ testService: testService{+ info: &api.ServiceInfo{Type: api.ServiceType_SERVICE_TYPE_INFERENCE, Name: name},+ handler: handler,+ },+ models: models,+ }+}++// newTestFacade builds a facade with inert seams; tests override as needed.+func newTestFacade() *openAIFacade {+ return &openAIFacade{+ ttl: time.Minute,+ localPeerID: func() string { return "" },+ localServices: func() []Service { return nil },+ discover: func(_ context.Context) ([]*api.DiscoveredProvider, error) {+ return nil, nil+ },+ remoteModels: func(_ context.Context, _, _ string) ([]string, error) {+ return nil, nil+ },+ }+}++func decodeModelsResponse(t *testing.T, body io.Reader) map[string]string {+ t.Helper()+ var resp struct {+ Object string `json:"object"`+ Data []struct {+ ID string `json:"id"`+ OwnedBy string `json:"owned_by"`+ } `json:"data"`+ }+ if err := json.NewDecoder(body).Decode(&resp); err != nil {+ t.Fatalf("decode models response: %v", err)+ }+ if resp.Object != "list" {+ t.Errorf("object: got %q, want %q", resp.Object, "list")+ }+ out := map[string]string{}+ for _, m := range resp.Data {+ out[m.ID] = m.OwnedBy+ }+ return out+}++func TestFacade_HandleModels_AggregatesLocalAndRemote(t *testing.T) {+ f := newTestFacade()+ f.localServices = func() []Service {+ return []Service{newFakeModelService("local-llm", nil, "local-model", "shared-model")}+ }+ f.discover = func(_ context.Context) ([]*api.DiscoveredProvider, error) {+ return []*api.DiscoveredProvider{+ {PeerId: "peerA", SrvName: "srvA"},+ {PeerId: "peerB", SrvName: "srvB"},+ }, nil+ }+ f.remoteModels = func(_ context.Context, peerID, _ string) ([]string, error) {+ if peerID == "peerA" {+ return []string{"remote-model", "shared-model"}, nil+ }+ return nil, fmt.Errorf("unreachable")+ }++ rec := httptest.NewRecorder()+ f.handleModels(rec, httptest.NewRequest(http.MethodGet, "/v1/models", nil))++ if rec.Code != http.StatusOK {+ t.Fatalf("status: got %d, want %d", rec.Code, http.StatusOK)+ }+ got := decodeModelsResponse(t, rec.Body)+ if len(got) != 3 {+ t.Fatalf("models: got %v, want 3 entries", got)+ }+ if got["local-model"] != "local" {+ t.Errorf("local-model owned_by: got %q, want %q", got["local-model"], "local")+ }+ // Local providers sort first, so a model served both locally and remotely+ // is reported as local.+ if got["shared-model"] != "local" {+ t.Errorf("shared-model owned_by: got %q, want %q", got["shared-model"], "local")+ }+ if got["remote-model"] != "peerA" {+ t.Errorf("remote-model owned_by: got %q, want %q", got["remote-model"], "peerA")+ }+}++func TestFacade_HandleModels_MethodNotAllowed(t *testing.T) {+ f := newTestFacade()+ rec := httptest.NewRecorder()+ f.handleModels(rec, httptest.NewRequest(http.MethodPost, "/v1/models", nil))+ if rec.Code != http.StatusMethodNotAllowed {+ t.Fatalf("status: got %d, want %d", rec.Code, http.StatusMethodNotAllowed)+ }+}++func TestFacade_Completions_ForwardsToRemote(t *testing.T) {+ f := newTestFacade()+ f.discover = func(_ context.Context) ([]*api.DiscoveredProvider, error) {+ return []*api.DiscoveredProvider{{PeerId: "peerA", SrvName: "srvA"}}, nil+ }+ f.remoteModels = func(_ context.Context, _, _ string) ([]string, error) {+ return []string{"m1"}, nil+ }+ var gotPath, gotBody, gotAuth, gotSamAuth string+ f.forward = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ gotPath = r.URL.Path+ b, _ := io.ReadAll(r.Body)+ gotBody = string(b)+ gotAuth = r.Header.Get("Authorization")+ gotSamAuth = r.Header.Get(api.HeaderSamAuthentication)+ w.WriteHeader(http.StatusOK)+ })++ body := `{"model":"m1","messages":[{"role":"user","content":"hi"}]}`+ req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(body))+ // Post-gate state: the gate stripped the credential it consumed; a+ // surviving Authorization header is the backend's own credential.+ req.Header.Set("Authorization", "Bearer backend-credential")+ req.Header.Set(api.HeaderSamAuthentication, "Bearer sidecar-token")+ rec := httptest.NewRecorder()+ f.handleCompletions(rec, req)++ if rec.Code != http.StatusOK {+ t.Fatalf("status: got %d, want %d", rec.Code, http.StatusOK)+ }+ if want := "/sam/peerA/inference/srvA/v1/chat/completions"; gotPath != want {+ t.Errorf("forwarded path: got %q, want %q", gotPath, want)+ }+ if gotBody != body {+ t.Errorf("forwarded body: got %q, want %q", gotBody, body)+ }+ if gotSamAuth != "" {+ t.Errorf("%s must never travel off-node, got %q", api.HeaderSamAuthentication, gotSamAuth)+ }+ if gotAuth != "Bearer backend-credential" {+ t.Errorf("backend Authorization must pass through, got %q", gotAuth)+ }+}++func TestFacade_Completions_LegacyCompletionsPath(t *testing.T) {+ f := newTestFacade()+ f.discover = func(_ context.Context) ([]*api.DiscoveredProvider, error) {+ return []*api.DiscoveredProvider{{PeerId: "peerA", SrvName: "srvA"}}, nil+ }+ f.remoteModels = func(_ context.Context, _, _ string) ([]string, error) {+ return []string{"m1"}, nil+ }+ var gotPath string+ f.forward = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ gotPath = r.URL.Path+ })++ req := httptest.NewRequest(http.MethodPost, "/v1/completions", strings.NewReader(`{"model":"m1","prompt":"hi"}`))+ f.handleCompletions(httptest.NewRecorder(), req)++ if want := "/sam/peerA/inference/srvA/v1/completions"; gotPath != want {+ t.Errorf("forwarded path: got %q, want %q", gotPath, want)+ }+}++func TestFacade_Completions_PrefersLocal(t *testing.T) {+ f := newTestFacade()+ localHit := false+ var gotPeerHeader string+ local := newFakeModelService("local-llm", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ localHit = true+ gotPeerHeader = r.Header.Get("X-Peer-Id")+ w.WriteHeader(http.StatusOK)+ }), "m1")+ f.localPeerID = func() string { return "selfPeer" }+ f.localServices = func() []Service { return []Service{local} }+ f.discover = func(_ context.Context) ([]*api.DiscoveredProvider, error) {+ return []*api.DiscoveredProvider{{PeerId: "peerA", SrvName: "srvA"}}, nil+ }+ f.remoteModels = func(_ context.Context, _, _ string) ([]string, error) {+ return []string{"m1"}, nil+ }+ f.forward = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ t.Error("remote forward must not be used when a local provider exists")+ })++ req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{"model":"m1"}`))+ rec := httptest.NewRecorder()+ f.handleCompletions(rec, req)++ if !localHit {+ t.Fatal("local service was not invoked")+ }+ if gotPeerHeader != "selfPeer" {+ t.Errorf("X-Peer-Id: got %q, want %q", gotPeerHeader, "selfPeer")+ }+}++func TestFacade_Completions_UnknownModel(t *testing.T) {+ f := newTestFacade()+ req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{"model":"nope"}`))+ rec := httptest.NewRecorder()+ f.handleCompletions(rec, req)+ if rec.Code != http.StatusNotFound {+ t.Fatalf("status: got %d, want %d", rec.Code, http.StatusNotFound)+ }+ if !strings.Contains(rec.Body.String(), "model_not_found") {+ t.Errorf("body missing model_not_found code: %s", rec.Body.String())+ }+}++func TestFacade_Completions_BadRequests(t *testing.T) {+ tests := []struct {+ name string+ method string+ body string+ want int+ }{+ {"missing model", http.MethodPost, `{"messages":[]}`, http.StatusBadRequest},+ {"invalid JSON", http.MethodPost, `{`, http.StatusBadRequest},+ {"wrong method", http.MethodGet, ``, http.StatusMethodNotAllowed},+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ f := newTestFacade()+ req := httptest.NewRequest(tt.method, "/v1/chat/completions", strings.NewReader(tt.body))+ rec := httptest.NewRecorder()+ f.handleCompletions(rec, req)+ if rec.Code != tt.want {+ t.Fatalf("status: got %d, want %d", rec.Code, tt.want)+ }+ })+ }+}++func TestFacade_RegistryCacheAndForcedRefresh(t *testing.T) {+ f := newTestFacade()+ discoverCalls := 0+ f.discover = func(_ context.Context) ([]*api.DiscoveredProvider, error) {+ discoverCalls+++ return []*api.DiscoveredProvider{{PeerId: "peerA", SrvName: "srvA"}}, nil+ }+ f.remoteModels = func(_ context.Context, _, _ string) ([]string, error) {+ return []string{"m1"}, nil+ }++ ctx := context.Background()+ // First resolution populates the cache.+ if got := f.providersFor(ctx, "m1"); len(got) != 1 {+ t.Fatalf("providersFor(m1): got %v, want 1 provider", got)+ }+ if discoverCalls != 1 {+ t.Fatalf("discover calls after first resolve: got %d, want 1", discoverCalls)+ }+ // Cached hit within TTL: no new discovery.+ f.providersFor(ctx, "m1")+ if discoverCalls != 1 {+ t.Fatalf("discover calls after cached resolve: got %d, want 1", discoverCalls)+ }+ // Unknown model on a fresh cache forces one refresh.+ f.providersFor(ctx, "m2")+ if discoverCalls != 2 {+ t.Fatalf("discover calls after unknown-model resolve: got %d, want 2", discoverCalls)+ }+}++func TestPickProvider(t *testing.T) {+ local := modelProvider{service: "local-llm"}+ remoteA := modelProvider{peerID: "peerA", service: "srvA"}+ remoteB := modelProvider{peerID: "peerB", service: "srvB"}++ if got := pickProvider([]modelProvider{local, remoteA}); got != local {+ t.Errorf("pickProvider with local first: got %+v, want local", got)+ }++ seen := map[string]bool{}+ for range 10 {+ seen[pickProvider([]modelProvider{remoteA, remoteB}).peerID] = true+ }+ if !seen["peerA"] || !seen["peerB"] {+ t.Errorf("round-robin should hit both remotes, got %v", seen)+ }+}diff --git a/internal/node/sidecar.go b/internal/node/sidecar.goindex 43ceec6a..6416a2b6 100644--- a/internal/node/sidecar.go+++ b/internal/node/sidecar.go@@ -61,7 +61,18 @@ func StartSidecarServer(node *SamNode, addr, token, certFile, keyFile, caFile st// Mount Egress Proxy. allowAuthorizationFallback=false is required here: this// handler forwards Authorization to the destination service, so it must never// also accept it as the local gate credential (would leak the sidecar token off-node).- mux.Handle("/sam/", withAuth(token, false, withMeshConnection(node, createEgressProxy(node))))+ egress := createEgressProxy(node)+ mux.Handle("/sam/", withAuth(token, false, withMeshConnection(node, egress)))++ // OpenAI-compatible facade: point any OpenAI SDK at the sidecar.+ // allowAuthorizationFallback=true lets SDKs send the sidecar token as their+ // api_key; withAuth strips whichever header carried it, so an Authorization+ // header that survives the gate is the backend's own credential and is+ // forwarded like on the egress path.+ facade := newOpenAIFacade(node, egress)+ mux.Handle("/v1/models", withAuth(token, true, withMeshConnection(node, http.HandlerFunc(facade.handleModels))))+ mux.Handle("/v1/chat/completions", withAuth(token, true, withMeshConnection(node, http.HandlerFunc(facade.handleCompletions))))+ mux.Handle("/v1/completions", withAuth(token, true, withMeshConnection(node, http.HandlerFunc(facade.handleCompletions))))// Mount MCP handlermcpHandler := NewMCPHandler(node)@@ -242,6 +253,12 @@ func withAuth(token string, allowAuthorizationFallback bool, next http.Handler)return}+ // The gate credential is local-only: strip exactly the header it came in+ // on so it can never flow past the gate. Anything left (e.g. Authorization+ // when the gate was passed via X-Sam-Authentication) is the destination+ // service's own credential and passes through untouched.+ r.Header.Del(headerName)+next.ServeHTTP(w, r)})}diff --git a/internal/node/sidecar_test.go b/internal/node/sidecar_test.goindex d3e17a19..3ad787c7 100644--- a/internal/node/sidecar_test.go+++ b/internal/node/sidecar_test.go@@ -93,6 +93,45 @@ func TestWithAuth(t *testing.T) {}})+ t.Run("Consumed gate header is stripped, other headers survive", func(t *testing.T) {+ var gotSamAuth, gotAuth string+ inspect := withAuth(token, true, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ gotSamAuth = r.Header.Get(api.HeaderSamAuthentication)+ gotAuth = r.Header.Get("Authorization")+ w.WriteHeader(http.StatusOK)+ }))++ req := httptest.NewRequest("GET", "/any", nil)+ req.Header.Set(api.HeaderSamAuthentication, "Bearer "+token)+ req.Header.Set("Authorization", "Bearer backend-credential")+ rr := httptest.NewRecorder()+ inspect.ServeHTTP(rr, req)++ if gotSamAuth != "" {+ t.Errorf("consumed %s header must be stripped, got %q", api.HeaderSamAuthentication, gotSamAuth)+ }+ if gotAuth != "Bearer backend-credential" {+ t.Errorf("Authorization must survive when not consumed, got %q", gotAuth)+ }+ })++ t.Run("Consumed Authorization fallback is stripped", func(t *testing.T) {+ var gotAuth string+ inspect := withAuth(token, true, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ gotAuth = r.Header.Get("Authorization")+ w.WriteHeader(http.StatusOK)+ }))++ req := httptest.NewRequest("GET", "/any", nil)+ req.Header.Set("Authorization", "Bearer "+token)+ rr := httptest.NewRecorder()+ inspect.ServeHTTP(rr, req)++ if gotAuth != "" {+ t.Errorf("consumed Authorization header must be stripped, got %q", gotAuth)+ }+ })+t.Run("Authorization fallback rejected in strict mode", func(t *testing.T) {strictHandler := withAuth(token, false, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {w.WriteHeader(http.StatusOK)