Agent Mesh
Go a2aa2a-protocolagent-meshagentic-aiinferencemcpmeshmesh-networknetwork-securitynetworking-aip2p-network
add inference service
Antonio Ojea26ef0e79106d
diff --git a/cmd/sam-node/inference_metrics.go b/cmd/sam-node/inference_metrics.gonew file mode 100644index 00000000..7a29977f--- /dev/null+++ b/cmd/sam-node/inference_metrics.go@@ -0,0 +1,46 @@+// Copyright 2026 Google LLC+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package main++import (+ "github.com/prometheus/client_golang/prometheus"+ "github.com/prometheus/client_golang/prometheus/promauto"+)++var (+ inferenceTokensTotal = promauto.NewCounterVec(+ prometheus.CounterOpts{+ Name: "sam_node_inference_tokens_total",+ Help: "Total number of inference tokens tracked by sam-node",+ },+ []string{"peer_id", "model", "token_type"},+ )+)++// recordTokens increments the prompt and completion token metrics.+func recordTokens(peerID, model string, promptTokens, completionTokens int) {+ if model == "" {+ model = "unknown"+ }+ if peerID == "" {+ peerID = "unknown"+ }+ if promptTokens > 0 {+ inferenceTokensTotal.WithLabelValues(peerID, model, "prompt").Add(float64(promptTokens))+ }+ if completionTokens > 0 {+ inferenceTokensTotal.WithLabelValues(peerID, model, "completion").Add(float64(completionTokens))+ }+}diff --git a/cmd/sam-node/inference_service.go b/cmd/sam-node/inference_service.gonew file mode 100644index 00000000..22004060--- /dev/null+++ b/cmd/sam-node/inference_service.go@@ -0,0 +1,219 @@+// Copyright 2026 Google LLC+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package main++import (+ "bufio"+ "context"+ "encoding/json"+ "fmt"+ "io"+ "net"+ "net/http"+ "net/http/httputil"+ "net/url"+ "strings"++ "github.com/google/sam/api"+ "github.com/libp2p/go-libp2p/core/peer"+ "github.com/openai/openai-go"+)++// InferenceService provides intelligent LLM gateway features: traffic routing+// and token usage tracking for OpenAI-compatible endpoints.+type InferenceService struct {+ baseService+ backendURL *url.URL+}++func (s *InferenceService) Init(ctx context.Context) error {+ switch x := s.backend.(type) {+ case *api.RegisterServiceRequest_TargetUrl:+ u, err := url.Parse(x.TargetUrl)+ if err != nil {+ return fmt.Errorf("invalid inference backend URL %q: %w", x.TargetUrl, err)+ }+ s.backendURL = u+ s.handler = s.newInferenceProxy()+ case *api.RegisterServiceRequest_Command:+ return s.baseService.Init(ctx)+ default:+ return fmt.Errorf("unsupported backend type %T for InferenceService", s.backend)+ }+ return nil+}++func (s *InferenceService) newInferenceProxy() http.Handler {+ return &httputil.ReverseProxy{+ Director: func(req *http.Request) {+ if _, ok := req.Header["User-Agent"]; !ok {+ req.Header.Set("User-Agent", "")+ }+ },+ Transport: &inferenceTransport{+ backend: s.backendURL,+ base: http.DefaultTransport,+ },+ }+}++type inferenceTransport struct {+ backend *url.URL+ base http.RoundTripper+}++func (t *inferenceTransport) RoundTrip(req *http.Request) (*http.Response, error) {+ attemptReq := req.Clone(req.Context())++ attemptReq.URL.Scheme = t.backend.Scheme+ attemptReq.URL.Host = t.backend.Host+ attemptReq.URL.Path = singleJoiningSlash(t.backend.Path, req.URL.Path)+ if t.backend.RawQuery == "" || attemptReq.URL.RawQuery == "" {+ attemptReq.URL.RawQuery = t.backend.RawQuery + attemptReq.URL.RawQuery+ } else {+ attemptReq.URL.RawQuery = t.backend.RawQuery + "&" + attemptReq.URL.RawQuery+ }+ attemptReq.Host = t.backend.Host++ resp, err := t.base.RoundTrip(attemptReq)+ if err != nil {+ return nil, err+ }++ if resp.StatusCode == http.StatusOK {+ peerID := getPeerID(req)+ isSSE := resp.Header.Get("Content-Type") == "text/event-stream"+ resp.Body = newInterceptingReader(resp.Body, peerID, isSSE)+ }++ return resp, nil+}++func singleJoiningSlash(a, b string) string {+ aslash := strings.HasSuffix(a, "/")+ bslash := strings.HasPrefix(b, "/")+ switch {+ case aslash && bslash:+ return a + b[1:]+ case !aslash && !bslash:+ return a + "/" + b+ }+ return a + b+}++func getPeerID(r *http.Request) string {+ remoteAddr := r.RemoteAddr+ if remoteAddr != "" {+ if _, err := peer.Decode(remoteAddr); err == nil {+ return remoteAddr+ }+ if host, _, err := net.SplitHostPort(remoteAddr); err == nil {+ if _, err := peer.Decode(host); err == nil {+ return host+ }+ }+ }+ if peerHeader := r.Header.Get("X-Peer-Id"); peerHeader != "" {+ return peerHeader+ }+ return "unknown"+}++type interceptingReader struct {+ body io.ReadCloser+ pw *io.PipeWriter+ done chan struct{}+}++func newInterceptingReader(body io.ReadCloser, peerID string, isSSE bool) io.ReadCloser {+ pr, pw := io.Pipe()+ ir := &interceptingReader{+ body: body,+ pw: pw,+ done: make(chan struct{}),+ }++ go func() {+ defer func() { _ = pr.Close() }()+ defer close(ir.done)+ parseResponseStream(pr, peerID, isSSE)+ }()++ return ir+}++func (ir *interceptingReader) Read(p []byte) (n int, err error) {+ n, err = ir.body.Read(p)+ if n > 0 {+ _, _ = ir.pw.Write(p[:n])+ }+ if err != nil {+ _ = ir.pw.CloseWithError(err)+ }+ return n, err+}++func (ir *interceptingReader) Close() error {+ err := ir.body.Close()+ _ = ir.pw.CloseWithError(io.EOF)+ return err+}++func parseResponseStream(r io.Reader, peerID string, isSSE bool) {+ if isSSE {+ parseSSEResponse(r, peerID)+ } else {+ parseJSONResponse(r, peerID)+ }+}++func parseJSONResponse(r io.Reader, peerID string) {+ var resp openai.ChatCompletion+ if err := json.NewDecoder(r).Decode(&resp); err != nil {+ return+ }+ if resp.Usage.PromptTokens > 0 || resp.Usage.CompletionTokens > 0 {+ recordTokens(peerID, resp.Model, int(resp.Usage.PromptTokens), int(resp.Usage.CompletionTokens))+ }+}++func parseSSEResponse(r io.Reader, peerID string) {+ scanner := bufio.NewScanner(r)+ var model string+ for scanner.Scan() {+ line := strings.TrimSpace(scanner.Text())+ if line == "" {+ continue+ }+ if !strings.HasPrefix(line, "data:") {+ continue+ }+ dataContent := strings.TrimSpace(strings.TrimPrefix(line, "data:"))+ if dataContent == "[DONE]" {+ break+ }++ var chunk openai.ChatCompletionChunk+ if err := json.Unmarshal([]byte(dataContent), &chunk); err == nil {+ if chunk.Model != "" {+ model = chunk.Model+ }+ if chunk.Usage.PromptTokens > 0 || chunk.Usage.CompletionTokens > 0 {+ recordTokens(peerID, model, int(chunk.Usage.PromptTokens), int(chunk.Usage.CompletionTokens))+ break+ }+ }+ }+}diff --git a/cmd/sam-node/inference_test.go b/cmd/sam-node/inference_test.gonew file mode 100644index 00000000..5d558867--- /dev/null+++ b/cmd/sam-node/inference_test.go@@ -0,0 +1,169 @@+// Copyright 2026 Google LLC+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package main++import (+ "bytes"+ "context"+ "fmt"+ "io"+ "net/http"+ "net/http/httptest"+ "strings"+ "testing"+ "time"++ "github.com/google/sam/api"+ dto "github.com/prometheus/client_model/go"+)++func getCounterValue(peerID, model, tokenType string) float64 {+ metric, err := inferenceTokensTotal.GetMetricWithLabelValues(peerID, model, tokenType)+ if err != nil {+ return 0+ }+ var m dto.Metric+ if err := metric.Write(&m); err != nil {+ return 0+ }+ return m.GetCounter().GetValue()+}++func TestInferenceService_TokenAccountability_JSON(t *testing.T) {+ peerID := "test-peer-json"+ modelName := "test-model-json"++ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ w.Header().Set("Content-Type", "application/json")+ w.WriteHeader(http.StatusOK)+ _, _ = fmt.Fprintf(w, `{+ "model": "%s",+ "usage": {+ "prompt_tokens": 42,+ "completion_tokens": 99+ }+ }`, modelName)+ }))+ defer server.Close()++ svc := &InferenceService{+ baseService: baseService{+ info: &api.ServiceInfo{Type: api.ServiceType_SERVICE_TYPE_INFERENCE, Name: "test-inference-json"},+ backend: &api.RegisterServiceRequest_TargetUrl{TargetUrl: server.URL},+ },+ }+ if err := svc.Init(context.Background()); err != nil {+ t.Fatalf("Init failed: %v", err)+ }++ req := httptest.NewRequest("POST", "http://localhost/chat", bytes.NewReader([]byte(`{"test":true}`)))+ req.Header.Set("X-Peer-Id", peerID)+ w := httptest.NewRecorder()++ initialPromptTokens := getCounterValue(peerID, modelName, "prompt")+ initialCompletionTokens := getCounterValue(peerID, modelName, "completion")++ svc.Handler().ServeHTTP(w, req)++ resp := w.Result()+ body, _ := io.ReadAll(resp.Body)+ _ = resp.Body.Close()++ if resp.StatusCode != http.StatusOK {+ t.Fatalf("Expected status OK, got %d, body: %s", resp.StatusCode, string(body))+ }++ // Wait briefly for async recording goroutine to finish+ time.Sleep(50 * time.Millisecond)++ promptTokens := getCounterValue(peerID, modelName, "prompt") - initialPromptTokens+ completionTokens := getCounterValue(peerID, modelName, "completion") - initialCompletionTokens++ if promptTokens != 42 {+ t.Errorf("Expected 42 prompt tokens, got %f", promptTokens)+ }+ if completionTokens != 99 {+ t.Errorf("Expected 99 completion tokens, got %f", completionTokens)+ }+}++func TestInferenceService_TokenAccountability_SSE(t *testing.T) {+ peerID := "test-peer-sse"+ modelName := "test-model-sse"++ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ w.Header().Set("Content-Type", "text/event-stream")+ w.Header().Set("Cache-Control", "no-cache")+ w.Header().Set("Connection", "keep-alive")+ w.WriteHeader(http.StatusOK)++ flusher, _ := w.(http.Flusher)++ _, _ = fmt.Fprintf(w, "data: {\"model\":\"%s\"}\n\n", modelName)+ flusher.Flush()++ _, _ = fmt.Fprintf(w, "data: {\"model\":\"%s\",\"usage\":{\"prompt_tokens\":50,\"completion_tokens\":150}}\n\n", modelName)+ flusher.Flush()++ _, _ = fmt.Fprintf(w, "data: [DONE]\n\n")+ flusher.Flush()+ }))+ defer server.Close()++ svc := &InferenceService{+ baseService: baseService{+ info: &api.ServiceInfo{Type: api.ServiceType_SERVICE_TYPE_INFERENCE, Name: "test-inference-sse"},+ backend: &api.RegisterServiceRequest_TargetUrl{TargetUrl: server.URL},+ },+ }+ if err := svc.Init(context.Background()); err != nil {+ t.Fatalf("Init failed: %v", err)+ }++ req := httptest.NewRequest("POST", "http://localhost/chat", bytes.NewReader([]byte(`{"test":true}`)))+ req.Header.Set("X-Peer-Id", peerID)+ w := httptest.NewRecorder()++ initialPromptTokens := getCounterValue(peerID, modelName, "prompt")+ initialCompletionTokens := getCounterValue(peerID, modelName, "completion")++ svc.Handler().ServeHTTP(w, req)++ resp := w.Result()+ body, _ := io.ReadAll(resp.Body)+ _ = resp.Body.Close()++ if resp.StatusCode != http.StatusOK {+ t.Fatalf("Expected status OK, got %d, body: %s", resp.StatusCode, string(body))+ }++ // Let's assert that we actually got the SSE stream+ if !strings.Contains(string(body), "[DONE]") {+ t.Fatalf("Expected body to contain [DONE], got: %q", string(body))+ }++ // Wait briefly for async recording goroutine to finish+ time.Sleep(50 * time.Millisecond)++ promptTokens := getCounterValue(peerID, modelName, "prompt") - initialPromptTokens+ completionTokens := getCounterValue(peerID, modelName, "completion") - initialCompletionTokens++ if promptTokens != 50 {+ t.Errorf("Expected 50 prompt tokens, got %f", promptTokens)+ }+ if completionTokens != 150 {+ t.Errorf("Expected 150 completion tokens, got %f", completionTokens)+ }+}diff --git a/cmd/sam-node/service.go b/cmd/sam-node/service.goindex 77d34e74..16126e0f 100644--- a/cmd/sam-node/service.go+++ b/cmd/sam-node/service.go@@ -93,10 +93,9 @@ func (b *baseService) Teardown() error {return b.cmd.Process.Kill()}-// InferenceService and A2AService are zero-override embeddings. They exist+// A2AService is zero-override embedding. They exist// so the factory produces a distinct type per ServiceType, leaving room for// future per-kind behaviour without churn.-type InferenceService struct{ baseService }type A2AService struct{ baseService }func NewServiceFromRequest(req *api.RegisterServiceRequest) (Service, error) {diff --git a/go.mod b/go.modindex daac4656..3d3c7f71 100644--- a/go.mod+++ b/go.mod@@ -19,6 +19,7 @@ require (github.com/multiformats/go-multiaddr v0.16.1github.com/multiformats/go-multihash v0.2.3github.com/prometheus/client_golang v1.23.2+ github.com/prometheus/client_model v0.6.2github.com/spf13/cobra v1.10.2go.etcd.io/bbolt v1.4.3golang.org/x/oauth2 v0.36.0@@ -82,6 +83,7 @@ require (github.com/multiformats/go-multistream v0.6.1 // indirectgithub.com/multiformats/go-varint v0.1.0 // indirectgithub.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect+ github.com/openai/openai-go v1.12.0 // indirectgithub.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirectgithub.com/pion/datachannel v1.6.0 // indirectgithub.com/pion/dtls/v3 v3.1.2 // indirect@@ -100,7 +102,6 @@ require (github.com/pion/turn/v5 v5.0.3 // indirectgithub.com/pion/webrtc/v4 v4.2.11 // indirectgithub.com/polydawn/refmt v0.89.1-0.20231129105047-37766d95467a // indirect- github.com/prometheus/client_model v0.6.2 // indirectgithub.com/prometheus/common v0.67.5 // indirectgithub.com/prometheus/procfs v0.20.1 // indirectgithub.com/quic-go/qpack v0.6.0 // indirect@@ -110,6 +111,10 @@ require (github.com/segmentio/encoding v0.5.4 // indirectgithub.com/spaolacci/murmur3 v1.1.0 // indirectgithub.com/spf13/pflag v1.0.10 // indirect+ github.com/tidwall/gjson v1.14.4 // indirect+ github.com/tidwall/match v1.1.1 // indirect+ github.com/tidwall/pretty v1.2.1 // indirect+ github.com/tidwall/sjson v1.2.5 // indirectgithub.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 // indirectgithub.com/wlynxg/anet v0.0.5 // indirectgithub.com/yosida95/uritemplate/v3 v3.0.2 // indirectdiff --git a/go.sum b/go.sumindex d8f22dc3..1f1b278f 100644--- a/go.sum+++ b/go.sum@@ -372,6 +372,8 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oqgithub.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJEU3ofeGjhHklVoIGuVj85JJwZ6kWPaJwCIxgnFmo=github.com/neelance/sourcemap v0.0.0-20200213170602-2833bce08e4c/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM=+github.com/openai/openai-go v1.12.0 h1:NBQCnXzqOTv5wsgNC36PrFEiskGfO5wccfCWDo9S1U0=+github.com/openai/openai-go v1.12.0/go.mod h1:g461MYGXEXBVdV5SaR/5tNzNbSfwTBBefwc+LlDCK0Y=github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2DcNVpwGmV9E1BkGknEliJkfwQj0=github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y=@@ -480,6 +482,16 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=+github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=+github.com/tidwall/gjson v1.14.4 h1:uo0p8EbA09J7RQaflQ1aBRffTR7xedD2bcIVSYxLnkM=+github.com/tidwall/gjson v1.14.4/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=+github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=+github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=+github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=+github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4=+github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=+github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY=+github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28=github.com/urfave/cli v1.22.10/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=github.com/warpfork/go-wish v0.0.0-20220906213052-39a1cc7a02d0 h1:GDDkbFiaK8jsSDJfjId/PEGEShv6ugrt4kYsC5UIDaQ=github.com/warpfork/go-wish v0.0.0-20220906213052-39a1cc7a02d0/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw=