Skip to content

Agent Mesh

add inference service

Antonio Ojea26ef0e79106d
diff --git a/cmd/sam-node/inference_metrics.go b/cmd/sam-node/inference_metrics.go
new file mode 100644
index 00000000..7a29977f
--- /dev/null
+++ b/cmd/sam-node/inference_metrics.go
@@ -0,0 +1,46 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+ "github.com/prometheus/client_golang/prometheus"
+ "github.com/prometheus/client_golang/prometheus/promauto"
+)
+
+var (
+ inferenceTokensTotal = promauto.NewCounterVec(
+ prometheus.CounterOpts{
+ Name: "sam_node_inference_tokens_total",
+ Help: "Total number of inference tokens tracked by sam-node",
+ },
+ []string{"peer_id", "model", "token_type"},
+ )
+)
+
+// recordTokens increments the prompt and completion token metrics.
+func recordTokens(peerID, model string, promptTokens, completionTokens int) {
+ if model == "" {
+ model = "unknown"
+ }
+ if peerID == "" {
+ peerID = "unknown"
+ }
+ if promptTokens > 0 {
+ inferenceTokensTotal.WithLabelValues(peerID, model, "prompt").Add(float64(promptTokens))
+ }
+ if completionTokens > 0 {
+ inferenceTokensTotal.WithLabelValues(peerID, model, "completion").Add(float64(completionTokens))
+ }
+}
diff --git a/cmd/sam-node/inference_service.go b/cmd/sam-node/inference_service.go
new file mode 100644
index 00000000..22004060
--- /dev/null
+++ b/cmd/sam-node/inference_service.go
@@ -0,0 +1,219 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+ "bufio"
+ "context"
+ "encoding/json"
+ "fmt"
+ "io"
+ "net"
+ "net/http"
+ "net/http/httputil"
+ "net/url"
+ "strings"
+
+ "github.com/google/sam/api"
+ "github.com/libp2p/go-libp2p/core/peer"
+ "github.com/openai/openai-go"
+)
+
+// InferenceService provides intelligent LLM gateway features: traffic routing
+// and token usage tracking for OpenAI-compatible endpoints.
+type InferenceService struct {
+ baseService
+ backendURL *url.URL
+}
+
+func (s *InferenceService) Init(ctx context.Context) error {
+ switch x := s.backend.(type) {
+ case *api.RegisterServiceRequest_TargetUrl:
+ u, err := url.Parse(x.TargetUrl)
+ if err != nil {
+ return fmt.Errorf("invalid inference backend URL %q: %w", x.TargetUrl, err)
+ }
+ s.backendURL = u
+ s.handler = s.newInferenceProxy()
+ case *api.RegisterServiceRequest_Command:
+ return s.baseService.Init(ctx)
+ default:
+ return fmt.Errorf("unsupported backend type %T for InferenceService", s.backend)
+ }
+ return nil
+}
+
+func (s *InferenceService) newInferenceProxy() http.Handler {
+ return &httputil.ReverseProxy{
+ Director: func(req *http.Request) {
+ if _, ok := req.Header["User-Agent"]; !ok {
+ req.Header.Set("User-Agent", "")
+ }
+ },
+ Transport: &inferenceTransport{
+ backend: s.backendURL,
+ base: http.DefaultTransport,
+ },
+ }
+}
+
+type inferenceTransport struct {
+ backend *url.URL
+ base http.RoundTripper
+}
+
+func (t *inferenceTransport) RoundTrip(req *http.Request) (*http.Response, error) {
+ attemptReq := req.Clone(req.Context())
+
+ attemptReq.URL.Scheme = t.backend.Scheme
+ attemptReq.URL.Host = t.backend.Host
+ attemptReq.URL.Path = singleJoiningSlash(t.backend.Path, req.URL.Path)
+ if t.backend.RawQuery == "" || attemptReq.URL.RawQuery == "" {
+ attemptReq.URL.RawQuery = t.backend.RawQuery + attemptReq.URL.RawQuery
+ } else {
+ attemptReq.URL.RawQuery = t.backend.RawQuery + "&" + attemptReq.URL.RawQuery
+ }
+ attemptReq.Host = t.backend.Host
+
+ resp, err := t.base.RoundTrip(attemptReq)
+ if err != nil {
+ return nil, err
+ }
+
+ if resp.StatusCode == http.StatusOK {
+ peerID := getPeerID(req)
+ isSSE := resp.Header.Get("Content-Type") == "text/event-stream"
+ resp.Body = newInterceptingReader(resp.Body, peerID, isSSE)
+ }
+
+ return resp, nil
+}
+
+func singleJoiningSlash(a, b string) string {
+ aslash := strings.HasSuffix(a, "/")
+ bslash := strings.HasPrefix(b, "/")
+ switch {
+ case aslash && bslash:
+ return a + b[1:]
+ case !aslash && !bslash:
+ return a + "/" + b
+ }
+ return a + b
+}
+
+func getPeerID(r *http.Request) string {
+ remoteAddr := r.RemoteAddr
+ if remoteAddr != "" {
+ if _, err := peer.Decode(remoteAddr); err == nil {
+ return remoteAddr
+ }
+ if host, _, err := net.SplitHostPort(remoteAddr); err == nil {
+ if _, err := peer.Decode(host); err == nil {
+ return host
+ }
+ }
+ }
+ if peerHeader := r.Header.Get("X-Peer-Id"); peerHeader != "" {
+ return peerHeader
+ }
+ return "unknown"
+}
+
+type interceptingReader struct {
+ body io.ReadCloser
+ pw *io.PipeWriter
+ done chan struct{}
+}
+
+func newInterceptingReader(body io.ReadCloser, peerID string, isSSE bool) io.ReadCloser {
+ pr, pw := io.Pipe()
+ ir := &interceptingReader{
+ body: body,
+ pw: pw,
+ done: make(chan struct{}),
+ }
+
+ go func() {
+ defer func() { _ = pr.Close() }()
+ defer close(ir.done)
+ parseResponseStream(pr, peerID, isSSE)
+ }()
+
+ return ir
+}
+
+func (ir *interceptingReader) Read(p []byte) (n int, err error) {
+ n, err = ir.body.Read(p)
+ if n > 0 {
+ _, _ = ir.pw.Write(p[:n])
+ }
+ if err != nil {
+ _ = ir.pw.CloseWithError(err)
+ }
+ return n, err
+}
+
+func (ir *interceptingReader) Close() error {
+ err := ir.body.Close()
+ _ = ir.pw.CloseWithError(io.EOF)
+ return err
+}
+
+func parseResponseStream(r io.Reader, peerID string, isSSE bool) {
+ if isSSE {
+ parseSSEResponse(r, peerID)
+ } else {
+ parseJSONResponse(r, peerID)
+ }
+}
+
+func parseJSONResponse(r io.Reader, peerID string) {
+ var resp openai.ChatCompletion
+ if err := json.NewDecoder(r).Decode(&resp); err != nil {
+ return
+ }
+ if resp.Usage.PromptTokens > 0 || resp.Usage.CompletionTokens > 0 {
+ recordTokens(peerID, resp.Model, int(resp.Usage.PromptTokens), int(resp.Usage.CompletionTokens))
+ }
+}
+
+func parseSSEResponse(r io.Reader, peerID string) {
+ scanner := bufio.NewScanner(r)
+ var model string
+ for scanner.Scan() {
+ line := strings.TrimSpace(scanner.Text())
+ if line == "" {
+ continue
+ }
+ if !strings.HasPrefix(line, "data:") {
+ continue
+ }
+ dataContent := strings.TrimSpace(strings.TrimPrefix(line, "data:"))
+ if dataContent == "[DONE]" {
+ break
+ }
+
+ var chunk openai.ChatCompletionChunk
+ if err := json.Unmarshal([]byte(dataContent), &chunk); err == nil {
+ if chunk.Model != "" {
+ model = chunk.Model
+ }
+ if chunk.Usage.PromptTokens > 0 || chunk.Usage.CompletionTokens > 0 {
+ recordTokens(peerID, model, int(chunk.Usage.PromptTokens), int(chunk.Usage.CompletionTokens))
+ break
+ }
+ }
+ }
+}
diff --git a/cmd/sam-node/inference_test.go b/cmd/sam-node/inference_test.go
new file mode 100644
index 00000000..5d558867
--- /dev/null
+++ b/cmd/sam-node/inference_test.go
@@ -0,0 +1,169 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+ "bytes"
+ "context"
+ "fmt"
+ "io"
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/google/sam/api"
+ dto "github.com/prometheus/client_model/go"
+)
+
+func getCounterValue(peerID, model, tokenType string) float64 {
+ metric, err := inferenceTokensTotal.GetMetricWithLabelValues(peerID, model, tokenType)
+ if err != nil {
+ return 0
+ }
+ var m dto.Metric
+ if err := metric.Write(&m); err != nil {
+ return 0
+ }
+ return m.GetCounter().GetValue()
+}
+
+func TestInferenceService_TokenAccountability_JSON(t *testing.T) {
+ peerID := "test-peer-json"
+ modelName := "test-model-json"
+
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("Content-Type", "application/json")
+ w.WriteHeader(http.StatusOK)
+ _, _ = fmt.Fprintf(w, `{
+ "model": "%s",
+ "usage": {
+ "prompt_tokens": 42,
+ "completion_tokens": 99
+ }
+ }`, modelName)
+ }))
+ defer server.Close()
+
+ svc := &InferenceService{
+ baseService: baseService{
+ info: &api.ServiceInfo{Type: api.ServiceType_SERVICE_TYPE_INFERENCE, Name: "test-inference-json"},
+ backend: &api.RegisterServiceRequest_TargetUrl{TargetUrl: server.URL},
+ },
+ }
+ if err := svc.Init(context.Background()); err != nil {
+ t.Fatalf("Init failed: %v", err)
+ }
+
+ req := httptest.NewRequest("POST", "http://localhost/chat", bytes.NewReader([]byte(`{"test":true}`)))
+ req.Header.Set("X-Peer-Id", peerID)
+ w := httptest.NewRecorder()
+
+ initialPromptTokens := getCounterValue(peerID, modelName, "prompt")
+ initialCompletionTokens := getCounterValue(peerID, modelName, "completion")
+
+ svc.Handler().ServeHTTP(w, req)
+
+ resp := w.Result()
+ body, _ := io.ReadAll(resp.Body)
+ _ = resp.Body.Close()
+
+ if resp.StatusCode != http.StatusOK {
+ t.Fatalf("Expected status OK, got %d, body: %s", resp.StatusCode, string(body))
+ }
+
+ // Wait briefly for async recording goroutine to finish
+ time.Sleep(50 * time.Millisecond)
+
+ promptTokens := getCounterValue(peerID, modelName, "prompt") - initialPromptTokens
+ completionTokens := getCounterValue(peerID, modelName, "completion") - initialCompletionTokens
+
+ if promptTokens != 42 {
+ t.Errorf("Expected 42 prompt tokens, got %f", promptTokens)
+ }
+ if completionTokens != 99 {
+ t.Errorf("Expected 99 completion tokens, got %f", completionTokens)
+ }
+}
+
+func TestInferenceService_TokenAccountability_SSE(t *testing.T) {
+ peerID := "test-peer-sse"
+ modelName := "test-model-sse"
+
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("Content-Type", "text/event-stream")
+ w.Header().Set("Cache-Control", "no-cache")
+ w.Header().Set("Connection", "keep-alive")
+ w.WriteHeader(http.StatusOK)
+
+ flusher, _ := w.(http.Flusher)
+
+ _, _ = fmt.Fprintf(w, "data: {\"model\":\"%s\"}\n\n", modelName)
+ flusher.Flush()
+
+ _, _ = fmt.Fprintf(w, "data: {\"model\":\"%s\",\"usage\":{\"prompt_tokens\":50,\"completion_tokens\":150}}\n\n", modelName)
+ flusher.Flush()
+
+ _, _ = fmt.Fprintf(w, "data: [DONE]\n\n")
+ flusher.Flush()
+ }))
+ defer server.Close()
+
+ svc := &InferenceService{
+ baseService: baseService{
+ info: &api.ServiceInfo{Type: api.ServiceType_SERVICE_TYPE_INFERENCE, Name: "test-inference-sse"},
+ backend: &api.RegisterServiceRequest_TargetUrl{TargetUrl: server.URL},
+ },
+ }
+ if err := svc.Init(context.Background()); err != nil {
+ t.Fatalf("Init failed: %v", err)
+ }
+
+ req := httptest.NewRequest("POST", "http://localhost/chat", bytes.NewReader([]byte(`{"test":true}`)))
+ req.Header.Set("X-Peer-Id", peerID)
+ w := httptest.NewRecorder()
+
+ initialPromptTokens := getCounterValue(peerID, modelName, "prompt")
+ initialCompletionTokens := getCounterValue(peerID, modelName, "completion")
+
+ svc.Handler().ServeHTTP(w, req)
+
+ resp := w.Result()
+ body, _ := io.ReadAll(resp.Body)
+ _ = resp.Body.Close()
+
+ if resp.StatusCode != http.StatusOK {
+ t.Fatalf("Expected status OK, got %d, body: %s", resp.StatusCode, string(body))
+ }
+
+ // Let's assert that we actually got the SSE stream
+ if !strings.Contains(string(body), "[DONE]") {
+ t.Fatalf("Expected body to contain [DONE], got: %q", string(body))
+ }
+
+ // Wait briefly for async recording goroutine to finish
+ time.Sleep(50 * time.Millisecond)
+
+ promptTokens := getCounterValue(peerID, modelName, "prompt") - initialPromptTokens
+ completionTokens := getCounterValue(peerID, modelName, "completion") - initialCompletionTokens
+
+ if promptTokens != 50 {
+ t.Errorf("Expected 50 prompt tokens, got %f", promptTokens)
+ }
+ if completionTokens != 150 {
+ t.Errorf("Expected 150 completion tokens, got %f", completionTokens)
+ }
+}
diff --git a/cmd/sam-node/service.go b/cmd/sam-node/service.go
index 77d34e74..16126e0f 100644
--- a/cmd/sam-node/service.go
+++ b/cmd/sam-node/service.go
@@ -93,10 +93,9 @@ func (b *baseService) Teardown() error {
return b.cmd.Process.Kill()
}
-// InferenceService and A2AService are zero-override embeddings. They exist
+// A2AService is zero-override embedding. They exist
// so the factory produces a distinct type per ServiceType, leaving room for
// future per-kind behaviour without churn.
-type InferenceService struct{ baseService }
type A2AService struct{ baseService }
func NewServiceFromRequest(req *api.RegisterServiceRequest) (Service, error) {
diff --git a/go.mod b/go.mod
index daac4656..3d3c7f71 100644
--- a/go.mod
+++ b/go.mod
@@ -19,6 +19,7 @@ require (
github.com/multiformats/go-multiaddr v0.16.1
github.com/multiformats/go-multihash v0.2.3
github.com/prometheus/client_golang v1.23.2
+ github.com/prometheus/client_model v0.6.2
github.com/spf13/cobra v1.10.2
go.etcd.io/bbolt v1.4.3
golang.org/x/oauth2 v0.36.0
@@ -82,6 +83,7 @@ require (
github.com/multiformats/go-multistream v0.6.1 // indirect
github.com/multiformats/go-varint v0.1.0 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+ github.com/openai/openai-go v1.12.0 // indirect
github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect
github.com/pion/datachannel v1.6.0 // indirect
github.com/pion/dtls/v3 v3.1.2 // indirect
@@ -100,7 +102,6 @@ require (
github.com/pion/turn/v5 v5.0.3 // indirect
github.com/pion/webrtc/v4 v4.2.11 // indirect
github.com/polydawn/refmt v0.89.1-0.20231129105047-37766d95467a // indirect
- github.com/prometheus/client_model v0.6.2 // indirect
github.com/prometheus/common v0.67.5 // indirect
github.com/prometheus/procfs v0.20.1 // indirect
github.com/quic-go/qpack v0.6.0 // indirect
@@ -110,6 +111,10 @@ require (
github.com/segmentio/encoding v0.5.4 // indirect
github.com/spaolacci/murmur3 v1.1.0 // indirect
github.com/spf13/pflag v1.0.10 // indirect
+ github.com/tidwall/gjson v1.14.4 // indirect
+ github.com/tidwall/match v1.1.1 // indirect
+ github.com/tidwall/pretty v1.2.1 // indirect
+ github.com/tidwall/sjson v1.2.5 // indirect
github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 // indirect
github.com/wlynxg/anet v0.0.5 // indirect
github.com/yosida95/uritemplate/v3 v3.0.2 // indirect
diff --git a/go.sum b/go.sum
index d8f22dc3..1f1b278f 100644
--- a/go.sum
+++ b/go.sum
@@ -372,6 +372,8 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJEU3ofeGjhHklVoIGuVj85JJwZ6kWPaJwCIxgnFmo=
github.com/neelance/sourcemap v0.0.0-20200213170602-2833bce08e4c/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM=
+github.com/openai/openai-go v1.12.0 h1:NBQCnXzqOTv5wsgNC36PrFEiskGfO5wccfCWDo9S1U0=
+github.com/openai/openai-go v1.12.0/go.mod h1:g461MYGXEXBVdV5SaR/5tNzNbSfwTBBefwc+LlDCK0Y=
github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2DcNVpwGmV9E1BkGknEliJkfwQj0=
github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y=
@@ -480,6 +482,16 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
+github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
+github.com/tidwall/gjson v1.14.4 h1:uo0p8EbA09J7RQaflQ1aBRffTR7xedD2bcIVSYxLnkM=
+github.com/tidwall/gjson v1.14.4/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
+github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
+github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
+github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
+github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4=
+github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
+github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY=
+github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28=
github.com/urfave/cli v1.22.10/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
github.com/warpfork/go-wish v0.0.0-20220906213052-39a1cc7a02d0 h1:GDDkbFiaK8jsSDJfjId/PEGEShv6ugrt4kYsC5UIDaQ=
github.com/warpfork/go-wish v0.0.0-20220906213052-39a1cc7a02d0/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw=