Agent Mesh
Go a2aa2a-protocolagent-meshagentic-aiinferencemcpmeshmesh-networknetwork-securitynetworking-aip2p-network
ingress: deliver into a sandbox that cannot be dialled
Antonio Ojea84b1660cf7e6
diff --git a/cmd/nano-init/ingress.go b/cmd/nano-init/ingress.gonew file mode 100644index 00000000..2210ce82--- /dev/null+++ b/cmd/nano-init/ingress.go@@ -0,0 +1,172 @@+// Copyright 2026 Google LLC+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package main++import (+ "bufio"+ "context"+ "fmt"+ "io"+ "log"+ "net"+ "strconv"+ "strings"+ "sync"+ "time"+)++// An agent that serves the mesh needs traffic delivered to it, and delivery is+// the direction the sandbox is built to prevent. The gateway cannot dial the+// agent: a sandbox has a network namespace of its own, so the gateway's+// 127.0.0.1 is its own loopback and not the agent's. That is true of every+// profile -- a microVM, a container with no network, and a pod where nano-init+// made the namespace itself -- because it is a consequence of the isolation+// rather than of any one runtime.+//+// The way out is the way in. Egress already crosses the boundary over a+// pathname Unix socket, which network namespaces do not apply to because it is+// a filesystem object. So the reverse channel is another one, listened on by+// this process, which is inside the namespace and can therefore reach the+// agent at the address the gateway meant.+//+// The handshake is Firecracker's, deliberately: connect, send "CONNECT <port>",+// read "OK". A microVM can offer the identical protocol over vsock without the+// gateway learning the difference.++const (+ ingressConnectTimeout = 10 * time.Second+ // A port is at most five digits and a line at most one, so anything longer+ // is a client that has misunderstood.+ ingressMaxHandshake = 64+)++// serveIngress accepts the gateway's inbound connections and joins each one to+// the port the agent serves.+func serveIngress(ctx context.Context, socketPath string) error {+ listener, err := net.Listen("unix", socketPath)+ if err != nil {+ return fmt.Errorf("listen on the ingress socket %s: %w", socketPath, err)+ }+ go func() {+ <-ctx.Done()+ _ = listener.Close()+ }()++ log.Printf("serving ingress on %s", socketPath)+ for {+ conn, err := listener.Accept()+ if err != nil {+ if ctx.Err() != nil {+ return nil+ }+ return fmt.Errorf("accept on the ingress socket: %w", err)+ }+ go func() {+ if err := handleIngress(ctx, conn); err != nil {+ log.Printf("ingress connection: %v", err)+ }+ }()+ }+}++// handleIngress reads which port the gateway is asking for and connects it.+func handleIngress(ctx context.Context, conn net.Conn) error {+ defer func() { _ = conn.Close() }()++ _ = conn.SetReadDeadline(time.Now().Add(ingressConnectTimeout))+ reader := bufio.NewReaderSize(conn, ingressMaxHandshake)+ line, err := reader.ReadString('\n')+ if err != nil {+ return fmt.Errorf("read the ingress handshake: %w", err)+ }+ _ = conn.SetReadDeadline(time.Time{})++ port, err := parseIngressConnect(line)+ if err != nil {+ // Answered rather than dropped: a gateway that gets nothing back+ // cannot tell a refusal from a sandbox that never started.+ _, _ = io.WriteString(conn, "ERR "+err.Error()+"\n")+ return err+ }++ // The agent is in this namespace, which is the whole reason this hop+ // exists: here 127.0.0.1 means what the gateway intended.+ dialer := net.Dialer{Timeout: ingressConnectTimeout}+ agent, err := dialer.DialContext(ctx, "tcp", net.JoinHostPort("127.0.0.1", strconv.Itoa(port)))+ if err != nil {+ _, _ = io.WriteString(conn, "ERR the agent is not listening\n")+ return fmt.Errorf("dial the agent on port %d: %w", port, err)+ }+ defer func() { _ = agent.Close() }()++ if _, err := io.WriteString(conn, "OK\n"); err != nil {+ return fmt.Errorf("acknowledge the ingress handshake: %w", err)+ }++ // Anything the gateway sent after the handshake is already buffered.+ if n := reader.Buffered(); n > 0 {+ pending, err := reader.Peek(n)+ if err != nil {+ return fmt.Errorf("recover buffered request bytes: %w", err)+ }+ if _, err := agent.Write(pending); err != nil {+ return fmt.Errorf("forward buffered request bytes: %w", err)+ }+ }++ relay(conn, agent)+ return nil+}++// parseIngressConnect reads the one line the gateway sends first.+func parseIngressConnect(line string) (int, error) {+ fields := strings.Fields(strings.TrimSpace(line))+ if len(fields) != 2 || !strings.EqualFold(fields[0], "CONNECT") {+ return 0, fmt.Errorf("expected \"CONNECT <port>\"")+ }+ port, err := strconv.Atoi(fields[1])+ if err != nil || port < 1 || port > 65535 {+ return 0, fmt.Errorf("%q is not a port", fields[1])+ }+ return port, nil+}++// relay joins two connections until either end is done with the other.+func relay(a, b net.Conn) {+ var wg sync.WaitGroup+ wg.Add(2)+ go func() {+ defer wg.Done()+ _, _ = io.Copy(a, b)+ closeWrite(a)+ }()+ go func() {+ defer wg.Done()+ _, _ = io.Copy(b, a)+ closeWrite(b)+ }()+ wg.Wait()+}++// closeWrite ends one direction so the far side sees EOF, falling back to a+// full close for connections that cannot half-close.+func closeWrite(c net.Conn) {+ type closeWriter interface{ CloseWrite() error }+ if cw, ok := c.(closeWriter); ok {+ _ = cw.CloseWrite()+ return+ }+ _ = c.Close()+}diff --git a/cmd/nano-init/main.go b/cmd/nano-init/main.goindex c0ae81a5..3a01ed17 100644--- a/cmd/nano-init/main.go+++ b/cmd/nano-init/main.go@@ -48,6 +48,7 @@ import ("os""os/exec""os/signal"+ "strings""syscall""time"@@ -95,28 +96,57 @@ func main() {}case "run":- args := os.Args[2:]- createNS := false- if len(args) > 0 && args[0] == "--create-namespaces" {- createNS, args = true, args[1:]- }+ createNS, ingressSocket, args := parseRunFlags(os.Args[2:])if len(args) < 2 {- log.Fatalf("usage: %s run [--create-namespaces] <boundary-socket> <cmd> [args...]", os.Args[0])+ usage()}- run(createNS, args[0], args[1], args[2:])+ run(createNS, ingressSocket, args[0], args[1], args[2:])default:usage()}}+// parseRunFlags reads our own flags and stops at the first argument that is not+// one, because everything after that belongs to the agent and must reach it+// untouched.+func parseRunFlags(args []string) (createNS bool, ingressSocket string, rest []string) {+ for len(args) > 0 {+ switch {+ case args[0] == "--create-namespaces":+ createNS, args = true, args[1:]+ case args[0] == "--ingress-socket":+ if len(args) < 2 {+ log.Fatalf("--ingress-socket needs a path")+ }+ ingressSocket, args = args[1], args[2:]+ case strings.HasPrefix(args[0], "--ingress-socket="):+ ingressSocket, args = strings.TrimPrefix(args[0], "--ingress-socket="), args[1:]+ default:+ return createNS, ingressSocket, args+ }+ }+ return createNS, ingressSocket, nil+}++// runFlags rebuilds the arguments for the re-executed half, so it is given+// what this one was given.+func runFlags(ingressSocket, boundarySocket, cmdName string, cmdArgs []string) []string {+ args := []string{"run", "--create-namespaces"}+ if ingressSocket != "" {+ args = append(args, "--ingress-socket", ingressSocket)+ }+ args = append(args, boundarySocket, cmdName)+ return append(args, cmdArgs...)+}+func usage() {- log.Fatalf("usage:\n %s copy <dest>\n %s run [--create-namespaces] <boundary-socket> <cmd> [args...]",+ log.Fatalf("usage:\n %s copy <dest>\n %s run [--create-namespaces] [--ingress-socket <path>] <boundary-socket> <cmd> [args...]",os.Args[0], os.Args[0])}// run wires the sandbox up and hands it to the agent.-func run(createNS bool, boundarySocket, cmdName string, cmdArgs []string) {+func run(createNS bool, ingressSocket, boundarySocket, cmdName string, cmdArgs []string) {ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT)defer cancel()@@ -133,7 +163,7 @@ func run(createNS bool, boundarySocket, cmdName string, cmdArgs []string) {if err != nil {log.Fatalf("locate this binary to re-execute it: %v", err)}- args := append([]string{"run", "--create-namespaces", boundarySocket, cmdName}, cmdArgs...)+ args := runFlags(ingressSocket, boundarySocket, cmdName, cmdArgs)code, err := runAgent(ctx, cancel, self, args, withNamespaces(userNS))if err != nil {log.Fatalf("create the sandbox namespaces: %v\n%s", err, namespaceHint(err))@@ -167,6 +197,17 @@ func run(createNS bool, boundarySocket, cmdName string, cmdArgs []string) {log.Fatalf("set up sandbox network: %v", err)}+ // Started here rather than earlier because it only makes sense once the+ // sandbox exists: this is the one process that can reach the agent at the+ // address the gateway will name.+ if ingressSocket != "" {+ go func() {+ if err := serveIngress(ctx, ingressSocket); err != nil {+ log.Printf("ingress: %v", err)+ }+ }()+ }+code, err := runAgent(ctx, cancel, cmdName, cmdArgs)if err != nil {log.Fatalf("start agent: %v", err)diff --git a/cmd/sam-box/main.go b/cmd/sam-box/main.goindex 44f613a0..e53aa6d7 100644--- a/cmd/sam-box/main.go+++ b/cmd/sam-box/main.go@@ -49,6 +49,8 @@ func main() {insecure boolmetricsAddr stringlogLevel string++ agentIngressSocket string)rootCmd := &cobra.Command{@@ -76,7 +78,7 @@ func main() {if err := verifyBundleCredential(cmd.Context(), bundlePath, issuer, audience, insecure); err != nil {return err}- ingress, err := resolveIngress(bundlePath, sidecarSocket)+ ingress, err := resolveIngress(bundlePath, sidecarSocket, agentIngressSocket)if err != nil {return err}@@ -133,6 +135,7 @@ func main() {runCmd.Flags().StringVar(&audience, "credential-audience", "", "Audience an agent's credential must be scoped to; required with --bundle")runCmd.Flags().BoolVar(&insecure, "insecure-unverified-bundle", false, "Trust the bundle's declared identity without a credential to back it, letting whoever can write the file decide which agent this sandbox is")runCmd.Flags().StringVar(&metricsAddr, "metrics-addr", "", "Serve unauthenticated Prometheus metrics on this address, e.g. 127.0.0.1:9600; off by default")+ runCmd.Flags().StringVar(&agentIngressSocket, "agent-ingress-socket", "", "Path to the sandbox's reverse channel, served by nano-init --ingress-socket; required to reach an agent that serves the mesh, because an isolated sandbox cannot be dialled")runCmd.Flags().StringVar(&logLevel, "log-level", "info", "Log level (debug, info, warn, error)")for _, required := range []string{"socket", "sidecar-socket"} {if err := runCmd.MarkFlagRequired(required); err != nil {@@ -219,7 +222,7 @@ func verifyBundleCredential(ctx context.Context, bundlePath, issuer, audience st// resolveIngress prepares what the agent is permitted to serve. Nil means// nothing, which is the case for a sandbox that only calls out.-func resolveIngress(bundlePath, sidecarSocket string) (*sambox.IngressManager, error) {+func resolveIngress(bundlePath, sidecarSocket, agentIngressSocket string) (*sambox.IngressManager, error) {if bundlePath == "" {return nil, nil}@@ -230,9 +233,17 @@ func resolveIngress(bundlePath, sidecarSocket string) (*sambox.IngressManager, eif len(bundle.Ingress) == 0 {return nil, nil}+ if agentIngressSocket == "" {+ // Not fatal: an agent sharing this process's network namespace is+ // reachable without one. That is no sandbox, so it is worth saying.+ logger.Warnf("Agent %s may serve the mesh, but no --agent-ingress-socket was given: "++ "delivery will dial the agent directly, which only reaches an agent sharing this "++ "network namespace and therefore not a sandboxed one", bundle.Agent.ID)+ }logger.Infof("Agent %s may serve %d mesh service(s) once it announces them", bundle.Agent.ID, len(bundle.Ingress))return &sambox.IngressManager{SidecarSocket: sidecarSocket,Allowed: bundle.Ingress,+ AgentSocket: agentIngressSocket,}, nil}diff --git a/internal/sambox/ingress.go b/internal/sambox/ingress.goindex 124e07d2..70e3611a 100644--- a/internal/sambox/ingress.go+++ b/internal/sambox/ingress.go@@ -15,6 +15,7 @@package samboximport (+ "bufio""bytes""context""encoding/json"@@ -26,6 +27,7 @@ import ("net/http/httputil""net/url""strconv"+ "strings""sync""time"@@ -63,10 +65,20 @@ type IngressManager struct {// agent may serve nothing.Allowed []BundleIngress- // AgentAddr resolves where the agent listens inside its sandbox. It exists- // because reaching in is topology-dependent: a sandbox sharing a network- // namespace with the gateway is reachable on loopback, while an isolated- // one needs a reverse channel that does not exist yet.+ // AgentSocket is the sandbox's reverse channel: a Unix socket nano-init+ // listens on from inside the sandbox. It is how an isolated agent is+ // reached at all, because every sandbox has a network namespace of its own+ // and the gateway's 127.0.0.1 is therefore not the agent's. A pathname+ // socket crosses that boundary for the same reason the egress one does: it+ // is a filesystem object, and network namespaces do not apply to it.+ //+ // Empty means the agent shares this process's network namespace and can be+ // dialled directly, which is true of no sandboxed profile.+ AgentSocket string++ // AgentAddr resolves where the agent listens inside its sandbox. Setting it+ // overrides both of the above, which is how tests point the forwarder at a+ // server of their own.AgentAddr func(port int) stringmu sync.Mutex@@ -176,6 +188,7 @@ func (m *IngressManager) ensureListening() (string, error) {// service name the gateway added so the agent sees the path it published.func (m *IngressManager) forwarder() http.Handler {proxy := &httputil.ReverseProxy{+ Transport: m.AgentTransport(),Rewrite: func(r *httputil.ProxyRequest) {name, rest := splitServicePath(r.In.URL.Path)@@ -206,6 +219,60 @@ func (m *IngressManager) agentAddr(port int) string {return net.JoinHostPort("127.0.0.1", strconv.Itoa(port))}+// AgentTransport reaches the sandbox over its reverse channel when there is+// one, and returns nil when the agent can be dialled directly.+//+// The address the forwarder writes is still 127.0.0.1:<port>, because that is+// what the port means where it is going. Only the dialling changes: the port is+// carried in the handshake and the connection is made by the process inside the+// sandbox, which is the one that can.+func (m *IngressManager) AgentTransport() http.RoundTripper {+ if m.AgentSocket == "" {+ return nil // the default transport dials the address directly+ }+ socket := m.AgentSocket+ return &http.Transport{+ DialContext: func(ctx context.Context, _, addr string) (net.Conn, error) {+ _, port, err := net.SplitHostPort(addr)+ if err != nil {+ return nil, fmt.Errorf("ingress target %q: %w", addr, err)+ }+ return dialSandbox(ctx, socket, port)+ },+ }+}++// dialSandbox opens one connection through the sandbox's reverse channel.+//+// The handshake is Firecracker's -- "CONNECT <port>", then "OK" -- so a microVM+// can offer the same protocol over vsock and nothing here has to know which+// kind of sandbox it is talking to.+func dialSandbox(ctx context.Context, socket, port string) (net.Conn, error) {+ var d net.Dialer+ conn, err := d.DialContext(ctx, "unix", socket)+ if err != nil {+ return nil, fmt.Errorf("reach the sandbox's ingress socket %s: %w", socket, err)+ }+ if deadline, ok := ctx.Deadline(); ok {+ _ = conn.SetDeadline(deadline)+ }+ if _, err := fmt.Fprintf(conn, "CONNECT %s\n", port); err != nil {+ _ = conn.Close()+ return nil, fmt.Errorf("ask the sandbox for port %s: %w", port, err)+ }+ reply, err := bufio.NewReader(io.LimitReader(conn, 128)).ReadString('\n')+ if err != nil {+ _ = conn.Close()+ return nil, fmt.Errorf("read the sandbox's answer for port %s: %w", port, err)+ }+ if strings.TrimSpace(reply) != "OK" {+ _ = conn.Close()+ return nil, fmt.Errorf("the sandbox refused port %s: %s", port, strings.TrimSpace(reply))+ }+ _ = conn.SetDeadline(time.Time{})+ return conn, nil+}+// register tells the node to advertise the service, with a target only the// gateway could have chosen.func (m *IngressManager) register(ctx context.Context, decl ingressDeclaration, target string) error {diff --git a/tests/integration/sandbox_ingress_test.go b/tests/integration/sandbox_ingress_test.gonew file mode 100644index 00000000..8886df9a--- /dev/null+++ b/tests/integration/sandbox_ingress_test.go@@ -0,0 +1,189 @@+// Copyright 2026 Google LLC+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package integration_test++import (+ "context"+ "fmt"+ "io"+ "net"+ "net/http"+ "os"+ "os/exec"+ "path/filepath"+ "strconv"+ "strings"+ "sync"+ "testing"+ "time"++ "github.com/google/sam/internal/sambox"+)++// TestSandboxServesThroughTheReverseChannel covers the hop that the rest of the+// ingress tests replace.+//+// Those tests set AgentAddr to a server of their own, which exercises the+// declaration, the policy check and the forwarding logic but skips the only+// part that depends on where the agent actually is. In every sandbox profile+// the agent has a network namespace of its own, so the gateway's 127.0.0.1 is+// its own loopback: dialling the agent directly reaches nothing, and no test+// noticed because no test dialled.+//+// Here nano-init builds a real sandbox, the agent listens inside it, and the+// gateway reaches it the only way it can -- over a Unix socket that nano-init+// serves from inside the namespace, because a pathname socket is a filesystem+// object and network namespaces do not apply to it.+func TestSandboxServesThroughTheReverseChannel(t *testing.T) {+ const served = "the agent answered from inside its sandbox"++ if _, err := os.Stat("/dev/net/tun"); err != nil {+ t.Skipf("a sandbox needs /dev/net/tun: %v", err)+ }+ nanoInitBin := buildBinary(t, "./cmd/nano-init")++ sockDir, err := os.MkdirTemp("", "ingress")+ if err != nil {+ t.Fatalf("MkdirTemp: %v", err)+ }+ defer func() { _ = os.RemoveAll(sockDir) }()++ boundarySocket := filepath.Join(sockDir, "agent.sock")+ ingressSocket := filepath.Join(sockDir, "ingress.sock")++ // The sandbox needs a boundary to exist before it will start, though this+ // test never sends anything out through it.+ boundary, err := sambox.ListenSandboxSocket(boundarySocket)+ if err != nil {+ t.Fatalf("ListenSandboxSocket: %v", err)+ }+ defer func() { _ = boundary.Close() }()++ // A port inside the sandbox, which is a different 127.0.0.1 from this+ // process's and is the whole point of the exercise.+ const agentPort = 18080+ agent := fmt.Sprintf(+ `while true; do printf 'HTTP/1.1 200 OK\r\nContent-Length: %d\r\nConnection: close\r\n\r\n%s' | nc -l -p %d; done`,+ len(served), served, agentPort)++ ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)+ defer cancel()++ sandbox := exec.CommandContext(ctx, nanoInitBin, "run",+ "--create-namespaces", "--ingress-socket", ingressSocket,+ boundarySocket, "sh", "-c", agent)+ // One writer, two streams: exec copies stdout and stderr on separate+ // goroutines, so the sink has to tolerate both.+ out := &lockedBuilder{}+ sandbox.Stdout, sandbox.Stderr = out, out+ if err := sandbox.Start(); err != nil {+ t.Fatalf("start the sandbox: %v", err)+ }+ defer func() {+ _ = sandbox.Process.Kill()+ _, _ = sandbox.Process.Wait()+ }()++ // The reverse channel appears once the sandbox is up.+ deadline := time.Now().Add(30 * time.Second)+ for time.Now().Before(deadline) {+ if _, err := os.Stat(ingressSocket); err == nil {+ break+ }+ if strings.Contains(out.String(), "user namespaces are disabled") {+ t.Skipf("this host does not allow the namespaces a sandbox needs:\n%s", out.String())+ }+ time.Sleep(200 * time.Millisecond)+ }+ if _, err := os.Stat(ingressSocket); err != nil {+ t.Fatalf("the sandbox never served its ingress socket: %v\n%s", err, out.String())+ }++ // What the gateway used to do, and what every ingress test still simulates+ // by supplying an address of its own. The agent is listening on this port,+ // but in a namespace this process cannot see, so the dial must fail --+ // otherwise the reverse channel below would be proving nothing.+ direct, err := net.DialTimeout("tcp",+ net.JoinHostPort("127.0.0.1", strconv.Itoa(agentPort)), 2*time.Second)+ if err == nil {+ _ = direct.Close()+ t.Fatalf("dialling 127.0.0.1:%d from outside reached something: the sandbox is not isolated, "++ "so this test cannot show that the reverse channel is what carries ingress", agentPort)+ }++ // Exactly how the gateway is configured in a pod: it knows the socket, not+ // the agent's whereabouts.+ manager := &sambox.IngressManager{AgentSocket: ingressSocket}+ client := &http.Client{Transport: managerTransport(t, manager)}++ var resp *http.Response+ for time.Now().Before(deadline) {+ req, reqErr := http.NewRequestWithContext(ctx, http.MethodGet,+ fmt.Sprintf("http://127.0.0.1:%d/", agentPort), nil)+ if reqErr != nil {+ t.Fatalf("build the request: %v", reqErr)+ }+ // The agent inside serves one connection at a time, so reusing one+ // would arrive at a listener that has already moved on.+ req.Close = true+ resp, err = client.Do(req)+ if err == nil {+ break+ }+ time.Sleep(200 * time.Millisecond)+ }+ if err != nil {+ t.Fatalf("reaching the agent through the reverse channel: %v\nsandbox said:\n%s", err, out.String())+ }+ defer func() { _ = resp.Body.Close() }()++ body, err := io.ReadAll(resp.Body)+ if err != nil {+ t.Fatalf("read the agent's answer: %v", err)+ }+ if !strings.Contains(string(body), served) {+ t.Errorf("the agent answered %q, want %q", string(body), served)+ }+}++// managerTransport exposes the transport the forwarder would use, so the test+// drives the same dialling the gateway does rather than a copy of it.+func managerTransport(t *testing.T, m *sambox.IngressManager) http.RoundTripper {+ t.Helper()+ tr := m.AgentTransport()+ if tr == nil {+ t.Fatal("no transport: with AgentSocket set the manager must dial the sandbox")+ }+ return tr+}++// lockedBuilder collects a subprocess's stdout and stderr, which arrive on+// different goroutines.+type lockedBuilder struct {+ mu sync.Mutex+ b strings.Builder+}++func (l *lockedBuilder) Write(p []byte) (int, error) {+ l.mu.Lock()+ defer l.mu.Unlock()+ return l.b.Write(p)+}++func (l *lockedBuilder) String() string {+ l.mu.Lock()+ defer l.mu.Unlock()+ return l.b.String()+}