Skip to content

Agent Mesh

sdk/meshnet: harden WhoIs, in-memory store, StateDir lock, and lifecycle safety

Antonio Ojea9d75f7bd9681
diff --git a/internal/node/store.go b/internal/node/store.go
index c2307cf9..84c7b981 100644
--- a/internal/node/store.go
+++ b/internal/node/store.go
@@ -23,6 +23,7 @@ import (
"path/filepath"
"strconv"
"strings"
+ "sync"
"time"
"github.com/google/agentmesh/api"
@@ -64,21 +65,36 @@ var legacyKeys = []string{
// Store is the node's persistent state: its private key and its
// api.MemberCredential, in a bbolt database whose file lock also tells a
-// second agentmesh-node that this data directory is in use.
+// second agentmesh-node that this data directory is in use (or purely in memory
+// when constructed with NewMemoryStore).
type Store struct {
db *bbolt.DB
+ memMu sync.RWMutex
+ memKey []byte
+ memCred *api.MemberCredential
onChange func() error
}
+// NewMemoryStore returns an ephemeral in-memory Store that creates no files on
+// disk.
+func NewMemoryStore() *Store {
+ return &Store{memCred: &api.MemberCredential{}}
+}
+
// SetOnChange registers an optional post-commit callback invoked after key or
// credential mutations complete.
func (s *Store) SetOnChange(fn func() error) {
+ s.memMu.Lock()
s.onChange = fn
+ s.memMu.Unlock()
}
func (s *Store) notifyChange() error {
- if s.onChange != nil {
- return s.onChange()
+ s.memMu.RLock()
+ fn := s.onChange
+ s.memMu.RUnlock()
+ if fn != nil {
+ return fn()
}
return nil
}
@@ -223,6 +239,12 @@ func getCredential(b *bbolt.Bucket) (*api.MemberCredential, error) {
// view reads the credential; an empty message stands for a node that has not enrolled.
func (s *Store) view(fn func(c *api.MemberCredential) error) error {
+ if s.db == nil {
+ s.memMu.RLock()
+ cloned := proto.Clone(s.memCred).(*api.MemberCredential)
+ s.memMu.RUnlock()
+ return fn(cloned)
+ }
return s.db.View(func(tx *bbolt.Tx) error {
c, err := getCredential(tx.Bucket([]byte(bucketIdentity)))
if err != nil {
@@ -234,6 +256,19 @@ func (s *Store) view(fn func(c *api.MemberCredential) error) error {
// update applies fn to the credential and writes it back in one transaction.
func (s *Store) update(fn func(c *api.MemberCredential) error) error {
+ if s.db == nil {
+ s.memMu.Lock()
+ next := proto.Clone(s.memCred).(*api.MemberCredential)
+ err := fn(next)
+ if err == nil {
+ s.memCred = next
+ }
+ s.memMu.Unlock()
+ if err != nil {
+ return err
+ }
+ return s.notifyChange()
+ }
if err := s.db.Update(func(tx *bbolt.Tx) error {
b := tx.Bucket([]byte(bucketIdentity))
c, err := getCredential(b)
@@ -267,6 +302,12 @@ func (s *Store) SetCredential(c *api.MemberCredential) error {
if c == nil {
c = &api.MemberCredential{}
}
+ if s.db == nil {
+ s.memMu.Lock()
+ s.memCred = proto.Clone(c).(*api.MemberCredential)
+ s.memMu.Unlock()
+ return s.notifyChange()
+ }
if err := s.db.Update(func(tx *bbolt.Tx) error {
return putCredential(tx.Bucket([]byte(bucketIdentity)), c)
}); err != nil {
@@ -361,6 +402,12 @@ func (s *Store) LoadOIDCConfig() (string, string, string, error) {
}
func (s *Store) SaveKey(key []byte) error {
+ if s.db == nil {
+ s.memMu.Lock()
+ s.memKey = append([]byte(nil), key...)
+ s.memMu.Unlock()
+ return s.notifyChange()
+ }
if err := s.db.Update(func(tx *bbolt.Tx) error {
return tx.Bucket([]byte(bucketIdentity)).Put([]byte(keyPrivKey), key)
}); err != nil {
@@ -370,6 +417,12 @@ func (s *Store) SaveKey(key []byte) error {
}
func (s *Store) LoadKey() ([]byte, error) {
+ if s.db == nil {
+ s.memMu.RLock()
+ val := append([]byte(nil), s.memKey...)
+ s.memMu.RUnlock()
+ return val, nil
+ }
var val []byte
_ = s.db.View(func(tx *bbolt.Tx) error {
val = append([]byte(nil), tx.Bucket([]byte(bucketIdentity)).Get([]byte(keyPrivKey))...)
@@ -481,6 +534,12 @@ func (s *Store) LoadControlPlaneURL() (string, error) {
// a different mesh. It keeps the long-lived libp2p key (node_private_key), so
// the node's PeerID survives the switch.
func (s *Store) ResetMeshIdentity() error {
+ if s.db == nil {
+ s.memMu.Lock()
+ s.memCred = &api.MemberCredential{}
+ s.memMu.Unlock()
+ return s.notifyChange()
+ }
if err := s.db.Update(func(tx *bbolt.Tx) error {
return tx.Bucket([]byte(bucketIdentity)).Delete([]byte(keyCredential))
}); err != nil {
@@ -537,6 +596,9 @@ func containsKeyBytes(keys [][]byte, want []byte) bool {
}
func (s *Store) Close() error {
+ if s.db == nil {
+ return nil
+ }
return s.db.Close()
}
diff --git a/internal/node/store_test.go b/internal/node/store_test.go
index bd4627e2..bbce4585 100644
--- a/internal/node/store_test.go
+++ b/internal/node/store_test.go
@@ -617,3 +617,79 @@ func TestStore_SetOnChange(t *testing.T) {
t.Fatalf("SaveRefreshToken with failing onChange = %v, want %v", err, hookErr)
}
}
+
+func TestStore_NewMemoryStore(t *testing.T) {
+ emptyTmp := t.TempDir()
+ t.Setenv("TMPDIR", emptyTmp)
+
+ store := NewMemoryStore()
+ defer func() {
+ if err := store.Close(); err != nil {
+ t.Errorf("Close: %v", err)
+ }
+ }()
+
+ var mirroredKey []byte
+ var mirroredCred *api.MemberCredential
+ store.SetOnChange(func() error {
+ k, err := store.LoadKey()
+ if err != nil {
+ return err
+ }
+ c, err := store.Credential()
+ if err != nil {
+ return err
+ }
+ mirroredKey = k
+ mirroredCred = c
+ return nil
+ })
+
+ key := []byte("mem-private-key")
+ if err := store.SaveKey(key); err != nil {
+ t.Fatalf("SaveKey: %v", err)
+ }
+ key[0] = 'X' // verify defensive copy
+ gotKey, err := store.LoadKey()
+ if err != nil || string(gotKey) != "mem-private-key" {
+ t.Fatalf("LoadKey = %q, %v; want mem-private-key", gotKey, err)
+ }
+ if string(mirroredKey) != "mem-private-key" {
+ t.Fatalf("mirroredKey = %q, want mem-private-key", mirroredKey)
+ }
+
+ cpPub := bytes.Repeat([]byte{9}, ed25519.PublicKeySize)
+ if err := store.SaveIdentity([]byte("mem-biscuit")); err != nil {
+ t.Fatalf("SaveIdentity: %v", err)
+ }
+ if err := store.SaveIdentityExpiration(1790000000); err != nil {
+ t.Fatalf("SaveIdentityExpiration: %v", err)
+ }
+ if err := store.SaveMeshConfig(cpPub, []string{"/ip4/127.0.0.1/tcp/4001"}); err != nil {
+ t.Fatalf("SaveMeshConfig: %v", err)
+ }
+ if err := store.SaveRefreshToken("rt-mem"); err != nil {
+ t.Fatalf("SaveRefreshToken: %v", err)
+ }
+ if mirroredCred == nil || mirroredCred.GetOidcSession().GetRefreshToken() != "rt-mem" {
+ t.Fatalf("mirroredCred refresh_token = %q, want rt-mem", mirroredCred.GetOidcSession().GetRefreshToken())
+ }
+
+ if err := store.ResetMeshIdentity(); err != nil {
+ t.Fatalf("ResetMeshIdentity: %v", err)
+ }
+ if _, err := store.LoadIdentity(); err == nil {
+ t.Fatal("LoadIdentity succeeded after ResetMeshIdentity, want error")
+ }
+ if afterResetKey, _ := store.LoadKey(); string(afterResetKey) != "mem-private-key" {
+ t.Fatalf("LoadKey after ResetMeshIdentity = %q, want mem-private-key", afterResetKey)
+ }
+
+ entries, err := os.ReadDir(emptyTmp)
+ if err != nil {
+ t.Fatalf("ReadDir(TMPDIR): %v", err)
+ }
+ if len(entries) != 0 {
+ t.Fatalf("NewMemoryStore created %d entries in TMPDIR (%v), want 0", len(entries), entries)
+ }
+}
diff --git a/sdk/meshnet/README.md b/sdk/meshnet/README.md
index 13a22bcf..31696032 100644
--- a/sdk/meshnet/README.md
+++ b/sdk/meshnet/README.md
@@ -6,10 +6,10 @@ For a full walkthrough, see the [Embedded Go SDK guide](../../site/content/docs/
## Capabilities
-- **Lifecycle & State Compatibility (`Server.Up`, `WriteStateDir`, `ReadStateDir`):** Built directly on `internal/node.AgentMeshNode` so embedded Go applications and `agentmesh-node` share one PEP. Resumes cached credentials from `StateDir` (including public-client `OIDCSession` recovery that persists rotated `refresh_token`s before `POST /register`), enrolls via `BootstrapTokenPath` / `BootstrapToken` / `JWTPath` / `TokenSource`, returns from `Up` as soon as the first router admits the member while remaining dials up to `Routers` finish in the background, refreshes credentials proactively, backs off exponentially (`2s` to `5m`) and signals `Done()` / `Err()` / `OnCredentialLapse` (`ErrCredentialLapsed`, `Status.CredentialLapsed`) if a credential lapses, verifies signed `/keys` sets, and enforces live `/mesh/events/v1` FloodSub revocations and bans.
+- **Lifecycle & State Compatibility (`Server.Up`, `WriteStateDir`, `ReadStateDir`):** Built directly on `internal/node.AgentMeshNode` so embedded Go applications and `agentmesh-node` share one PEP. Holds an exclusive non-blocking lock (`ErrStateDirLocked`) on `StateDir` while running (or runs purely in memory with zero disk files when `StateDir` is empty), resumes cached credentials from `StateDir` (including public-client `OIDCSession` recovery that persists rotated `refresh_token`s before `POST /register`), enrolls via `BootstrapTokenPath` / `BootstrapToken` / `JWTPath` / `TokenSource`, returns from `Up` as soon as the first router admits the member while remaining dials up to `Routers` finish in the background, refreshes credentials proactively, backs off exponentially (`2s` to `5m`) and signals `Done()` / `Err()` / `OnCredentialLapse` (`ErrCredentialLapsed`, `Status.CredentialLapsed`) if a credential lapses, verifies signed `/keys` sets, and enforces live `/mesh/events/v1` FloodSub revocations and bans.
- **In-Process A2A Serving (`Server.ListenA2A`):** Announces `a2a://<name>` to the router directory (unless `NoAnnounce: true`) and returns in-memory `net.Listener`s (`PrivateListener` over `/libp2p-http` and optional `PublicListener` over `/mesh/ingress/1.0.0` in `EDGE` or `PASSTHROUGH` mode) plus canonical `PrivateURL` and `PublicURL` endpoints (`PublicURL` is `nil` when `NoAnnounce: true`, when no router advertises an `ingress_url`, or when the hub policy ceiling withholds public exposure). Automatically relativizes and rewrites A2A `AgentCard` `supportedInterfaces` URLs per caller.
- **Per-Server Prometheus Metrics (`Server.Registerer`):** Optionally registers `peer_id`-labeled state gauges and refusal/ingress counters (`agentmesh_meshnet_*`) so multiple `Server` instances can share a `prometheus.Registerer` without collision, unregistering all collectors on `Close`.
-- **Unspoofable Caller Inspection (`Server.WhoIs`):** Maps `http.Request.RemoteAddr` on `PrivateListener` and `PublicListener` to the cryptographically verified `*CallerInfo` (`PeerID`, `Principal`, `Roles`, `Labels`, `Task`) without trusting HTTP headers.
+- **Unspoofable Caller Inspection (`Server.WhoIs`):** Maps `http.Request.RemoteAddr` on `PrivateListener` and `PublicListener` to the cryptographically verified `*CallerInfo` (`PeerID`, `Principal`, `Roles`, `Labels`, `Task`) without trusting HTTP headers. Returns `(nil, false)` for unauthenticated external requests (`INGRESS_AUTH_MODE_PASSTHROUGH` and anonymous `GET /.well-known/agent-card.json` discovery), and sets `PeerID` to `""` for verified external `EDGE` callers without a mesh node identity (such as OIDC users).
- **Outbound Mesh `*http.Client` (`Server.HTTPClient`, `Server.Discover`):** Routes `http://agentmesh.local/mesh/<peer_id>/<type>/<name>[/<rest>]` over authenticated libp2p streams, verifying provider Biscuits, `allowed_publish` Datalog grants, `EgressRequireLabels`, and `X-Mesh-Required-Labels` before sending request bytes.
- **Offline Task-Scoped Attenuation (`Server.AttenuateToken`):** Appends a validated `TaskAuthorizationRule` (`tar_block` fact, 0 Datalog rules, 0 checks) and optionally seals the Biscuit in memory to delegate task-scoped credentials to sub-agents or external workers without their own mesh client.
- **Local Progressive-Discovery MCP Server (`Server.MCPServer`, `Server.MCPHandler`):** Exposes the 6 mesh-operating tools (`list_local_services`, `discover_remote_services`, `get_mesh_info`, `find_remote_tools`, `describe_remote_tool`, `call_remote_tool`) in-process or over Streamable HTTP (for example over a local Unix domain socket).
diff --git a/sdk/meshnet/a2a.go b/sdk/meshnet/a2a.go
index 5fbac58a..08054858 100644
--- a/sdk/meshnet/a2a.go
+++ b/sdk/meshnet/a2a.go
@@ -18,6 +18,7 @@ import (
"bytes"
"cmp"
"context"
+ "crypto/ed25519"
"encoding/json"
"errors"
"fmt"
@@ -37,6 +38,7 @@ import (
"github.com/a2aproject/a2a-go/v2/a2a"
"github.com/google/agentmesh/api"
"github.com/google/agentmesh/internal/node"
+ "github.com/libp2p/go-libp2p/core/crypto"
"github.com/libp2p/go-libp2p/core/peer"
)
@@ -221,6 +223,7 @@ func (svc *meshnetA2AService) Handler() http.Handler {
targetLn := svc.privateLn
if externalIngress {
if !svc.ExternalIngressAllowed() {
+ svc.s.log().Debug("refusing external ingress request: service ingress not allowed or inactive", "service", svc.name)
http.Error(w, "Service not found", http.StatusNotFound)
return
}
@@ -236,6 +239,12 @@ func (svc *meshnetA2AService) Handler() http.Handler {
// keyed by http.Request.RemoteAddr. Because connections are in-memory net.Pipe
// pairs tracked in Server's active table, WhoIs cannot be spoofed by HTTP
// headers.
+//
+// WhoIs returns (nil, false) for unauthenticated external requests, such as
+// INGRESS_AUTH_MODE_PASSTHROUGH requests and anonymous
+// GET /.well-known/agent-card.json discovery requests. For verified external
+// EDGE callers that carry a user Biscuit rather than a mesh node identity,
+// CallerInfo.PeerID is empty.
func (s *Server) WhoIs(remoteAddr string) (*CallerInfo, bool) {
v, ok := s.activeConns.Load(remoteAddr)
if !ok {
@@ -258,9 +267,11 @@ func (s *Server) WhoIs(remoteAddr string) (*CallerInfo, bool) {
// it to the mesh directory (unless NoAnnounce is true), and returns the
// in-memory listeners and canonical URLs to pass to http.Server.Serve.
func (s *Server) ListenA2A(ctx context.Context, opts A2AOptions) (*Endpoints, error) {
- if !s.isRunning() || s.node.Services() == nil {
+ n := s.runningNode()
+ if n == nil || n.Services() == nil {
return nil, errors.New("meshnet: Server is not running (call Up first)")
}
+ _, _, peerID, _ := s.stateSnapshot()
name := strings.TrimSpace(opts.Name)
if name == "" {
return nil, errors.New("meshnet: A2AOptions.Name is required")
@@ -268,7 +279,7 @@ func (s *Server) ListenA2A(ctx context.Context, opts A2AOptions) (*Endpoints, er
if err := api.ValidateServiceFormat(api.ServiceTypeStringA2A + "://" + name); err != nil {
return nil, fmt.Errorf("meshnet: invalid A2A service name %q: %w", name, err)
}
- if s.node.IsServiceRegistered(name) {
+ if n.IsServiceRegistered(name) {
return nil, fmt.Errorf("meshnet: A2A service %q is already registered", name)
}
@@ -297,7 +308,7 @@ func (s *Server) ListenA2A(ctx context.Context, opts A2AOptions) (*Endpoints, er
}
}
- privURL, err := url.Parse(fmt.Sprintf("http://agentmesh.local/mesh/%s/%s/%s", s.peerID.String(), api.ServiceTypeStringA2A, name))
+ privURL, err := url.Parse(fmt.Sprintf("http://agentmesh.local/mesh/%s/%s/%s", peerID.String(), api.ServiceTypeStringA2A, name))
if err != nil {
return nil, err
}
@@ -319,7 +330,7 @@ func (s *Server) ListenA2A(ctx context.Context, opts A2AOptions) (*Endpoints, er
privateURL: privURL,
}
- if err := s.node.Services().Register(ctx, svc); err != nil {
+ if err := n.Services().Register(ctx, svc); err != nil {
_ = svc.Teardown()
return nil, fmt.Errorf("meshnet: register A2A service %q: %w", name, err)
}
@@ -327,11 +338,11 @@ func (s *Server) ListenA2A(ctx context.Context, opts A2AOptions) (*Endpoints, er
// Announce this member's peer record immediately alongside its service keys
// so peers discovering the new service can resolve its addresses without
// waiting for the initial reprovide delay.
- if !opts.NoAnnounce && s.node.Directory != nil {
+ if !opts.NoAnnounce && n.Directory != nil {
peerCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
- if err := s.node.Directory.Provide(peerCtx, api.PeerKey(s.peerID), ""); err != nil && ctx.Err() == nil {
+ if err := n.Directory.Provide(peerCtx, api.PeerKey(peerID), ""); err != nil && ctx.Err() == nil {
s.log().Debug("peer directory announcement skipped or failed during ListenA2A",
- "peer_id", s.peerID.String(),
+ "peer_id", peerID.String(),
"service", name,
"error", err,
)
@@ -354,7 +365,7 @@ func (s *Server) ListenA2A(ctx context.Context, opts A2AOptions) (*Endpoints, er
closeOnce.Do(func() {
uCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
- if uErr := s.node.UnregisterService(uCtx, name); uErr != nil {
+ if uErr := n.UnregisterService(uCtx, name); uErr != nil {
s.log().Warn("failed unregistering A2A service", "service", name, "error", uErr)
}
_ = svc.Teardown()
@@ -372,14 +383,15 @@ func (s *Server) ListenA2A(ctx context.Context, opts A2AOptions) (*Endpoints, er
}
func (s *Server) computePublicURL(name string) *url.URL {
- if s.node == nil {
+ n, _, peerID, _ := s.stateSnapshot()
+ if n == nil {
return nil
}
- base := s.node.PreferredRouterIngressURL()
+ base := n.PreferredRouterIngressURL()
if base == "" {
return nil
}
- raw := fmt.Sprintf("%s/%s/%s/%s", base, s.peerID.String(), api.ServiceTypeStringA2A, name)
+ raw := fmt.Sprintf("%s/%s/%s/%s", base, peerID.String(), api.ServiceTypeStringA2A, name)
u, err := url.Parse(raw)
if err != nil {
s.log().Warn("failed parsing router ingress URL", "service", name, "raw_url", raw, "error", err)
@@ -389,20 +401,21 @@ func (s *Server) computePublicURL(name string) *url.URL {
}
func (s *Server) candidateCardRoots(serviceName string) []string {
- roots := []string{"/mesh/" + s.peerID.String() + "/" + api.ServiceTypeStringA2A + "/" + serviceName}
+ n, _, peerID, _ := s.stateSnapshot()
+ roots := []string{"/mesh/" + peerID.String() + "/" + api.ServiceTypeStringA2A + "/" + serviceName}
if pubURL := s.computePublicURL(serviceName); pubURL != nil {
if p := strings.TrimSuffix(pubURL.EscapedPath(), "/"); p != "" && !slices.Contains(roots, p) {
roots = append(roots, p)
}
}
- if s.node != nil {
- for _, base := range s.node.RouterIngressURLs() {
+ if n != nil {
+ for _, base := range n.RouterIngressURLs() {
u, err := url.Parse(base)
if err != nil {
s.log().Debug("ignoring invalid router ingress base URL when computing card roots", "base_url", base, "error", err)
continue
}
- p := strings.TrimSuffix(u.EscapedPath(), "/") + "/" + s.peerID.String() + "/" + api.ServiceTypeStringA2A + "/" + serviceName
+ p := strings.TrimSuffix(u.EscapedPath(), "/") + "/" + peerID.String() + "/" + api.ServiceTypeStringA2A + "/" + serviceName
if !slices.Contains(roots, p) {
roots = append(roots, p)
}
@@ -426,8 +439,14 @@ func (s *Server) deliverHTTP(
}
seq := s.nextConnSeq.Add(1)
- remoteKey := fmt.Sprintf("meshnet:%s#%d", caller.PeerID, seq)
- s.activeConns.Store(remoteKey, caller)
+ peerLabel := "unverified"
+ if caller != nil {
+ peerLabel = cmp.Or(caller.PeerID, "external")
+ }
+ remoteKey := fmt.Sprintf("meshnet:%s#%d", peerLabel, seq)
+ if caller != nil {
+ s.activeConns.Store(remoteKey, caller)
+ }
clientSide, serverSide := net.Pipe()
serverConn := &trackedConn{
@@ -435,7 +454,9 @@ func (s *Server) deliverHTTP(
local: ln.Addr(),
remote: meshnetAddr(remoteKey),
onClose: func() {
- s.activeConns.Delete(remoteKey)
+ if caller != nil {
+ s.activeConns.Delete(remoteKey)
+ }
},
}
@@ -451,7 +472,9 @@ func (s *Server) deliverHTTP(
dialUsed = true
if eErr := ln.enqueue(dCtx, serverConn); eErr != nil {
_ = clientSide.Close()
- s.activeConns.Delete(remoteKey)
+ if caller != nil {
+ s.activeConns.Delete(remoteKey)
+ }
err = eErr
return
}
@@ -471,7 +494,9 @@ func (s *Server) deliverHTTP(
if !dialUsed {
_ = clientSide.Close()
_ = serverConn.Close()
- s.activeConns.Delete(remoteKey)
+ if caller != nil {
+ s.activeConns.Delete(remoteKey)
+ }
}
}()
@@ -486,6 +511,7 @@ func (s *Server) deliverHTTP(
pr.Out.Header.Del(api.HeaderMeshNoTrailingSlash)
},
ErrorHandler: func(rw http.ResponseWriter, _ *http.Request, err error) {
+ s.log().Warn("A2A reverse proxy stream delivery failed", "remote", remoteKey, "error", err)
if errors.Is(err, net.ErrClosed) {
http.Error(rw, "Service listener closed", http.StatusServiceUnavailable)
return
@@ -496,35 +522,65 @@ func (s *Server) deliverHTTP(
proxy.ServeHTTP(w, r)
}
+func canonicalPeerID(raw string) string {
+ if raw == "" {
+ return ""
+ }
+ if id, err := peer.Decode(raw); err == nil {
+ return id.String()
+ }
+ return raw
+}
+
+// authorityKeyPeerID returns the synthetic peer.ID derived from a Biscuit
+// authority Ed25519 public key. Control-plane /oauth/token exchanges without an
+// admitted actor node bind actor_node/client_peer_id to this synthetic ID; it
+// is not a mesh peer and must not be reported as CallerInfo.PeerID.
+func authorityKeyPeerID(pub ed25519.PublicKey) string {
+ if len(pub) == 0 {
+ return ""
+ }
+ ck, err := crypto.UnmarshalEd25519PublicKey(pub)
+ if err != nil {
+ return ""
+ }
+ id, err := peer.IDFromPublicKey(ck)
+ if err != nil {
+ return ""
+ }
+ return id.String()
+}
+
func (s *Server) callerInfoFromRequest(r *http.Request) (*CallerInfo, bool) {
externalIngress := r.Header.Get(api.HeaderMeshIngress) != ""
- caller := &CallerInfo{
- PeerID: r.Header.Get(api.HeaderPeerID),
- }
- if raw := node.CallerBiscuitFromContext(r.Context()); len(raw) > 0 && s.node != nil {
- if claims, err := s.node.VerifyLocalBiscuit(raw); err == nil && claims != nil {
- caller.Principal = claims.Principal()
- caller.Roles = slices.Clone(claims.Roles)
- caller.Labels = maps.Clone(claims.Labels)
- caller.Task = claims.InnermostTaskName()
- if externalIngress {
- if actor := cmp.Or(claims.ActorNodePeerID, claims.NodePeerID); actor != "" {
- if id, pErr := peer.Decode(actor); pErr == nil {
- caller.PeerID = id.String()
- } else {
- caller.PeerID = actor
- }
- }
- }
+ raw := node.CallerBiscuitFromContext(r.Context())
+ n, _, _, _ := s.stateSnapshot()
+ if len(raw) == 0 || n == nil {
+ return nil, externalIngress
+ }
+ claims, err := n.VerifyLocalBiscuit(raw)
+ if err != nil || claims == nil {
+ return nil, externalIngress
+ }
+
+ var peerID string
+ if externalIngress {
+ if claims.NodePeerID != "" {
+ peerID = canonicalPeerID(claims.NodePeerID)
+ } else if actorID := canonicalPeerID(claims.ActorNodePeerID); actorID != "" && actorID != authorityKeyPeerID(claims.VerifyingKey) {
+ peerID = actorID
}
+ } else {
+ peerID = canonicalPeerID(cmp.Or(claims.ClientPeerID, claims.NodePeerID, claims.ActorNodePeerID, r.Header.Get(api.HeaderPeerID)))
}
- if caller.Principal == "" {
- caller.Principal = r.Header.Get(api.HeaderMeshPrincipal)
- }
- if caller.Task == "" {
- caller.Task = r.Header.Get(api.HeaderMeshTask)
- }
- return caller, externalIngress
+
+ return &CallerInfo{
+ PeerID: peerID,
+ Principal: cmp.Or(claims.Principal(), r.Header.Get(api.HeaderMeshPrincipal)),
+ Roles: slices.Clone(claims.Roles),
+ Labels: maps.Clone(claims.Labels),
+ Task: cmp.Or(claims.InnermostTaskName(), r.Header.Get(api.HeaderMeshTask)),
+ }, externalIngress
}
const (
diff --git a/sdk/meshnet/httpclient.go b/sdk/meshnet/httpclient.go
index a74d085a..cbbdd1f0 100644
--- a/sdk/meshnet/httpclient.go
+++ b/sdk/meshnet/httpclient.go
@@ -49,7 +49,8 @@ type meshRoundTripper struct {
func (rt *meshRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
s := rt.s
- if !s.isRunning() {
+ n := s.runningNode()
+ if n == nil {
return nil, errors.New("meshnet: Server is not running (call Up first)")
}
if req.URL == nil {
@@ -68,7 +69,8 @@ func (rt *meshRoundTripper) RoundTrip(req *http.Request) (*http.Response, error)
if len(parts) < 3 || parts[0] == "" || parts[1] == "" || parts[2] == "" {
return httpErrorResponse(req, http.StatusBadRequest, "Invalid mesh path: expected /mesh/<peer_id>/<type>/<name>"), nil
}
- if _, err := peer.Decode(parts[0]); err != nil {
+ pid, err := peer.Decode(parts[0])
+ if err != nil {
return httpErrorResponse(req, http.StatusBadRequest, "Invalid peer ID"), nil
}
svcType, err := api.ParseServiceType(strings.ToLower(parts[1]))
@@ -77,6 +79,19 @@ func (rt *meshRoundTripper) RoundTrip(req *http.Request) (*http.Response, error)
}
outReq := req.Clone(req.Context())
+ rawPeerSeg := parts[0]
+ canonPeerSeg := pid.String()
+ if rawPeerSeg != canonPeerSeg {
+ outReq.URL.Path = "/mesh/" + canonPeerSeg + strings.TrimPrefix(trimmed, "mesh/"+rawPeerSeg)
+ if outReq.URL.RawPath != "" {
+ trimmedRaw := strings.TrimPrefix(outReq.URL.RawPath, "/")
+ if strings.HasPrefix(trimmedRaw, "mesh/"+rawPeerSeg) {
+ outReq.URL.RawPath = "/mesh/" + canonPeerSeg + strings.TrimPrefix(trimmedRaw, "mesh/"+rawPeerSeg)
+ } else {
+ outReq.URL.RawPath = ""
+ }
+ }
+ }
outReq.Header = sanitizeCallerMeshHeaders(req.Header)
hostHeader := req.Host
if hostHeader == "" {
@@ -89,14 +104,31 @@ func (rt *meshRoundTripper) RoundTrip(req *http.Request) (*http.Response, error)
pr, pw := io.Pipe()
rw := newPipeResponseWriter(req, pw)
- handler := s.node.EgressHandler()
+ handler := n.EgressHandler()
- go func() {
- defer rw.finish()
- handler.ServeHTTP(rw, outReq)
- }()
+ go serveEgressSafe(s, req.URL.Redacted(), outReq, rw, handler)
- return rw.waitResponse(pr), nil
+ return rw.waitResponse(pr)
+}
+
+func serveEgressSafe(s *Server, reqURL string, outReq *http.Request, rw *pipeResponseWriter, handler http.Handler) {
+ defer func() {
+ if p := recover(); p != nil {
+ var abortErr error
+ if p == http.ErrAbortHandler {
+ abortErr = fmt.Errorf("meshnet: upstream stream aborted: %w", io.ErrUnexpectedEOF)
+ } else {
+ abortErr = fmt.Errorf("meshnet: egress handler panic: %v", p)
+ }
+ if s != nil {
+ s.log().Warn("egress HTTP round trip aborted", "url", reqURL, "error", abortErr)
+ }
+ rw.abort(abortErr)
+ return
+ }
+ rw.finish()
+ }()
+ handler.ServeHTTP(rw, outReq)
}
func sanitizeCallerMeshHeaders(in http.Header) http.Header {
@@ -128,6 +160,7 @@ type pipeResponseWriter struct {
ready chan struct{}
statusCode int
respHdr http.Header
+ abortErr error
}
func newPipeResponseWriter(req *http.Request, pw *io.PipeWriter) *pipeResponseWriter {
@@ -166,8 +199,20 @@ func (w *pipeResponseWriter) finish() {
_ = w.pw.Close()
}
-func (w *pipeResponseWriter) waitResponse(pr *io.PipeReader) *http.Response {
+func (w *pipeResponseWriter) abort(err error) {
+ w.once.Do(func() {
+ w.abortErr = err
+ close(w.ready)
+ })
+ _ = w.pw.CloseWithError(err)
+}
+
+func (w *pipeResponseWriter) waitResponse(pr *io.PipeReader) (*http.Response, error) {
<-w.ready
+ if w.abortErr != nil {
+ _ = pr.Close()
+ return nil, w.abortErr
+ }
contentLen := int64(-1)
if cl := w.respHdr.Get("Content-Length"); cl != "" {
if n, err := strconv.ParseInt(cl, 10, 64); err == nil && n >= 0 {
@@ -184,7 +229,7 @@ func (w *pipeResponseWriter) waitResponse(pr *io.PipeReader) *http.Response {
Body: pr,
ContentLength: contentLen,
Request: w.req,
- }
+ }, nil
}
func httpErrorResponse(req *http.Request, code int, msg string) *http.Response {
diff --git a/sdk/meshnet/meshnet_test.go b/sdk/meshnet/meshnet_test.go
index 5d623bd4..e2c47d1a 100644
--- a/sdk/meshnet/meshnet_test.go
+++ b/sdk/meshnet/meshnet_test.go
@@ -480,8 +480,11 @@ func TestHTTPClientAuthorizerRunLimitReturns504And500(t *testing.T) {
if err != nil {
t.Fatalf("POST with 1ns BiscuitTimeout: %v", err)
}
- body504, _ := io.ReadAll(resp504.Body)
+ body504, readErr := io.ReadAll(resp504.Body)
_ = resp504.Body.Close()
+ if readErr != nil {
+ t.Fatalf("io.ReadAll(resp504): %v", readErr)
+ }
if resp504.StatusCode != http.StatusGatewayTimeout {
t.Fatalf("authorizer timeout status = %d (%s), want 504 Gateway Timeout", resp504.StatusCode, body504)
}
@@ -504,8 +507,11 @@ func TestHTTPClientAuthorizerRunLimitReturns504And500(t *testing.T) {
if err != nil {
t.Fatalf("POST over Datalog fact limit: %v", err)
}
- body500, _ := io.ReadAll(resp500.Body)
+ body500, readErr := io.ReadAll(resp500.Body)
_ = resp500.Body.Close()
+ if readErr != nil {
+ t.Fatalf("io.ReadAll(resp500): %v", readErr)
+ }
if resp500.StatusCode != http.StatusInternalServerError {
t.Fatalf("authorizer max-facts status = %d (%s), want 500 Internal Server Error", resp500.StatusCode, body500)
}
@@ -1208,8 +1214,11 @@ func TestExternalIngressCanonicalizesActorNodePeerID(t *testing.T) {
if err != nil {
t.Fatalf("ReadResponse: %v", err)
}
- respBody, _ := io.ReadAll(resp.Body)
+ respBody, readErr := io.ReadAll(resp.Body)
_ = resp.Body.Close()
+ if readErr != nil {
+ t.Fatalf("io.ReadAll(resp): %v", readErr)
+ }
if resp.StatusCode != http.StatusOK {
t.Fatalf("external ingress status = %d (%s), want 200", resp.StatusCode, string(respBody))
}
@@ -1221,6 +1230,344 @@ func TestExternalIngressCanonicalizesActorNodePeerID(t *testing.T) {
}
}
+func TestWhoIsUnverifiedExternalAndOIDCUserPeerID(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
+ defer cancel()
+
+ env := newStartedTestServer(t, ctx)
+ s := env.server
+
+ edgeEP, err := s.ListenA2A(ctx, A2AOptions{
+ Name: "edge-agent",
+ Public: true,
+ IngressAuth: api.IngressAuthMode_INGRESS_AUTH_MODE_EDGE,
+ NoAnnounce: true,
+ })
+ if err != nil {
+ t.Fatalf("ListenA2A(edge-agent): %v", err)
+ }
+ t.Cleanup(func() { _ = edgeEP.Close() })
+
+ passEP, err := s.ListenA2A(ctx, A2AOptions{
+ Name: "pass-agent",
+ Public: true,
+ IngressAuth: api.IngressAuthMode_INGRESS_AUTH_MODE_PASSTHROUGH,
+ NoAnnounce: true,
+ })
+ if err != nil {
+ t.Fatalf("ListenA2A(pass-agent): %v", err)
+ }
+ t.Cleanup(func() { _ = passEP.Close() })
+
+ type whoIsObs struct {
+ ok bool
+ info *CallerInfo
+ }
+ var obsMu sync.Mutex
+ var lastObs whoIsObs
+
+ recordHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ info, ok := s.WhoIs(r.RemoteAddr)
+ obsMu.Lock()
+ lastObs = whoIsObs{ok: ok, info: info}
+ obsMu.Unlock()
+ w.Header().Set("Content-Type", "application/json")
+ _, _ = w.Write([]byte(`{"name":"agent"}`))
+ })
+
+ edgePubSrv := &http.Server{Handler: recordHandler}
+ passPubSrv := &http.Server{Handler: recordHandler}
+ go func() { _ = edgePubSrv.Serve(edgeEP.PublicListener) }()
+ go func() { _ = passPubSrv.Serve(passEP.PublicListener) }()
+ t.Cleanup(func() {
+ _ = edgePubSrv.Close()
+ _ = passPubSrv.Close()
+ })
+
+ routerPeer := newAdmittedTestPeer(t, ctx, s, env.cpPriv, api.RoleRouter)
+ routerBiscuit := mintTestBiscuit(t, env.cpPriv, routerPeer.ID().String(), api.RoleRouter, time.Now().Add(time.Hour))
+
+ sendIngressReq := func(t *testing.T, method, rawURL string, hdr http.Header) {
+ t.Helper()
+ req, err := http.NewRequestWithContext(ctx, method, rawURL, strings.NewReader(`{}`))
+ if err != nil {
+ t.Fatalf("NewRequestWithContext: %v", err)
+ }
+ for k, vals := range hdr {
+ for _, v := range vals {
+ req.Header.Add(k, v)
+ }
+ }
+ req.Header.Set(api.HeaderMeshBiscuit, base64.RawURLEncoding.EncodeToString(routerBiscuit))
+
+ st, err := routerPeer.NewStream(ctx, s.node.Host.ID(), api.IngressProtocolID)
+ if err != nil {
+ t.Fatalf("NewStream: %v", err)
+ }
+ defer func() { _ = st.Close() }()
+ if err := req.Write(st); err != nil {
+ t.Fatalf("req.Write: %v", err)
+ }
+ _ = st.SetReadDeadline(time.Now().Add(5 * time.Second))
+ resp, err := http.ReadResponse(bufio.NewReader(st), req)
+ if err != nil {
+ t.Fatalf("ReadResponse: %v", err)
+ }
+ respBody, readErr := io.ReadAll(resp.Body)
+ _ = resp.Body.Close()
+ if readErr != nil {
+ t.Fatalf("io.ReadAll: %v", readErr)
+ }
+ if resp.StatusCode != http.StatusOK {
+ t.Fatalf("status = %d (%s), want 200", resp.StatusCode, string(respBody))
+ }
+ }
+
+ // 1. Anonymous GET /.well-known/agent-card.json in EDGE mode -> WhoIs returns (nil, false).
+ sendIngressReq(t, http.MethodGet, "http://mesh/a2a/edge-agent/.well-known/agent-card.json", http.Header{
+ api.HeaderMeshIngress: {api.IngressAuthModeStringEdge},
+ })
+ obsMu.Lock()
+ anonObs := lastObs
+ obsMu.Unlock()
+ if anonObs.ok || anonObs.info != nil {
+ t.Fatalf("anonymous agent-card WhoIs = (%+v, %v), want (nil, false)", anonObs.info, anonObs.ok)
+ }
+
+ // 2. PASSTHROUGH request -> WhoIs returns (nil, false), never router's PeerID.
+ sendIngressReq(t, http.MethodPost, "http://mesh/a2a/pass-agent/invoke", http.Header{
+ api.HeaderMeshIngress: {api.IngressAuthModeStringPassthrough},
+ "Authorization": {"Bearer pair-token"},
+ })
+ obsMu.Lock()
+ passObs := lastObs
+ obsMu.Unlock()
+ if passObs.ok || passObs.info != nil {
+ t.Fatalf("PASSTHROUGH WhoIs = (%+v, %v), want (nil, false)", passObs.info, passObs.ok)
+ }
+
+ // 3. Verified standalone external OIDC user Biscuit from /oauth/token (actor_node bound to
+ // control-plane Biscuit authority signing key, not a mesh node) -> WhoIs returns ok=true with PeerID == "".
+ cpAuthorityPeerID := authorityKeyPeerID(env.cpPub)
+ if cpAuthorityPeerID == "" {
+ t.Fatal("authorityKeyPeerID(env.cpPub) returned empty string")
+ }
+ userBuilder := biscuit.NewBuilder(env.cpPriv)
+ for _, f := range []biscuit.Fact{
+ {Predicate: biscuit.Predicate{Name: api.FactActorNode, IDs: []biscuit.Term{biscuit.String(cpAuthorityPeerID)}}},
+ {Predicate: biscuit.Predicate{Name: api.FactClientPeerID, IDs: []biscuit.Term{biscuit.String(cpAuthorityPeerID)}}},
+ {Predicate: biscuit.Predicate{Name: api.FactRole, IDs: []biscuit.Term{biscuit.String(api.RoleNode)}}},
+ {Predicate: biscuit.Predicate{Name: api.FactEmail, IDs: []biscuit.Term{biscuit.String("[email protected]")}}},
+ {Predicate: biscuit.Predicate{Name: api.FactExpiration, IDs: []biscuit.Term{biscuit.Date(time.Now().Add(time.Hour))}}},
+ api.MarkerFact(api.FactTargetUnrestricted),
+ } {
+ if err := userBuilder.AddAuthorityFact(f); err != nil {
+ t.Fatalf("AddAuthorityFact: %v", err)
+ }
+ }
+ userTok, err := userBuilder.Build()
+ if err != nil {
+ t.Fatalf("Build: %v", err)
+ }
+ userBytes, err := userTok.Serialize()
+ if err != nil {
+ t.Fatalf("Serialize: %v", err)
+ }
+ sendIngressReq(t, http.MethodPost, "http://mesh/a2a/edge-agent/invoke", http.Header{
+ api.HeaderMeshIngress: {api.IngressAuthModeStringEdge},
+ api.HeaderMeshIngressCallerBiscuit: {base64.RawURLEncoding.EncodeToString(userBytes)},
+ })
+ obsMu.Lock()
+ userObs := lastObs
+ obsMu.Unlock()
+ if !userObs.ok || userObs.info == nil {
+ t.Fatal("verified external OIDC user WhoIs returned (nil, false), want verified CallerInfo")
+ }
+ if userObs.info.Principal != "[email protected]" {
+ t.Errorf("user WhoIs Principal = %q, want [email protected]", userObs.info.Principal)
+ }
+ if userObs.info.PeerID != "" {
+ t.Errorf("user WhoIs PeerID = %q, want empty string (must not leak router %q or authority key %q)", userObs.info.PeerID, routerPeer.ID(), cpAuthorityPeerID)
+ }
+
+ // 4. Delegated Biscuit exchanged by an actual mesh actor node -> WhoIs preserves actor node PeerID.
+ actorHost, err := libp2p.New(libp2p.NoListenAddrs)
+ if err != nil {
+ t.Fatalf("libp2p.New(actorHost): %v", err)
+ }
+ t.Cleanup(func() { _ = actorHost.Close() })
+ actorBuilder := biscuit.NewBuilder(env.cpPriv)
+ for _, f := range []biscuit.Fact{
+ {Predicate: biscuit.Predicate{Name: api.FactActorNode, IDs: []biscuit.Term{biscuit.String(actorHost.ID().String())}}},
+ {Predicate: biscuit.Predicate{Name: api.FactClientPeerID, IDs: []biscuit.Term{biscuit.String(actorHost.ID().String())}}},
+ {Predicate: biscuit.Predicate{Name: api.FactRole, IDs: []biscuit.Term{biscuit.String(api.RoleNode)}}},
+ {Predicate: biscuit.Predicate{Name: api.FactEmail, IDs: []biscuit.Term{biscuit.String("[email protected]")}}},
+ {Predicate: biscuit.Predicate{Name: api.FactExpiration, IDs: []biscuit.Term{biscuit.Date(time.Now().Add(time.Hour))}}},
+ api.MarkerFact(api.FactTargetUnrestricted),
+ } {
+ if err := actorBuilder.AddAuthorityFact(f); err != nil {
+ t.Fatalf("AddAuthorityFact(actor): %v", err)
+ }
+ }
+ actorTok, err := actorBuilder.Build()
+ if err != nil {
+ t.Fatalf("Build(actor): %v", err)
+ }
+ actorBytes, err := actorTok.Serialize()
+ if err != nil {
+ t.Fatalf("Serialize(actor): %v", err)
+ }
+ sendIngressReq(t, http.MethodPost, "http://mesh/a2a/edge-agent/invoke", http.Header{
+ api.HeaderMeshIngress: {api.IngressAuthModeStringEdge},
+ api.HeaderMeshIngressCallerBiscuit: {base64.RawURLEncoding.EncodeToString(actorBytes)},
+ })
+ obsMu.Lock()
+ actorObs := lastObs
+ obsMu.Unlock()
+ if !actorObs.ok || actorObs.info == nil {
+ t.Fatal("actor-delegated WhoIs returned (nil, false), want verified CallerInfo")
+ }
+ if actorObs.info.PeerID != actorHost.ID().String() {
+ t.Errorf("actor-delegated WhoIs PeerID = %q, want %q", actorObs.info.PeerID, actorHost.ID().String())
+ }
+}
+
+func TestHTTPClientCanonicalizesCIDv1PeerIDAndRecoversAbort(t *testing.T) {
+ t.Run("canonicalizes_cidv1_peer_id_on_egress", func(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
+ defer cancel()
+
+ mesh := newInProcessMeshPair(t, ctx)
+ ep, err := mesh.provider.ListenA2A(ctx, A2AOptions{Name: "coder"})
+ if err != nil {
+ t.Fatalf("ListenA2A: %v", err)
+ }
+ srv := &http.Server{
+ Handler: http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ _, _ = w.Write([]byte("cidv1-ok"))
+ }),
+ }
+ go func() { _ = srv.Serve(ep.PrivateListener) }()
+ t.Cleanup(func() {
+ _ = srv.Close()
+ _ = ep.Close()
+ })
+
+ cidv1Provider := peer.ToCid(mesh.provider.peerID).String()
+ if cidv1Provider == mesh.provider.peerID.String() {
+ t.Fatalf("peer.ToCid(%s) did not produce distinct CIDv1 string", mesh.provider.peerID)
+ }
+
+ resp, err := mesh.caller.HTTPClient().Get(fmt.Sprintf("http://agentmesh.local/mesh/%s/a2a/coder/ping", cidv1Provider))
+ if err != nil {
+ t.Fatalf("GET with CIDv1 peer ID: %v", err)
+ }
+ body, readErr := io.ReadAll(resp.Body)
+ _ = resp.Body.Close()
+ if readErr != nil {
+ t.Fatalf("io.ReadAll: %v", readErr)
+ }
+ if resp.StatusCode != http.StatusOK || string(body) != "cidv1-ok" {
+ t.Fatalf("CIDv1 egress = %d %q, want 200 cidv1-ok", resp.StatusCode, string(body))
+ }
+ })
+
+ t.Run("recovers_ErrAbortHandler_before_headers", func(t *testing.T) {
+ req := httptest.NewRequest(http.MethodGet, "http://agentmesh.local/mesh/test", nil)
+ pr, pw := io.Pipe()
+ rw := newPipeResponseWriter(req, pw)
+
+ go serveEgressSafe(&Server{}, req.URL.String(), req, rw, http.HandlerFunc(func(http.ResponseWriter, *http.Request) {
+ panic(http.ErrAbortHandler)
+ }))
+
+ resp, err := rw.waitResponse(pr)
+ if !errors.Is(err, io.ErrUnexpectedEOF) {
+ t.Fatalf("waitResponse err = %v (resp=%v), want io.ErrUnexpectedEOF", err, resp)
+ }
+ })
+
+ t.Run("recovers_ErrAbortHandler_mid_stream", func(t *testing.T) {
+ req := httptest.NewRequest(http.MethodGet, "http://agentmesh.local/mesh/test", nil)
+ pr, pw := io.Pipe()
+ rw := newPipeResponseWriter(req, pw)
+
+ go serveEgressSafe(&Server{}, req.URL.String(), req, rw, http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ w.WriteHeader(http.StatusOK)
+ _, _ = w.Write([]byte("partial-chunk"))
+ panic(http.ErrAbortHandler)
+ }))
+
+ resp, err := rw.waitResponse(pr)
+ if err != nil {
+ t.Fatalf("waitResponse after headers: %v", err)
+ }
+ body, readErr := io.ReadAll(resp.Body)
+ _ = resp.Body.Close()
+ if string(body) != "partial-chunk" {
+ t.Errorf("partial body = %q, want partial-chunk", string(body))
+ }
+ if !errors.Is(readErr, io.ErrUnexpectedEOF) {
+ t.Fatalf("io.ReadAll after mid-stream abort = %v, want io.ErrUnexpectedEOF", readErr)
+ }
+ })
+
+ t.Run("recovers_unexpected_handler_panic", func(t *testing.T) {
+ req := httptest.NewRequest(http.MethodGet, "http://agentmesh.local/mesh/test", nil)
+ pr, pw := io.Pipe()
+ rw := newPipeResponseWriter(req, pw)
+
+ go serveEgressSafe(&Server{}, req.URL.String(), req, rw, http.HandlerFunc(func(http.ResponseWriter, *http.Request) {
+ panic("unexpected-boom")
+ }))
+
+ resp, err := rw.waitResponse(pr)
+ if err == nil || !strings.Contains(err.Error(), "unexpected-boom") {
+ t.Fatalf("waitResponse on unexpected panic = (%v, %v), want error containing unexpected-boom", resp, err)
+ }
+ })
+}
+
+func TestServerConcurrentStatusMetricsAndClose(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
+ defer cancel()
+
+ env := newStartedTestServer(t, ctx)
+ reg := prometheus.NewRegistry()
+ s := &Server{
+ ControlPlaneURL: env.cp.srv.URL,
+ BootstrapToken: "boot-secret",
+ ListenAddrs: []string{"/ip4/127.0.0.1/tcp/0"},
+ AllowInsecure: true,
+ Registerer: reg,
+ }
+ if _, err := s.Up(ctx); err != nil {
+ t.Fatalf("s.Up: %v", err)
+ }
+ t.Cleanup(func() { _ = s.Close() })
+
+ var wg sync.WaitGroup
+ for range 4 {
+ wg.Add(1)
+ go func() {
+ defer wg.Done()
+ for range 50 {
+ _ = s.Status()
+ _, _ = reg.Gather()
+ _, _, _ = s.AttenuateToken(&api.TaskAuthorizationRule{Name: "race-check"}, time.Minute, false)
+ s.onStreamRefused("unauthenticated", string(api.IngressProtocolID))
+ }
+ }()
+ }
+ wg.Add(1)
+ go func() {
+ defer wg.Done()
+ time.Sleep(5 * time.Millisecond)
+ _ = s.Close()
+ }()
+ wg.Wait()
+}
+
type inProcessMeshPair struct {
provider *Server
caller *Server
diff --git a/sdk/meshnet/metrics.go b/sdk/meshnet/metrics.go
index 3f8d7732..af9e62ec 100644
--- a/sdk/meshnet/metrics.go
+++ b/sdk/meshnet/metrics.go
@@ -127,14 +127,15 @@ func (c *serverStateCollector) Collect(ch chan<- prometheus.Metric) {
if s.credentialLapsed.Load() {
lapsedVal = 1
}
- if s.node != nil && s.node.Host != nil {
- peersVal = float64(len(s.node.Host.Network().Peers()))
- attachedVal = float64(len(s.node.AttachedRouterIDs()))
- if s.node.IsConnected() && lapsedVal == 0 {
+ n, store, _, _ := s.stateSnapshot()
+ if n != nil && n.Host != nil {
+ peersVal = float64(len(n.Host.Network().Peers()))
+ attachedVal = float64(len(n.AttachedRouterIDs()))
+ if n.IsConnected() && lapsedVal == 0 {
readyVal = 1
}
- if s.node.Services() != nil {
- servicesVal = float64(len(s.node.ListLocalServices(api.ServiceType_SERVICE_TYPE_UNSPECIFIED)))
+ if n.Services() != nil {
+ servicesVal = float64(len(n.ListLocalServices(api.ServiceType_SERVICE_TYPE_UNSPECIFIED)))
}
}
@@ -143,7 +144,7 @@ func (c *serverStateCollector) Collect(ch chan<- prometheus.Metric) {
ch <- prometheus.MustNewConstMetric(c.routers, prometheus.GaugeValue, attachedVal, "attached")
ch <- prometheus.MustNewConstMetric(c.services, prometheus.GaugeValue, servicesVal)
ch <- prometheus.MustNewConstMetric(c.credentialLapse, prometheus.GaugeValue, lapsedVal)
- if exp := s.credentialExpireTime(); !exp.IsZero() {
+ if exp := credentialExpireTimeFromStore(store); !exp.IsZero() {
ch <- prometheus.MustNewConstMetric(c.biscuitExpiry, prometheus.GaugeValue, float64(exp.Unix()))
}
}
@@ -156,8 +157,8 @@ func (s *Server) onStreamRefused(scope, proto string) {
}
s.refusedStreams[key]++
s.refusedMu.Unlock()
- if s.metrics != nil {
- s.metrics.streamsRefused.WithLabelValues(scope, proto).Inc()
+ if m := s.metrics.Load(); m != nil {
+ m.streamsRefused.WithLabelValues(scope, proto).Inc()
}
}
diff --git a/sdk/meshnet/server.go b/sdk/meshnet/server.go
index bb7ca14d..40998a9d 100644
--- a/sdk/meshnet/server.go
+++ b/sdk/meshnet/server.go
@@ -59,8 +59,11 @@ var ErrCredentialLapsed = errors.New("meshnet: member credential expired and ren
// the mesh in-process.
type Server struct {
// StateDir persists identity.key and credential.json (0700/0600, byte-for-byte
- // compatible with sdk/js, sdk/python, and `agentmesh-node state export|import`).
- // When empty, Server runs with an ephemeral in-memory Ed25519 identity.
+ // compatible with sdk/js, sdk/python, and `agentmesh-node state export|import`),
+ // guarded by an exclusive non-blocking directory lock (ErrStateDirLocked) so two
+ // Servers cannot clobber the same state directory concurrently.
+ // When empty, Server runs with an ephemeral in-memory Ed25519 identity that
+ // creates no files on disk.
StateDir string
// ControlPlaneURL is the Agent Mesh hub URL (e.g. "https://bananas.sam-mesh.dev").
@@ -137,13 +140,13 @@ type Server struct {
closed bool
runCancel context.CancelFunc
- privKey crypto.PrivKey
- peerID peer.ID
- cpURL string
- store *node.Store
- storeTmpDir string
- node *node.AgentMeshNode
- metrics *serverMetrics
+ privKey crypto.PrivKey
+ peerID peer.ID
+ cpURL string
+ store *node.Store
+ stateDirLock *os.File
+ node *node.AgentMeshNode
+ metrics atomic.Pointer[serverMetrics]
refusedMu sync.Mutex
refusedStreams map[string]uint64
@@ -208,10 +211,10 @@ func (s *Server) signalDone() {
})
}
-func (s *Server) handleCredentialLapse(err error) {
+func (s *Server) handleCredentialLapse(peerID peer.ID, err error) {
s.credentialLapsed.Store(true)
- if s.metrics != nil {
- s.metrics.credentialLapses.Inc()
+ if m := s.metrics.Load(); m != nil {
+ m.credentialLapses.Inc()
}
wrapped := fmt.Errorf("%w: %w", ErrCredentialLapsed, err)
s.errMu.Lock()
@@ -219,7 +222,7 @@ func (s *Server) handleCredentialLapse(err error) {
s.errMu.Unlock()
s.signalDone()
s.log().Error("member credential expired and renewal failed; retrying with exponential backoff",
- "peer_id", s.peerID.String(),
+ "peer_id", peerID.String(),
"error", err,
)
if s.OnCredentialLapse != nil {
@@ -227,9 +230,9 @@ func (s *Server) handleCredentialLapse(err error) {
}
}
-func (s *Server) handleCredentialRenewed() {
+func (s *Server) handleCredentialRenewed(peerID peer.ID) {
s.credentialLapsed.Store(false)
- s.log().Info("member credential renewed", "peer_id", s.peerID.String())
+ s.log().Info("member credential renewed", "peer_id", peerID.String())
}
// Done returns a channel that is closed when either the member's credential
@@ -249,17 +252,35 @@ func (s *Server) Err() error {
return s.lapseErr
}
-func (s *Server) isRunning() bool {
+func (s *Server) runningNode() *node.AgentMeshNode {
s.lifecycleMu.Lock()
defer s.lifecycleMu.Unlock()
- return s.started && !s.closed && s.node != nil && s.node.Host != nil
+ if !s.started || s.closed || s.node == nil || s.node.Host == nil {
+ return nil
+ }
+ return s.node
}
-func (s *Server) cleanupStoreTmpDir() {
- if s.storeTmpDir != "" {
- _ = os.RemoveAll(s.storeTmpDir)
- s.storeTmpDir = ""
+func (s *Server) stateSnapshot() (*node.AgentMeshNode, *node.Store, peer.ID, string) {
+ s.lifecycleMu.Lock()
+ defer s.lifecycleMu.Unlock()
+ if s.closed {
+ return nil, nil, "", ""
}
+ return s.node, s.store, s.peerID, s.cpURL
+}
+
+func (s *Server) closeStoreLocked() error {
+ var err error
+ if s.store != nil {
+ err = s.store.Close()
+ s.store = nil
+ }
+ if s.stateDirLock != nil {
+ unlockStateDir(s.stateDirLock)
+ s.stateDirLock = nil
+ }
+ return err
}
// Up loads or enrolls the member credential, syncs mesh trust material and
@@ -277,7 +298,7 @@ func (s *Server) Up(ctx context.Context) (*Status, error) {
return nil, errors.New("meshnet: Server has been closed")
}
if s.started {
- st := s.Status()
+ st := s.statusLocked()
return &st, nil
}
@@ -310,13 +331,11 @@ func (s *Server) Up(ctx context.Context) (*Status, error) {
if s.Registerer != nil {
m, err := newServerMetrics(s, s.Registerer)
if err != nil {
- _ = s.store.Close()
- s.cleanupStoreTmpDir()
+ _ = s.closeStoreLocked()
s.node = nil
- s.store = nil
return nil, fmt.Errorf("meshnet: register metrics: %w", err)
}
- s.metrics = m
+ s.metrics.Store(m)
}
runCtx, runCancel := context.WithCancel(context.Background())
@@ -325,28 +344,23 @@ func (s *Server) Up(ctx context.Context) (*Status, error) {
abortedByCaller := !stopAbort()
if startErr != nil {
runCancel()
- if s.metrics != nil {
- s.metrics.unregister()
- s.metrics = nil
+ if m := s.metrics.Swap(nil); m != nil {
+ m.unregister()
}
_ = s.node.Teardown()
- if s.store != nil {
- _ = s.store.Close()
- }
- s.cleanupStoreTmpDir()
+ _ = s.closeStoreLocked()
s.node = nil
- s.store = nil
if abortedByCaller && ctx.Err() != nil {
return nil, ctx.Err()
}
return nil, startErr
}
- s.startRenewalLoop(runCtx)
+ s.startRenewalLoop(runCtx, s.node, s.store, s.peerID, s.cpURL)
s.runCancel = runCancel
s.started = true
- st := s.Status()
+ st := s.statusLocked()
return &st, nil
}
@@ -356,6 +370,12 @@ func (s *Server) Up(ctx context.Context) (*Status, error) {
// router admission while additional router dials finish in the background,
// AttachedRouters may list fewer than Server.Routers immediately after Up.
func (s *Server) Status() Status {
+ s.lifecycleMu.Lock()
+ defer s.lifecycleMu.Unlock()
+ return s.statusLocked()
+}
+
+func (s *Server) statusLocked() Status {
var peerIDStr string
if s.peerID != "" {
peerIDStr = s.peerID.String()
@@ -363,7 +383,7 @@ func (s *Server) Status() Status {
var routerStrs []string
var ingressURLs []string
- if s.node != nil {
+ if s.node != nil && !s.closed {
if s.node.Host != nil {
routers := s.node.AttachedRouterIDs()
routerStrs = make([]string, 0, len(routers))
@@ -380,7 +400,7 @@ func (s *Server) Status() Status {
ControlPlaneURL: s.cpURL,
AttachedRouters: routerStrs,
IngressURLs: ingressURLs,
- ExpireTime: s.credentialExpireTime(),
+ ExpireTime: credentialExpireTimeFromStore(s.store),
CredentialLapsed: s.credentialLapsed.Load(),
}
}
@@ -398,9 +418,12 @@ func (s *Server) Close() error {
s.closed = true
s.signalDone()
- if s.metrics != nil {
- s.metrics.unregister()
- s.metrics = nil
+ if m := s.metrics.Swap(nil); m != nil {
+ m.unregister()
+ }
+
+ if s.runCancel != nil {
+ s.runCancel()
}
var errs []error
@@ -408,16 +431,11 @@ func (s *Server) Close() error {
if err := s.node.Teardown(); err != nil {
errs = append(errs, err)
}
+ s.node = nil
}
- if s.runCancel != nil {
- s.runCancel()
- }
- if s.store != nil {
- if err := s.store.Close(); err != nil {
- errs = append(errs, err)
- }
+ if err := s.closeStoreLocked(); err != nil {
+ errs = append(errs, err)
}
- s.cleanupStoreTmpDir()
return errors.Join(errs...)
}
@@ -426,7 +444,8 @@ func (s *Server) Close() error {
// type via parallel catalog fan-out). Returned DiscoveredProvider.LocalProxyUrl
// values can be passed directly to Server.HTTPClient.
func (s *Server) Discover(ctx context.Context, serviceType api.ServiceType, serviceName string) ([]*api.DiscoveredProvider, error) {
- if !s.isRunning() {
+ n := s.runningNode()
+ if n == nil {
return nil, errors.New("meshnet: Server is not running (call Up first)")
}
if serviceType == api.ServiceType_SERVICE_TYPE_UNSPECIFIED {
@@ -435,7 +454,7 @@ func (s *Server) Discover(ctx context.Context, serviceType api.ServiceType, serv
if _, err := api.ServiceTypeToString(serviceType); err != nil {
return nil, err
}
- return s.node.DiscoverRemoteServices(ctx, serviceType, serviceName)
+ return n.DiscoverRemoteServices(ctx, serviceType, serviceName)
}
// MCPServer returns an in-process *mcp.Server exposing the 6 progressive mesh
@@ -443,17 +462,19 @@ func (s *Server) Discover(ctx context.Context, serviceType api.ServiceType, serv
// discover_remote_services, find_remote_tools, describe_remote_tool,
// call_remote_tool). Returns nil if Up has not succeeded.
func (s *Server) MCPServer() *mcp.Server {
- if !s.isRunning() {
+ n := s.runningNode()
+ if n == nil {
return nil
}
- return node.NewMCPServer(s.node)
+ return node.NewMCPServer(n)
}
// MCPHandler returns a stateless StreamableHTTP handler serving Server.MCPServer
// at "/", "/mcp", and "/mcp/".
func (s *Server) MCPHandler() http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- if !s.isRunning() {
+ n := s.runningNode()
+ if n == nil {
http.Error(w, "meshnet: Server is not running", http.StatusServiceUnavailable)
return
}
@@ -464,7 +485,7 @@ func (s *Server) MCPHandler() http.Handler {
return
}
s.mcpHandlerOnce.Do(func() {
- s.mcpHTTPHandler = node.NewMCPHandler(s.node)
+ s.mcpHTTPHandler = node.NewMCPHandler(n)
})
if r.URL.Path != "/mcp" {
r2 := r.Clone(r.Context())
@@ -481,12 +502,13 @@ func (s *Server) MCPHandler() http.Handler {
// inference://* providers on the mesh.
func (s *Server) InferenceHandler() http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- if !s.isRunning() {
+ n := s.runningNode()
+ if n == nil {
http.Error(w, "meshnet: Server is not running", http.StatusServiceUnavailable)
return
}
s.infHandlerOnce.Do(func() {
- s.infHTTPHandler = s.node.InferenceHandler()
+ s.infHTTPHandler = n.InferenceHandler()
})
s.infHTTPHandler.ServeHTTP(w, r)
})
@@ -497,20 +519,21 @@ func (s *Server) InferenceHandler() http.Handler {
// a control-plane round trip, optionally sealing it so downstream holders
// cannot append further blocks.
func (s *Server) AttenuateToken(rule *api.TaskAuthorizationRule, ttl time.Duration, seal bool) (string, time.Time, error) {
- if s.node == nil {
+ n, store, _, _ := s.stateSnapshot()
+ if n == nil {
return "", time.Time{}, errors.New("meshnet: Server is not running (call Up first)")
}
if rule == nil {
return "", time.Time{}, errors.New("TaskAuthorizationRule is required")
}
- raw := s.node.GetIdentity()
+ raw := n.GetIdentity()
if len(raw) == 0 {
return "", time.Time{}, errors.New("no member credential loaded")
}
cloned := proto.Clone(rule).(*api.TaskAuthorizationRule)
now := time.Now()
- exp := s.credentialExpireTime()
+ exp := credentialExpireTimeFromStore(store)
if exp.IsZero() {
exp = now.Add(api.BiscuitTokenTTL)
}
diff --git a/sdk/meshnet/statedir.go b/sdk/meshnet/statedir.go
index 48446485..fe4e25e4 100644
--- a/sdk/meshnet/statedir.go
+++ b/sdk/meshnet/statedir.go
@@ -50,6 +50,10 @@ const minResumeLifetime = 5 * time.Minute
// and no join token or TokenSource was supplied to re-enroll.
var ErrCredentialRetired = errors.New("meshnet: saved credential signing key retired; provide a join token to re-enroll")
+// ErrStateDirLocked reports that another running Server already holds an
+// exclusive lock on StateDir.
+var ErrStateDirLocked = errors.New("meshnet: state directory is already locked by another Server")
+
// WriteStateDir writes a member's libp2p private key and MemberCredential to
// dir with owner-only permissions (0700 directory, 0600 files).
func WriteStateDir(dir string, privKey []byte, c *api.MemberCredential) error {
@@ -212,73 +216,73 @@ func (s *Server) newAgentMeshNode(store *node.Store, privKey crypto.PrivKey) (*n
return n, nil
}
-func (s *Server) openStore() (*node.Store, error) {
- tmpDir, err := os.MkdirTemp("", "meshnet-store-*")
- if err != nil {
- return nil, fmt.Errorf("create store workspace: %w", err)
+func (s *Server) openStore() (*node.Store, *os.File, error) {
+ store := node.NewMemoryStore()
+
+ dir := strings.TrimSpace(s.StateDir)
+ if dir == "" {
+ return store, nil, nil
}
- store, err := node.NewStore(tmpDir)
+
+ if err := os.MkdirAll(dir, 0o700); err != nil {
+ _ = store.Close()
+ return nil, nil, fmt.Errorf("create state directory %s: %w", dir, err)
+ }
+ lockFile, err := lockStateDir(dir)
if err != nil {
- _ = os.RemoveAll(tmpDir)
- return nil, err
+ _ = store.Close()
+ return nil, nil, err
}
- cleanupOnErr := func(e error) (*node.Store, error) {
+ cleanupOnErr := func(e error) (*node.Store, *os.File, error) {
_ = store.Close()
- _ = os.RemoveAll(tmpDir)
- return nil, e
+ unlockStateDir(lockFile)
+ return nil, nil, e
}
- dir := strings.TrimSpace(s.StateDir)
- if dir != "" {
- if err := os.MkdirAll(dir, 0o700); err != nil {
- return cleanupOnErr(fmt.Errorf("create state directory %s: %w", dir, err))
- }
- keyPath := filepath.Join(dir, StateDirIdentityFile)
- if rawKey, rErr := os.ReadFile(keyPath); rErr == nil {
- if _, uErr := crypto.UnmarshalPrivateKey(rawKey); uErr != nil {
- return cleanupOnErr(fmt.Errorf("%s: not a libp2p private key: %w", keyPath, uErr))
- }
- if sErr := store.SaveKey(rawKey); sErr != nil {
- return cleanupOnErr(sErr)
- }
- } else if !errors.Is(rErr, os.ErrNotExist) {
- return cleanupOnErr(rErr)
+ keyPath := filepath.Join(dir, StateDirIdentityFile)
+ if rawKey, rErr := os.ReadFile(keyPath); rErr == nil {
+ if _, uErr := crypto.UnmarshalPrivateKey(rawKey); uErr != nil {
+ return cleanupOnErr(fmt.Errorf("%s: not a libp2p private key: %w", keyPath, uErr))
}
-
- credPath := filepath.Join(dir, StateDirCredentialFile)
- if rawCred, rErr := os.ReadFile(credPath); rErr == nil {
- cred := &api.MemberCredential{}
- if uErr := protojson.Unmarshal(rawCred, cred); uErr != nil {
- return cleanupOnErr(fmt.Errorf("%s: %w", credPath, uErr))
- }
- if sErr := store.SetCredential(cred); sErr != nil {
- return cleanupOnErr(sErr)
- }
- } else if !errors.Is(rErr, os.ErrNotExist) {
- return cleanupOnErr(rErr)
+ if sErr := store.SaveKey(rawKey); sErr != nil {
+ return cleanupOnErr(sErr)
}
+ } else if !errors.Is(rErr, os.ErrNotExist) {
+ return cleanupOnErr(rErr)
+ }
- var persistMu sync.Mutex
- store.SetOnChange(func() error {
- persistMu.Lock()
- defer persistMu.Unlock()
- kb, lErr := store.LoadKey()
- if lErr != nil {
- return lErr
- }
- if len(kb) == 0 {
- return nil
- }
- c, cErr := store.Credential()
- if cErr != nil {
- return cErr
- }
- return WriteStateDir(dir, kb, c)
- })
+ credPath := filepath.Join(dir, StateDirCredentialFile)
+ if rawCred, rErr := os.ReadFile(credPath); rErr == nil {
+ cred := &api.MemberCredential{}
+ if uErr := protojson.Unmarshal(rawCred, cred); uErr != nil {
+ return cleanupOnErr(fmt.Errorf("%s: %w", credPath, uErr))
+ }
+ if sErr := store.SetCredential(cred); sErr != nil {
+ return cleanupOnErr(sErr)
+ }
+ } else if !errors.Is(rErr, os.ErrNotExist) {
+ return cleanupOnErr(rErr)
}
- s.storeTmpDir = tmpDir
- return store, nil
+ var persistMu sync.Mutex
+ store.SetOnChange(func() error {
+ persistMu.Lock()
+ defer persistMu.Unlock()
+ kb, lErr := store.LoadKey()
+ if lErr != nil {
+ return lErr
+ }
+ if len(kb) == 0 {
+ return nil
+ }
+ c, cErr := store.Credential()
+ if cErr != nil {
+ return cErr
+ }
+ return WriteStateDir(dir, kb, c)
+ })
+
+ return store, lockFile, nil
}
func (s *Server) verifyOwnBiscuit(raw []byte) bool {
@@ -323,15 +327,15 @@ func (s *Server) identityPredatesRotation() bool {
}
func (s *Server) ensureCredential(ctx context.Context) (retErr error) {
- store, err := s.openStore()
+ store, lockFile, err := s.openStore()
if err != nil {
return err
}
+ s.store = store
+ s.stateDirLock = lockFile
defer func() {
if retErr != nil {
- _ = store.Close()
- s.cleanupStoreTmpDir()
- s.store = nil
+ _ = s.closeStoreLocked()
s.node = nil
}
}()
@@ -346,7 +350,6 @@ func (s *Server) ensureCredential(ctx context.Context) (retErr error) {
}
s.privKey = privKey
s.peerID = peerID
- s.store = store
cred, err := store.Credential()
if err != nil {
@@ -420,7 +423,7 @@ func (s *Server) ensureCredential(ctx context.Context) (retErr error) {
if cred != nil && len(cred.GetBiscuit()) > 0 {
keysErr := n.SyncTrustedKeys(ctx, s.cpURL)
roleOK := s.verifyOwnBiscuit(cred.GetBiscuit())
- remaining := time.Until(s.credentialExpireTime())
+ remaining := time.Until(credentialExpireTimeFromStore(store))
if keysErr == nil && roleOK && remaining > minResumeLifetime && !s.identityPredatesRotation() {
s.log().Info("resumed mesh credential from state directory",
@@ -507,7 +510,25 @@ func isRetiredVerificationError(err error) bool {
return err != nil && strings.Contains(err.Error(), "/keys response rejected")
}
-func (s *Server) startRenewalLoop(ctx context.Context) {
+func nextRenewalDelay(now, expTime time.Time, failureBackoff, initBackoff time.Duration) time.Duration {
+ if failureBackoff > 0 {
+ return failureBackoff
+ }
+ if expTime.IsZero() {
+ return node.DefaultRenewalFallback
+ }
+ duration := expTime.Sub(now)
+ switch {
+ case duration > node.RenewalThreshold:
+ return duration - node.RenewalBuffer
+ case duration > 0:
+ return max(duration/2, initBackoff)
+ default:
+ return min(time.Second, initBackoff)
+ }
+}
+
+func (s *Server) startRenewalLoop(ctx context.Context, n *node.AgentMeshNode, store *node.Store, peerID peer.ID, cpURL string) {
initBackoff := s.renewalInitialBackoff
if initBackoff <= 0 {
initBackoff = 2 * time.Second
@@ -519,28 +540,7 @@ func (s *Server) startRenewalLoop(ctx context.Context) {
go func() {
var failureBackoff time.Duration
for {
- renewAfter := node.DefaultRenewalFallback
- if expTime := s.credentialExpireTime(); !expTime.IsZero() {
- duration := time.Until(expTime)
- switch {
- case duration > node.RenewalThreshold:
- renewAfter = duration - node.RenewalBuffer
- if failureBackoff > renewAfter {
- renewAfter = min(failureBackoff, duration/2)
- }
- case duration > 0:
- renewAfter = max(duration/2, initBackoff)
- if failureBackoff > renewAfter {
- renewAfter = min(failureBackoff, max(duration/2, initBackoff))
- }
- case failureBackoff > 0:
- renewAfter = failureBackoff
- default:
- renewAfter = min(time.Second, initBackoff)
- }
- } else if failureBackoff > 0 {
- renewAfter = failureBackoff
- }
+ renewAfter := nextRenewalDelay(time.Now(), credentialExpireTimeFromStore(store), failureBackoff, initBackoff)
timer := time.NewTimer(renewAfter)
select {
@@ -548,17 +548,17 @@ func (s *Server) startRenewalLoop(ctx context.Context) {
timer.Stop()
return
case <-timer.C:
- refreshErr := s.node.RefreshEnrollment(ctx)
+ refreshErr := n.RefreshEnrollment(ctx)
if refreshErr == nil {
failureBackoff = 0
- s.handleCredentialRenewed()
+ s.handleCredentialRenewed(peerID)
continue
}
- renewErr := s.fallbackReEnroll(ctx)
+ renewErr := s.fallbackReEnroll(ctx, n, store, cpURL)
if renewErr == nil {
failureBackoff = 0
- s.handleCredentialRenewed()
+ s.handleCredentialRenewed(peerID)
continue
}
@@ -568,35 +568,35 @@ func (s *Server) startRenewalLoop(ctx context.Context) {
failureBackoff = min(failureBackoff*2, maxBackoff)
}
s.log().Warn("credential renewal failed; scheduled backoff retry",
- "peer_id", s.peerID.String(),
+ "peer_id", peerID.String(),
"next_backoff", failureBackoff,
"refresh_error", refreshErr,
"reenroll_error", renewErr,
)
- if exp := s.credentialExpireTime(); !exp.IsZero() && !time.Now().Before(exp) {
- s.handleCredentialLapse(errors.Join(refreshErr, renewErr))
+ if exp := credentialExpireTimeFromStore(store); !exp.IsZero() && !time.Now().Before(exp) {
+ s.handleCredentialLapse(peerID, errors.Join(refreshErr, renewErr))
}
}
}
}()
}
-func (s *Server) fallbackReEnroll(ctx context.Context) error {
+func (s *Server) fallbackReEnroll(ctx context.Context, n *node.AgentMeshNode, store *node.Store, cpURL string) error {
var errs []error
if ts := s.tokenSource(); ts != nil {
newJWT, fetchErr := ts.FetchToken(ctx)
if fetchErr != nil {
errs = append(errs, fetchErr)
- } else if enrollErr := s.node.Enroll(ctx, s.cpURL, newJWT); enrollErr == nil {
+ } else if enrollErr := n.Enroll(ctx, cpURL, newJWT); enrollErr == nil {
return nil
} else {
errs = append(errs, enrollErr)
}
}
- if s.store != nil {
- if rt, _ := s.store.LoadRefreshToken(); rt != "" {
- if oidcErr := s.node.ReEnrollWithRefreshToken(ctx); oidcErr == nil {
+ if store != nil {
+ if rt, _ := store.LoadRefreshToken(); rt != "" {
+ if oidcErr := n.ReEnrollWithRefreshToken(ctx); oidcErr == nil {
return nil
} else {
errs = append(errs, oidcErr)
@@ -615,7 +615,7 @@ func (s *Server) fallbackReEnroll(ctx context.Context) error {
bootToken = strings.TrimSpace(s.BootstrapToken)
}
if bootToken != "" {
- if _, bErr := s.node.EnrollBootstrapCredential(ctx, s.cpURL, bootToken, true); bErr == nil {
+ if _, bErr := n.EnrollBootstrapCredential(ctx, cpURL, bootToken, true); bErr == nil {
return nil
} else {
errs = append(errs, bErr)
@@ -632,21 +632,22 @@ func (s *Server) fallbackReEnroll(ctx context.Context) error {
// configured), verifies the returned credential, persists it to StateDir, and
// re-authenticates attached router sessions.
func (s *Server) Refresh(ctx context.Context) error {
- if s.node == nil {
+ n, _, _, _ := s.stateSnapshot()
+ if n == nil {
return errors.New("meshnet: Server is not running (call Up first)")
}
- err := s.node.RefreshEnrollment(ctx)
+ err := n.RefreshEnrollment(ctx)
if err == nil {
s.credentialLapsed.Store(false)
}
return err
}
-func (s *Server) credentialExpireTime() time.Time {
- if s.store == nil {
+func credentialExpireTimeFromStore(store *node.Store) time.Time {
+ if store == nil {
return time.Time{}
}
- expUnix, err := s.store.LoadIdentityExpiration()
+ expUnix, err := store.LoadIdentityExpiration()
if err != nil || expUnix <= 0 {
return time.Time{}
}
diff --git a/sdk/meshnet/statedir_lock_unix.go b/sdk/meshnet/statedir_lock_unix.go
new file mode 100644
index 00000000..b582b10e
--- /dev/null
+++ b/sdk/meshnet/statedir_lock_unix.go
@@ -0,0 +1,47 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build unix
+
+package meshnet
+
+import (
+ "errors"
+ "fmt"
+ "os"
+ "syscall"
+)
+
+func lockStateDir(dir string) (*os.File, error) {
+ f, err := os.Open(dir)
+ if err != nil {
+ return nil, err
+ }
+ if err := syscall.Flock(int(f.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil {
+ _ = f.Close()
+ if errors.Is(err, syscall.EWOULDBLOCK) || errors.Is(err, syscall.EAGAIN) {
+ return nil, fmt.Errorf("%w: %s", ErrStateDirLocked, dir)
+ }
+ return nil, fmt.Errorf("lock state directory %s: %w", dir, err)
+ }
+ return f, nil
+}
+
+func unlockStateDir(f *os.File) {
+ if f == nil {
+ return
+ }
+ _ = syscall.Flock(int(f.Fd()), syscall.LOCK_UN)
+ _ = f.Close()
+}
diff --git a/sdk/meshnet/statedir_lock_windows.go b/sdk/meshnet/statedir_lock_windows.go
new file mode 100644
index 00000000..c37b960d
--- /dev/null
+++ b/sdk/meshnet/statedir_lock_windows.go
@@ -0,0 +1,53 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build windows
+
+package meshnet
+
+import (
+ "errors"
+ "fmt"
+ "os"
+ "path/filepath"
+
+ "golang.org/x/sys/windows"
+)
+
+func lockStateDir(dir string) (*os.File, error) {
+ lockPath := filepath.Join(dir, ".lock")
+ f, err := os.OpenFile(lockPath, os.O_CREATE|os.O_RDWR, 0o600)
+ if err != nil {
+ return nil, err
+ }
+ var ol windows.Overlapped
+ flags := uint32(windows.LOCKFILE_EXCLUSIVE_LOCK | windows.LOCKFILE_FAIL_IMMEDIATELY)
+ if err := windows.LockFileEx(windows.Handle(f.Fd()), flags, 0, 1, 0, &ol); err != nil {
+ _ = f.Close()
+ if errors.Is(err, windows.ERROR_LOCK_VIOLATION) {
+ return nil, fmt.Errorf("%w: %s", ErrStateDirLocked, dir)
+ }
+ return nil, fmt.Errorf("lock state directory %s: %w", dir, err)
+ }
+ return f, nil
+}
+
+func unlockStateDir(f *os.File) {
+ if f == nil {
+ return
+ }
+ var ol windows.Overlapped
+ _ = windows.UnlockFileEx(windows.Handle(f.Fd()), 0, 1, 0, &ol)
+ _ = f.Close()
+}
diff --git a/sdk/meshnet/statedir_test.go b/sdk/meshnet/statedir_test.go
index 00a2acfe..fb351d90 100644
--- a/sdk/meshnet/statedir_test.go
+++ b/sdk/meshnet/statedir_test.go
@@ -35,6 +35,7 @@ import (
"github.com/eclipse-biscuit/biscuit-go/v2"
"github.com/google/agentmesh/api"
+ "github.com/google/agentmesh/internal/node"
"github.com/libp2p/go-libp2p/core/crypto"
"github.com/libp2p/go-libp2p/core/peer"
"google.golang.org/protobuf/proto"
@@ -184,10 +185,12 @@ func TestEnrollmentAndRenewal(t *testing.T) {
BootstrapToken: "boot-secret",
AllowInsecure: true,
}
- t.Cleanup(func() { _ = seed.Close() })
if err := seed.ensureCredential(ctx); err != nil {
t.Fatalf("seed ensureCredential: %v", err)
}
+ if err := seed.Close(); err != nil {
+ t.Fatalf("seed.Close: %v", err)
+ }
enrollsBefore := cp.bootstrapEnrolls.Load()
refreshesBefore := cp.refreshes.Load()
@@ -196,10 +199,12 @@ func TestEnrollmentAndRenewal(t *testing.T) {
StateDir: stateDir,
AllowInsecure: true,
}
- t.Cleanup(func() { _ = online.Close() })
if err := online.ensureCredential(ctx); err != nil {
t.Fatalf("online resume ensureCredential: %v", err)
}
+ if err := online.Close(); err != nil {
+ t.Fatalf("online.Close: %v", err)
+ }
if cp.bootstrapEnrolls.Load() != enrollsBefore || cp.refreshes.Load() != refreshesBefore {
t.Errorf("online resume called enroll/refresh (%d/%d), want (%d/%d)",
cp.bootstrapEnrolls.Load(), cp.refreshes.Load(), enrollsBefore, refreshesBefore)
@@ -348,10 +353,12 @@ func TestEnrollmentAndRenewal(t *testing.T) {
BootstrapToken: "boot-secret",
AllowInsecure: true,
}
- t.Cleanup(func() { _ = seed.Close() })
if err := seed.ensureCredential(ctx); err != nil {
t.Fatalf("seed ensureCredential: %v", err)
}
+ if err := seed.Close(); err != nil {
+ t.Fatalf("seed.Close: %v", err)
+ }
// Rotate control-plane key completely (dropping old key from KeysResponse).
cp.rotateKeyHard(t)
@@ -380,6 +387,148 @@ func TestEnrollmentAndRenewal(t *testing.T) {
})
}
+func TestStateDirLockAndEphemeralMemoryIdentity(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
+ defer cancel()
+
+ env := newStartedTestServer(t, ctx)
+
+ t.Run("ephemeral_empty_StateDir_writes_zero_files_to_TMPDIR", func(t *testing.T) {
+ emptyTmp := t.TempDir()
+ t.Setenv("TMPDIR", emptyTmp)
+
+ ephemeral := &Server{
+ ControlPlaneURL: env.cp.srv.URL,
+ BootstrapToken: "boot-secret",
+ ListenAddrs: []string{"/ip4/127.0.0.1/tcp/0"},
+ AllowInsecure: true,
+ }
+ st, err := ephemeral.Up(ctx)
+ if err != nil {
+ t.Fatalf("ephemeral.Up: %v", err)
+ }
+ if st.PeerID == "" {
+ t.Fatal("ephemeral PeerID is empty")
+ }
+
+ duringEntries, err := os.ReadDir(emptyTmp)
+ if err != nil {
+ t.Fatalf("ReadDir(TMPDIR) while running: %v", err)
+ }
+ if len(duringEntries) != 0 {
+ t.Fatalf("ephemeral Server wrote %d entries to TMPDIR while running (%v), want 0", len(duringEntries), duringEntries)
+ }
+
+ if err := ephemeral.Close(); err != nil {
+ t.Fatalf("ephemeral.Close: %v", err)
+ }
+ afterEntries, err := os.ReadDir(emptyTmp)
+ if err != nil {
+ t.Fatalf("ReadDir(TMPDIR) after Close: %v", err)
+ }
+ if len(afterEntries) != 0 {
+ t.Fatalf("ephemeral Server left %d entries in TMPDIR (%v), want 0", len(afterEntries), afterEntries)
+ }
+ })
+
+ t.Run("concurrent_Servers_on_same_StateDir_return_ErrStateDirLocked", func(t *testing.T) {
+ sharedDir := t.TempDir()
+ s1 := &Server{
+ StateDir: sharedDir,
+ ControlPlaneURL: env.cp.srv.URL,
+ BootstrapToken: "boot-secret",
+ ListenAddrs: []string{"/ip4/127.0.0.1/tcp/0"},
+ AllowInsecure: true,
+ }
+ st1, err := s1.Up(ctx)
+ if err != nil {
+ t.Fatalf("s1.Up: %v", err)
+ }
+ t.Cleanup(func() { _ = s1.Close() })
+
+ s2 := &Server{
+ StateDir: sharedDir,
+ ControlPlaneURL: env.cp.srv.URL,
+ BootstrapToken: "boot-secret",
+ ListenAddrs: []string{"/ip4/127.0.0.1/tcp/0"},
+ AllowInsecure: true,
+ }
+ if _, err := s2.Up(ctx); !errors.Is(err, ErrStateDirLocked) {
+ _ = s2.Close()
+ t.Fatalf("s2.Up on locked StateDir = %v, want ErrStateDirLocked", err)
+ }
+
+ if err := s1.Close(); err != nil {
+ t.Fatalf("s1.Close: %v", err)
+ }
+
+ st2, err := s2.Up(ctx)
+ if err != nil {
+ t.Fatalf("s2.Up after s1.Close: %v", err)
+ }
+ t.Cleanup(func() { _ = s2.Close() })
+ if st2.PeerID != st1.PeerID {
+ t.Fatalf("s2 PeerID = %q, want %q from shared StateDir", st2.PeerID, st1.PeerID)
+ }
+ })
+}
+
+func TestNextRenewalDelay(t *testing.T) {
+ now := time.Date(2026, 11, 1, 12, 0, 0, 0, time.UTC)
+ initBackoff := 2 * time.Second
+
+ for _, tc := range []struct {
+ name string
+ expTime time.Time
+ failureBackoff time.Duration
+ want time.Duration
+ }{
+ {
+ name: "unknown_expiry_uses_fallback",
+ expTime: time.Time{},
+ want: node.DefaultRenewalFallback,
+ },
+ {
+ name: "healthy_long_lived_renews_before_expiry_buffer",
+ expTime: now.Add(24 * time.Hour),
+ want: 24*time.Hour - node.RenewalBuffer,
+ },
+ {
+ name: "short_lived_renews_at_half_lifetime",
+ expTime: now.Add(4 * time.Minute),
+ want: 2 * time.Minute,
+ },
+ {
+ name: "very_short_lived_floored_at_initBackoff",
+ expTime: now.Add(500 * time.Millisecond),
+ want: initBackoff,
+ },
+ {
+ name: "already_expired_without_prior_failure_retries_within_1s",
+ expTime: now.Add(-time.Minute),
+ want: time.Second,
+ },
+ {
+ name: "failure_backoff_honored_before_expiry",
+ expTime: now.Add(2 * time.Minute),
+ failureBackoff: 8 * time.Second,
+ want: 8 * time.Second,
+ },
+ {
+ name: "failure_backoff_honored_after_expiry",
+ expTime: now.Add(-time.Minute),
+ failureBackoff: 16 * time.Second,
+ want: 16 * time.Second,
+ },
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ if got := nextRenewalDelay(now, tc.expTime, tc.failureBackoff, initBackoff); got != tc.want {
+ t.Fatalf("nextRenewalDelay() = %v, want %v", got, tc.want)
+ }
+ })
+ }
+}
+
type fakeControlPlaneHTTP struct {
mu sync.Mutex
cpPub ed25519.PublicKey
diff --git a/site/content/docs/guides/go-sdk.md b/site/content/docs/guides/go-sdk.md
index bce41865..96651ec5 100644
--- a/site/content/docs/guides/go-sdk.md
+++ b/site/content/docs/guides/go-sdk.md
@@ -243,7 +243,11 @@ Public A2A: http://127.0.0.1:33776/ingress/12D3KooW.../a2a/coder
- **Unspoofable caller identity (`Server.WhoIs`):** Inside your HTTP handler,
`srv.WhoIs(r.RemoteAddr)` returns the cryptographically verified `*CallerInfo`
(`PeerID`, `Principal`, `Roles`, `Labels`, `Task`) bound to that in-memory
- connection—never trusting spoofable HTTP headers.
+ connection—never trusting spoofable HTTP headers. For unauthenticated external
+ requests (`INGRESS_AUTH_MODE_PASSTHROUGH` and anonymous
+ `GET /.well-known/agent-card.json` discovery), `WhoIs` returns `(nil, false)`.
+ For verified external `EDGE` callers without a mesh node identity (such as OIDC
+ users), `CallerInfo.PeerID` is empty (`""`).
- **`EDGE` vs. `PASSTHROUGH` ingress:** `INGRESS_AUTH_MODE_EDGE` (the default)
verifies Delegated Session Biscuits at both the router and the `meshnet`
server (`identity.AuthorizeMeshRequest` + `IngressAllow`), while still serving
@@ -447,7 +451,10 @@ Code, Gemini CLI, or a sandboxed coding tool), mount `srv.MCPHandler()` and
`Server.StateDir` persists `identity.key` and `credential.json` with owner-only
permissions (`0700` directory, `0600` files) using the exact protobuf/protojson
-layout shared by `agentmesh-node`, `sdk/js`, and `sdk/python`.
+layout shared by `agentmesh-node`, `sdk/js`, and `sdk/python`, guarded by an
+exclusive non-blocking directory lock (`meshnet.ErrStateDirLocked`) so two
+`Server` instances cannot clobber the same directory concurrently. When
+`StateDir` is empty, `Server` runs purely in memory with zero files on disk.
An operator can enroll interactively once with `agentmesh-node` and pass the
exported state directory directly to a Go binary built on `sdk/meshnet` (with no
diff --git a/tests/integration/meshnet_test.go b/tests/integration/meshnet_test.go
index 0a768002..765c5262 100644
--- a/tests/integration/meshnet_test.go
+++ b/tests/integration/meshnet_test.go
@@ -430,7 +430,8 @@ func TestMeshnetEndToEndIntegration(t *testing.T) {
pairedMux := http.NewServeMux()
pairedMux.Handle(a2asrv.WellKnownAgentCardPath, a2asrv.NewStaticAgentCardHandler(pairedCard))
pairedMux.Handle("/invoke", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- if r.Header.Get(api.HeaderMeshPrincipal) == "" && r.Header.Get(api.HeaderMeshIngress) == api.IngressAuthModeStringPassthrough {
+ _, hasWhoIs := srvA.WhoIs(r.RemoteAddr)
+ if !hasWhoIs && r.Header.Get(api.HeaderMeshPrincipal) == "" && r.Header.Get(api.HeaderMeshIngress) == api.IngressAuthModeStringPassthrough {
passthroughClean.Store(true)
}
if r.Header.Get("Authorization") != "Bearer pair-secret-123" {
@@ -567,6 +568,9 @@ func TestMeshnetEndToEndIntegration(t *testing.T) {
if gotPubWhoIs.Principal != "[email protected]" {
t.Fatalf("public WhoIs Principal = %q, want [email protected]", gotPubWhoIs.Principal)
}
+ if gotPubWhoIs.PeerID != "" {
+ t.Fatalf("public OIDC user WhoIs PeerID = %q, want empty string", gotPubWhoIs.PeerID)
+ }
// 5b. External OIDC user excluded by IngressAllow gets denied.
bobJWT := mintJWT(map[string]any{"sub": "bob-sub", "email": "[email protected]"})
@@ -609,6 +613,9 @@ func TestMeshnetEndToEndIntegration(t *testing.T) {
if gotTARWhoIs.Task != "audit-pr-99" {
t.Fatalf("TAR WhoIs Task = %q, want audit-pr-99", gotTARWhoIs.Task)
}
+ if gotTARWhoIs.PeerID != stB.PeerID {
+ t.Fatalf("TAR WhoIs PeerID = %q, want %q", gotTARWhoIs.PeerID, stB.PeerID)
+ }
// 5d. PASSTHROUGH service preserves upstream Bearer pair token and strips spoofed X-Mesh-*.
pubPairedCard, err := agentcard.NewResolver(extHTTP).Resolve(ctx, pairedEP.PublicURL.String())