Agent Mesh
Go a2aa2a-protocolagent-meshagentic-aiinferencemcpmeshmesh-networknetwork-securitynetworking-aip2p-network
sdk/meshnet: harden WhoIs, in-memory store, StateDir lock, and lifecycle safety
Antonio Ojea9d75f7bd9681
diff --git a/internal/node/store.go b/internal/node/store.goindex c2307cf9..84c7b981 100644--- a/internal/node/store.go+++ b/internal/node/store.go@@ -23,6 +23,7 @@ import ("path/filepath""strconv""strings"+ "sync""time""github.com/google/agentmesh/api"@@ -64,21 +65,36 @@ var legacyKeys = []string{// Store is the node's persistent state: its private key and its// api.MemberCredential, in a bbolt database whose file lock also tells a-// second agentmesh-node that this data directory is in use.+// second agentmesh-node that this data directory is in use (or purely in memory+// when constructed with NewMemoryStore).type Store struct {db *bbolt.DB+ memMu sync.RWMutex+ memKey []byte+ memCred *api.MemberCredentialonChange func() error}+// NewMemoryStore returns an ephemeral in-memory Store that creates no files on+// disk.+func NewMemoryStore() *Store {+ return &Store{memCred: &api.MemberCredential{}}+}+// SetOnChange registers an optional post-commit callback invoked after key or// credential mutations complete.func (s *Store) SetOnChange(fn func() error) {+ s.memMu.Lock()s.onChange = fn+ s.memMu.Unlock()}func (s *Store) notifyChange() error {- if s.onChange != nil {- return s.onChange()+ s.memMu.RLock()+ fn := s.onChange+ s.memMu.RUnlock()+ if fn != nil {+ return fn()}return nil}@@ -223,6 +239,12 @@ func getCredential(b *bbolt.Bucket) (*api.MemberCredential, error) {// view reads the credential; an empty message stands for a node that has not enrolled.func (s *Store) view(fn func(c *api.MemberCredential) error) error {+ if s.db == nil {+ s.memMu.RLock()+ cloned := proto.Clone(s.memCred).(*api.MemberCredential)+ s.memMu.RUnlock()+ return fn(cloned)+ }return s.db.View(func(tx *bbolt.Tx) error {c, err := getCredential(tx.Bucket([]byte(bucketIdentity)))if err != nil {@@ -234,6 +256,19 @@ func (s *Store) view(fn func(c *api.MemberCredential) error) error {// update applies fn to the credential and writes it back in one transaction.func (s *Store) update(fn func(c *api.MemberCredential) error) error {+ if s.db == nil {+ s.memMu.Lock()+ next := proto.Clone(s.memCred).(*api.MemberCredential)+ err := fn(next)+ if err == nil {+ s.memCred = next+ }+ s.memMu.Unlock()+ if err != nil {+ return err+ }+ return s.notifyChange()+ }if err := s.db.Update(func(tx *bbolt.Tx) error {b := tx.Bucket([]byte(bucketIdentity))c, err := getCredential(b)@@ -267,6 +302,12 @@ func (s *Store) SetCredential(c *api.MemberCredential) error {if c == nil {c = &api.MemberCredential{}}+ if s.db == nil {+ s.memMu.Lock()+ s.memCred = proto.Clone(c).(*api.MemberCredential)+ s.memMu.Unlock()+ return s.notifyChange()+ }if err := s.db.Update(func(tx *bbolt.Tx) error {return putCredential(tx.Bucket([]byte(bucketIdentity)), c)}); err != nil {@@ -361,6 +402,12 @@ func (s *Store) LoadOIDCConfig() (string, string, string, error) {}func (s *Store) SaveKey(key []byte) error {+ if s.db == nil {+ s.memMu.Lock()+ s.memKey = append([]byte(nil), key...)+ s.memMu.Unlock()+ return s.notifyChange()+ }if err := s.db.Update(func(tx *bbolt.Tx) error {return tx.Bucket([]byte(bucketIdentity)).Put([]byte(keyPrivKey), key)}); err != nil {@@ -370,6 +417,12 @@ func (s *Store) SaveKey(key []byte) error {}func (s *Store) LoadKey() ([]byte, error) {+ if s.db == nil {+ s.memMu.RLock()+ val := append([]byte(nil), s.memKey...)+ s.memMu.RUnlock()+ return val, nil+ }var val []byte_ = s.db.View(func(tx *bbolt.Tx) error {val = append([]byte(nil), tx.Bucket([]byte(bucketIdentity)).Get([]byte(keyPrivKey))...)@@ -481,6 +534,12 @@ func (s *Store) LoadControlPlaneURL() (string, error) {// a different mesh. It keeps the long-lived libp2p key (node_private_key), so// the node's PeerID survives the switch.func (s *Store) ResetMeshIdentity() error {+ if s.db == nil {+ s.memMu.Lock()+ s.memCred = &api.MemberCredential{}+ s.memMu.Unlock()+ return s.notifyChange()+ }if err := s.db.Update(func(tx *bbolt.Tx) error {return tx.Bucket([]byte(bucketIdentity)).Delete([]byte(keyCredential))}); err != nil {@@ -537,6 +596,9 @@ func containsKeyBytes(keys [][]byte, want []byte) bool {}func (s *Store) Close() error {+ if s.db == nil {+ return nil+ }return s.db.Close()}diff --git a/internal/node/store_test.go b/internal/node/store_test.goindex bd4627e2..bbce4585 100644--- a/internal/node/store_test.go+++ b/internal/node/store_test.go@@ -617,3 +617,79 @@ func TestStore_SetOnChange(t *testing.T) {t.Fatalf("SaveRefreshToken with failing onChange = %v, want %v", err, hookErr)}}++func TestStore_NewMemoryStore(t *testing.T) {+ emptyTmp := t.TempDir()+ t.Setenv("TMPDIR", emptyTmp)++ store := NewMemoryStore()+ defer func() {+ if err := store.Close(); err != nil {+ t.Errorf("Close: %v", err)+ }+ }()++ var mirroredKey []byte+ var mirroredCred *api.MemberCredential+ store.SetOnChange(func() error {+ k, err := store.LoadKey()+ if err != nil {+ return err+ }+ c, err := store.Credential()+ if err != nil {+ return err+ }+ mirroredKey = k+ mirroredCred = c+ return nil+ })++ key := []byte("mem-private-key")+ if err := store.SaveKey(key); err != nil {+ t.Fatalf("SaveKey: %v", err)+ }+ key[0] = 'X' // verify defensive copy+ gotKey, err := store.LoadKey()+ if err != nil || string(gotKey) != "mem-private-key" {+ t.Fatalf("LoadKey = %q, %v; want mem-private-key", gotKey, err)+ }+ if string(mirroredKey) != "mem-private-key" {+ t.Fatalf("mirroredKey = %q, want mem-private-key", mirroredKey)+ }++ cpPub := bytes.Repeat([]byte{9}, ed25519.PublicKeySize)+ if err := store.SaveIdentity([]byte("mem-biscuit")); err != nil {+ t.Fatalf("SaveIdentity: %v", err)+ }+ if err := store.SaveIdentityExpiration(1790000000); err != nil {+ t.Fatalf("SaveIdentityExpiration: %v", err)+ }+ if err := store.SaveMeshConfig(cpPub, []string{"/ip4/127.0.0.1/tcp/4001"}); err != nil {+ t.Fatalf("SaveMeshConfig: %v", err)+ }+ if err := store.SaveRefreshToken("rt-mem"); err != nil {+ t.Fatalf("SaveRefreshToken: %v", err)+ }+ if mirroredCred == nil || mirroredCred.GetOidcSession().GetRefreshToken() != "rt-mem" {+ t.Fatalf("mirroredCred refresh_token = %q, want rt-mem", mirroredCred.GetOidcSession().GetRefreshToken())+ }++ if err := store.ResetMeshIdentity(); err != nil {+ t.Fatalf("ResetMeshIdentity: %v", err)+ }+ if _, err := store.LoadIdentity(); err == nil {+ t.Fatal("LoadIdentity succeeded after ResetMeshIdentity, want error")+ }+ if afterResetKey, _ := store.LoadKey(); string(afterResetKey) != "mem-private-key" {+ t.Fatalf("LoadKey after ResetMeshIdentity = %q, want mem-private-key", afterResetKey)+ }++ entries, err := os.ReadDir(emptyTmp)+ if err != nil {+ t.Fatalf("ReadDir(TMPDIR): %v", err)+ }+ if len(entries) != 0 {+ t.Fatalf("NewMemoryStore created %d entries in TMPDIR (%v), want 0", len(entries), entries)+ }+}diff --git a/sdk/meshnet/README.md b/sdk/meshnet/README.mdindex 13a22bcf..31696032 100644--- a/sdk/meshnet/README.md+++ b/sdk/meshnet/README.md@@ -6,10 +6,10 @@ For a full walkthrough, see the [Embedded Go SDK guide](../../site/content/docs/## Capabilities-- **Lifecycle & State Compatibility (`Server.Up`, `WriteStateDir`, `ReadStateDir`):** Built directly on `internal/node.AgentMeshNode` so embedded Go applications and `agentmesh-node` share one PEP. Resumes cached credentials from `StateDir` (including public-client `OIDCSession` recovery that persists rotated `refresh_token`s before `POST /register`), enrolls via `BootstrapTokenPath` / `BootstrapToken` / `JWTPath` / `TokenSource`, returns from `Up` as soon as the first router admits the member while remaining dials up to `Routers` finish in the background, refreshes credentials proactively, backs off exponentially (`2s` to `5m`) and signals `Done()` / `Err()` / `OnCredentialLapse` (`ErrCredentialLapsed`, `Status.CredentialLapsed`) if a credential lapses, verifies signed `/keys` sets, and enforces live `/mesh/events/v1` FloodSub revocations and bans.+- **Lifecycle & State Compatibility (`Server.Up`, `WriteStateDir`, `ReadStateDir`):** Built directly on `internal/node.AgentMeshNode` so embedded Go applications and `agentmesh-node` share one PEP. Holds an exclusive non-blocking lock (`ErrStateDirLocked`) on `StateDir` while running (or runs purely in memory with zero disk files when `StateDir` is empty), resumes cached credentials from `StateDir` (including public-client `OIDCSession` recovery that persists rotated `refresh_token`s before `POST /register`), enrolls via `BootstrapTokenPath` / `BootstrapToken` / `JWTPath` / `TokenSource`, returns from `Up` as soon as the first router admits the member while remaining dials up to `Routers` finish in the background, refreshes credentials proactively, backs off exponentially (`2s` to `5m`) and signals `Done()` / `Err()` / `OnCredentialLapse` (`ErrCredentialLapsed`, `Status.CredentialLapsed`) if a credential lapses, verifies signed `/keys` sets, and enforces live `/mesh/events/v1` FloodSub revocations and bans.- **In-Process A2A Serving (`Server.ListenA2A`):** Announces `a2a://<name>` to the router directory (unless `NoAnnounce: true`) and returns in-memory `net.Listener`s (`PrivateListener` over `/libp2p-http` and optional `PublicListener` over `/mesh/ingress/1.0.0` in `EDGE` or `PASSTHROUGH` mode) plus canonical `PrivateURL` and `PublicURL` endpoints (`PublicURL` is `nil` when `NoAnnounce: true`, when no router advertises an `ingress_url`, or when the hub policy ceiling withholds public exposure). Automatically relativizes and rewrites A2A `AgentCard` `supportedInterfaces` URLs per caller.- **Per-Server Prometheus Metrics (`Server.Registerer`):** Optionally registers `peer_id`-labeled state gauges and refusal/ingress counters (`agentmesh_meshnet_*`) so multiple `Server` instances can share a `prometheus.Registerer` without collision, unregistering all collectors on `Close`.-- **Unspoofable Caller Inspection (`Server.WhoIs`):** Maps `http.Request.RemoteAddr` on `PrivateListener` and `PublicListener` to the cryptographically verified `*CallerInfo` (`PeerID`, `Principal`, `Roles`, `Labels`, `Task`) without trusting HTTP headers.+- **Unspoofable Caller Inspection (`Server.WhoIs`):** Maps `http.Request.RemoteAddr` on `PrivateListener` and `PublicListener` to the cryptographically verified `*CallerInfo` (`PeerID`, `Principal`, `Roles`, `Labels`, `Task`) without trusting HTTP headers. Returns `(nil, false)` for unauthenticated external requests (`INGRESS_AUTH_MODE_PASSTHROUGH` and anonymous `GET /.well-known/agent-card.json` discovery), and sets `PeerID` to `""` for verified external `EDGE` callers without a mesh node identity (such as OIDC users).- **Outbound Mesh `*http.Client` (`Server.HTTPClient`, `Server.Discover`):** Routes `http://agentmesh.local/mesh/<peer_id>/<type>/<name>[/<rest>]` over authenticated libp2p streams, verifying provider Biscuits, `allowed_publish` Datalog grants, `EgressRequireLabels`, and `X-Mesh-Required-Labels` before sending request bytes.- **Offline Task-Scoped Attenuation (`Server.AttenuateToken`):** Appends a validated `TaskAuthorizationRule` (`tar_block` fact, 0 Datalog rules, 0 checks) and optionally seals the Biscuit in memory to delegate task-scoped credentials to sub-agents or external workers without their own mesh client.- **Local Progressive-Discovery MCP Server (`Server.MCPServer`, `Server.MCPHandler`):** Exposes the 6 mesh-operating tools (`list_local_services`, `discover_remote_services`, `get_mesh_info`, `find_remote_tools`, `describe_remote_tool`, `call_remote_tool`) in-process or over Streamable HTTP (for example over a local Unix domain socket).diff --git a/sdk/meshnet/a2a.go b/sdk/meshnet/a2a.goindex 5fbac58a..08054858 100644--- a/sdk/meshnet/a2a.go+++ b/sdk/meshnet/a2a.go@@ -18,6 +18,7 @@ import ("bytes""cmp""context"+ "crypto/ed25519""encoding/json""errors""fmt"@@ -37,6 +38,7 @@ import ("github.com/a2aproject/a2a-go/v2/a2a""github.com/google/agentmesh/api""github.com/google/agentmesh/internal/node"+ "github.com/libp2p/go-libp2p/core/crypto""github.com/libp2p/go-libp2p/core/peer")@@ -221,6 +223,7 @@ func (svc *meshnetA2AService) Handler() http.Handler {targetLn := svc.privateLnif externalIngress {if !svc.ExternalIngressAllowed() {+ svc.s.log().Debug("refusing external ingress request: service ingress not allowed or inactive", "service", svc.name)http.Error(w, "Service not found", http.StatusNotFound)return}@@ -236,6 +239,12 @@ func (svc *meshnetA2AService) Handler() http.Handler {// keyed by http.Request.RemoteAddr. Because connections are in-memory net.Pipe// pairs tracked in Server's active table, WhoIs cannot be spoofed by HTTP// headers.+//+// WhoIs returns (nil, false) for unauthenticated external requests, such as+// INGRESS_AUTH_MODE_PASSTHROUGH requests and anonymous+// GET /.well-known/agent-card.json discovery requests. For verified external+// EDGE callers that carry a user Biscuit rather than a mesh node identity,+// CallerInfo.PeerID is empty.func (s *Server) WhoIs(remoteAddr string) (*CallerInfo, bool) {v, ok := s.activeConns.Load(remoteAddr)if !ok {@@ -258,9 +267,11 @@ func (s *Server) WhoIs(remoteAddr string) (*CallerInfo, bool) {// it to the mesh directory (unless NoAnnounce is true), and returns the// in-memory listeners and canonical URLs to pass to http.Server.Serve.func (s *Server) ListenA2A(ctx context.Context, opts A2AOptions) (*Endpoints, error) {- if !s.isRunning() || s.node.Services() == nil {+ n := s.runningNode()+ if n == nil || n.Services() == nil {return nil, errors.New("meshnet: Server is not running (call Up first)")}+ _, _, peerID, _ := s.stateSnapshot()name := strings.TrimSpace(opts.Name)if name == "" {return nil, errors.New("meshnet: A2AOptions.Name is required")@@ -268,7 +279,7 @@ func (s *Server) ListenA2A(ctx context.Context, opts A2AOptions) (*Endpoints, erif err := api.ValidateServiceFormat(api.ServiceTypeStringA2A + "://" + name); err != nil {return nil, fmt.Errorf("meshnet: invalid A2A service name %q: %w", name, err)}- if s.node.IsServiceRegistered(name) {+ if n.IsServiceRegistered(name) {return nil, fmt.Errorf("meshnet: A2A service %q is already registered", name)}@@ -297,7 +308,7 @@ func (s *Server) ListenA2A(ctx context.Context, opts A2AOptions) (*Endpoints, er}}- privURL, err := url.Parse(fmt.Sprintf("http://agentmesh.local/mesh/%s/%s/%s", s.peerID.String(), api.ServiceTypeStringA2A, name))+ privURL, err := url.Parse(fmt.Sprintf("http://agentmesh.local/mesh/%s/%s/%s", peerID.String(), api.ServiceTypeStringA2A, name))if err != nil {return nil, err}@@ -319,7 +330,7 @@ func (s *Server) ListenA2A(ctx context.Context, opts A2AOptions) (*Endpoints, erprivateURL: privURL,}- if err := s.node.Services().Register(ctx, svc); err != nil {+ if err := n.Services().Register(ctx, svc); err != nil {_ = svc.Teardown()return nil, fmt.Errorf("meshnet: register A2A service %q: %w", name, err)}@@ -327,11 +338,11 @@ func (s *Server) ListenA2A(ctx context.Context, opts A2AOptions) (*Endpoints, er// Announce this member's peer record immediately alongside its service keys// so peers discovering the new service can resolve its addresses without// waiting for the initial reprovide delay.- if !opts.NoAnnounce && s.node.Directory != nil {+ if !opts.NoAnnounce && n.Directory != nil {peerCtx, cancel := context.WithTimeout(ctx, 5*time.Second)- if err := s.node.Directory.Provide(peerCtx, api.PeerKey(s.peerID), ""); err != nil && ctx.Err() == nil {+ if err := n.Directory.Provide(peerCtx, api.PeerKey(peerID), ""); err != nil && ctx.Err() == nil {s.log().Debug("peer directory announcement skipped or failed during ListenA2A",- "peer_id", s.peerID.String(),+ "peer_id", peerID.String(),"service", name,"error", err,)@@ -354,7 +365,7 @@ func (s *Server) ListenA2A(ctx context.Context, opts A2AOptions) (*Endpoints, ercloseOnce.Do(func() {uCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)defer cancel()- if uErr := s.node.UnregisterService(uCtx, name); uErr != nil {+ if uErr := n.UnregisterService(uCtx, name); uErr != nil {s.log().Warn("failed unregistering A2A service", "service", name, "error", uErr)}_ = svc.Teardown()@@ -372,14 +383,15 @@ func (s *Server) ListenA2A(ctx context.Context, opts A2AOptions) (*Endpoints, er}func (s *Server) computePublicURL(name string) *url.URL {- if s.node == nil {+ n, _, peerID, _ := s.stateSnapshot()+ if n == nil {return nil}- base := s.node.PreferredRouterIngressURL()+ base := n.PreferredRouterIngressURL()if base == "" {return nil}- raw := fmt.Sprintf("%s/%s/%s/%s", base, s.peerID.String(), api.ServiceTypeStringA2A, name)+ raw := fmt.Sprintf("%s/%s/%s/%s", base, peerID.String(), api.ServiceTypeStringA2A, name)u, err := url.Parse(raw)if err != nil {s.log().Warn("failed parsing router ingress URL", "service", name, "raw_url", raw, "error", err)@@ -389,20 +401,21 @@ func (s *Server) computePublicURL(name string) *url.URL {}func (s *Server) candidateCardRoots(serviceName string) []string {- roots := []string{"/mesh/" + s.peerID.String() + "/" + api.ServiceTypeStringA2A + "/" + serviceName}+ n, _, peerID, _ := s.stateSnapshot()+ roots := []string{"/mesh/" + peerID.String() + "/" + api.ServiceTypeStringA2A + "/" + serviceName}if pubURL := s.computePublicURL(serviceName); pubURL != nil {if p := strings.TrimSuffix(pubURL.EscapedPath(), "/"); p != "" && !slices.Contains(roots, p) {roots = append(roots, p)}}- if s.node != nil {- for _, base := range s.node.RouterIngressURLs() {+ if n != nil {+ for _, base := range n.RouterIngressURLs() {u, err := url.Parse(base)if err != nil {s.log().Debug("ignoring invalid router ingress base URL when computing card roots", "base_url", base, "error", err)continue}- p := strings.TrimSuffix(u.EscapedPath(), "/") + "/" + s.peerID.String() + "/" + api.ServiceTypeStringA2A + "/" + serviceName+ p := strings.TrimSuffix(u.EscapedPath(), "/") + "/" + peerID.String() + "/" + api.ServiceTypeStringA2A + "/" + serviceNameif !slices.Contains(roots, p) {roots = append(roots, p)}@@ -426,8 +439,14 @@ func (s *Server) deliverHTTP(}seq := s.nextConnSeq.Add(1)- remoteKey := fmt.Sprintf("meshnet:%s#%d", caller.PeerID, seq)- s.activeConns.Store(remoteKey, caller)+ peerLabel := "unverified"+ if caller != nil {+ peerLabel = cmp.Or(caller.PeerID, "external")+ }+ remoteKey := fmt.Sprintf("meshnet:%s#%d", peerLabel, seq)+ if caller != nil {+ s.activeConns.Store(remoteKey, caller)+ }clientSide, serverSide := net.Pipe()serverConn := &trackedConn{@@ -435,7 +454,9 @@ func (s *Server) deliverHTTP(local: ln.Addr(),remote: meshnetAddr(remoteKey),onClose: func() {- s.activeConns.Delete(remoteKey)+ if caller != nil {+ s.activeConns.Delete(remoteKey)+ }},}@@ -451,7 +472,9 @@ func (s *Server) deliverHTTP(dialUsed = trueif eErr := ln.enqueue(dCtx, serverConn); eErr != nil {_ = clientSide.Close()- s.activeConns.Delete(remoteKey)+ if caller != nil {+ s.activeConns.Delete(remoteKey)+ }err = eErrreturn}@@ -471,7 +494,9 @@ func (s *Server) deliverHTTP(if !dialUsed {_ = clientSide.Close()_ = serverConn.Close()- s.activeConns.Delete(remoteKey)+ if caller != nil {+ s.activeConns.Delete(remoteKey)+ }}}()@@ -486,6 +511,7 @@ func (s *Server) deliverHTTP(pr.Out.Header.Del(api.HeaderMeshNoTrailingSlash)},ErrorHandler: func(rw http.ResponseWriter, _ *http.Request, err error) {+ s.log().Warn("A2A reverse proxy stream delivery failed", "remote", remoteKey, "error", err)if errors.Is(err, net.ErrClosed) {http.Error(rw, "Service listener closed", http.StatusServiceUnavailable)return@@ -496,35 +522,65 @@ func (s *Server) deliverHTTP(proxy.ServeHTTP(w, r)}+func canonicalPeerID(raw string) string {+ if raw == "" {+ return ""+ }+ if id, err := peer.Decode(raw); err == nil {+ return id.String()+ }+ return raw+}++// authorityKeyPeerID returns the synthetic peer.ID derived from a Biscuit+// authority Ed25519 public key. Control-plane /oauth/token exchanges without an+// admitted actor node bind actor_node/client_peer_id to this synthetic ID; it+// is not a mesh peer and must not be reported as CallerInfo.PeerID.+func authorityKeyPeerID(pub ed25519.PublicKey) string {+ if len(pub) == 0 {+ return ""+ }+ ck, err := crypto.UnmarshalEd25519PublicKey(pub)+ if err != nil {+ return ""+ }+ id, err := peer.IDFromPublicKey(ck)+ if err != nil {+ return ""+ }+ return id.String()+}+func (s *Server) callerInfoFromRequest(r *http.Request) (*CallerInfo, bool) {externalIngress := r.Header.Get(api.HeaderMeshIngress) != ""- caller := &CallerInfo{- PeerID: r.Header.Get(api.HeaderPeerID),- }- if raw := node.CallerBiscuitFromContext(r.Context()); len(raw) > 0 && s.node != nil {- if claims, err := s.node.VerifyLocalBiscuit(raw); err == nil && claims != nil {- caller.Principal = claims.Principal()- caller.Roles = slices.Clone(claims.Roles)- caller.Labels = maps.Clone(claims.Labels)- caller.Task = claims.InnermostTaskName()- if externalIngress {- if actor := cmp.Or(claims.ActorNodePeerID, claims.NodePeerID); actor != "" {- if id, pErr := peer.Decode(actor); pErr == nil {- caller.PeerID = id.String()- } else {- caller.PeerID = actor- }- }- }+ raw := node.CallerBiscuitFromContext(r.Context())+ n, _, _, _ := s.stateSnapshot()+ if len(raw) == 0 || n == nil {+ return nil, externalIngress+ }+ claims, err := n.VerifyLocalBiscuit(raw)+ if err != nil || claims == nil {+ return nil, externalIngress+ }++ var peerID string+ if externalIngress {+ if claims.NodePeerID != "" {+ peerID = canonicalPeerID(claims.NodePeerID)+ } else if actorID := canonicalPeerID(claims.ActorNodePeerID); actorID != "" && actorID != authorityKeyPeerID(claims.VerifyingKey) {+ peerID = actorID}+ } else {+ peerID = canonicalPeerID(cmp.Or(claims.ClientPeerID, claims.NodePeerID, claims.ActorNodePeerID, r.Header.Get(api.HeaderPeerID)))}- if caller.Principal == "" {- caller.Principal = r.Header.Get(api.HeaderMeshPrincipal)- }- if caller.Task == "" {- caller.Task = r.Header.Get(api.HeaderMeshTask)- }- return caller, externalIngress++ return &CallerInfo{+ PeerID: peerID,+ Principal: cmp.Or(claims.Principal(), r.Header.Get(api.HeaderMeshPrincipal)),+ Roles: slices.Clone(claims.Roles),+ Labels: maps.Clone(claims.Labels),+ Task: cmp.Or(claims.InnermostTaskName(), r.Header.Get(api.HeaderMeshTask)),+ }, externalIngress}const (diff --git a/sdk/meshnet/httpclient.go b/sdk/meshnet/httpclient.goindex a74d085a..cbbdd1f0 100644--- a/sdk/meshnet/httpclient.go+++ b/sdk/meshnet/httpclient.go@@ -49,7 +49,8 @@ type meshRoundTripper struct {func (rt *meshRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {s := rt.s- if !s.isRunning() {+ n := s.runningNode()+ if n == nil {return nil, errors.New("meshnet: Server is not running (call Up first)")}if req.URL == nil {@@ -68,7 +69,8 @@ func (rt *meshRoundTripper) RoundTrip(req *http.Request) (*http.Response, error)if len(parts) < 3 || parts[0] == "" || parts[1] == "" || parts[2] == "" {return httpErrorResponse(req, http.StatusBadRequest, "Invalid mesh path: expected /mesh/<peer_id>/<type>/<name>"), nil}- if _, err := peer.Decode(parts[0]); err != nil {+ pid, err := peer.Decode(parts[0])+ if err != nil {return httpErrorResponse(req, http.StatusBadRequest, "Invalid peer ID"), nil}svcType, err := api.ParseServiceType(strings.ToLower(parts[1]))@@ -77,6 +79,19 @@ func (rt *meshRoundTripper) RoundTrip(req *http.Request) (*http.Response, error)}outReq := req.Clone(req.Context())+ rawPeerSeg := parts[0]+ canonPeerSeg := pid.String()+ if rawPeerSeg != canonPeerSeg {+ outReq.URL.Path = "/mesh/" + canonPeerSeg + strings.TrimPrefix(trimmed, "mesh/"+rawPeerSeg)+ if outReq.URL.RawPath != "" {+ trimmedRaw := strings.TrimPrefix(outReq.URL.RawPath, "/")+ if strings.HasPrefix(trimmedRaw, "mesh/"+rawPeerSeg) {+ outReq.URL.RawPath = "/mesh/" + canonPeerSeg + strings.TrimPrefix(trimmedRaw, "mesh/"+rawPeerSeg)+ } else {+ outReq.URL.RawPath = ""+ }+ }+ }outReq.Header = sanitizeCallerMeshHeaders(req.Header)hostHeader := req.Hostif hostHeader == "" {@@ -89,14 +104,31 @@ func (rt *meshRoundTripper) RoundTrip(req *http.Request) (*http.Response, error)pr, pw := io.Pipe()rw := newPipeResponseWriter(req, pw)- handler := s.node.EgressHandler()+ handler := n.EgressHandler()- go func() {- defer rw.finish()- handler.ServeHTTP(rw, outReq)- }()+ go serveEgressSafe(s, req.URL.Redacted(), outReq, rw, handler)- return rw.waitResponse(pr), nil+ return rw.waitResponse(pr)+}++func serveEgressSafe(s *Server, reqURL string, outReq *http.Request, rw *pipeResponseWriter, handler http.Handler) {+ defer func() {+ if p := recover(); p != nil {+ var abortErr error+ if p == http.ErrAbortHandler {+ abortErr = fmt.Errorf("meshnet: upstream stream aborted: %w", io.ErrUnexpectedEOF)+ } else {+ abortErr = fmt.Errorf("meshnet: egress handler panic: %v", p)+ }+ if s != nil {+ s.log().Warn("egress HTTP round trip aborted", "url", reqURL, "error", abortErr)+ }+ rw.abort(abortErr)+ return+ }+ rw.finish()+ }()+ handler.ServeHTTP(rw, outReq)}func sanitizeCallerMeshHeaders(in http.Header) http.Header {@@ -128,6 +160,7 @@ type pipeResponseWriter struct {ready chan struct{}statusCode intrespHdr http.Header+ abortErr error}func newPipeResponseWriter(req *http.Request, pw *io.PipeWriter) *pipeResponseWriter {@@ -166,8 +199,20 @@ func (w *pipeResponseWriter) finish() {_ = w.pw.Close()}-func (w *pipeResponseWriter) waitResponse(pr *io.PipeReader) *http.Response {+func (w *pipeResponseWriter) abort(err error) {+ w.once.Do(func() {+ w.abortErr = err+ close(w.ready)+ })+ _ = w.pw.CloseWithError(err)+}++func (w *pipeResponseWriter) waitResponse(pr *io.PipeReader) (*http.Response, error) {<-w.ready+ if w.abortErr != nil {+ _ = pr.Close()+ return nil, w.abortErr+ }contentLen := int64(-1)if cl := w.respHdr.Get("Content-Length"); cl != "" {if n, err := strconv.ParseInt(cl, 10, 64); err == nil && n >= 0 {@@ -184,7 +229,7 @@ func (w *pipeResponseWriter) waitResponse(pr *io.PipeReader) *http.Response {Body: pr,ContentLength: contentLen,Request: w.req,- }+ }, nil}func httpErrorResponse(req *http.Request, code int, msg string) *http.Response {diff --git a/sdk/meshnet/meshnet_test.go b/sdk/meshnet/meshnet_test.goindex 5d623bd4..e2c47d1a 100644--- a/sdk/meshnet/meshnet_test.go+++ b/sdk/meshnet/meshnet_test.go@@ -480,8 +480,11 @@ func TestHTTPClientAuthorizerRunLimitReturns504And500(t *testing.T) {if err != nil {t.Fatalf("POST with 1ns BiscuitTimeout: %v", err)}- body504, _ := io.ReadAll(resp504.Body)+ body504, readErr := io.ReadAll(resp504.Body)_ = resp504.Body.Close()+ if readErr != nil {+ t.Fatalf("io.ReadAll(resp504): %v", readErr)+ }if resp504.StatusCode != http.StatusGatewayTimeout {t.Fatalf("authorizer timeout status = %d (%s), want 504 Gateway Timeout", resp504.StatusCode, body504)}@@ -504,8 +507,11 @@ func TestHTTPClientAuthorizerRunLimitReturns504And500(t *testing.T) {if err != nil {t.Fatalf("POST over Datalog fact limit: %v", err)}- body500, _ := io.ReadAll(resp500.Body)+ body500, readErr := io.ReadAll(resp500.Body)_ = resp500.Body.Close()+ if readErr != nil {+ t.Fatalf("io.ReadAll(resp500): %v", readErr)+ }if resp500.StatusCode != http.StatusInternalServerError {t.Fatalf("authorizer max-facts status = %d (%s), want 500 Internal Server Error", resp500.StatusCode, body500)}@@ -1208,8 +1214,11 @@ func TestExternalIngressCanonicalizesActorNodePeerID(t *testing.T) {if err != nil {t.Fatalf("ReadResponse: %v", err)}- respBody, _ := io.ReadAll(resp.Body)+ respBody, readErr := io.ReadAll(resp.Body)_ = resp.Body.Close()+ if readErr != nil {+ t.Fatalf("io.ReadAll(resp): %v", readErr)+ }if resp.StatusCode != http.StatusOK {t.Fatalf("external ingress status = %d (%s), want 200", resp.StatusCode, string(respBody))}@@ -1221,6 +1230,344 @@ func TestExternalIngressCanonicalizesActorNodePeerID(t *testing.T) {}}+func TestWhoIsUnverifiedExternalAndOIDCUserPeerID(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)+ defer cancel()++ env := newStartedTestServer(t, ctx)+ s := env.server++ edgeEP, err := s.ListenA2A(ctx, A2AOptions{+ Name: "edge-agent",+ Public: true,+ IngressAuth: api.IngressAuthMode_INGRESS_AUTH_MODE_EDGE,+ NoAnnounce: true,+ })+ if err != nil {+ t.Fatalf("ListenA2A(edge-agent): %v", err)+ }+ t.Cleanup(func() { _ = edgeEP.Close() })++ passEP, err := s.ListenA2A(ctx, A2AOptions{+ Name: "pass-agent",+ Public: true,+ IngressAuth: api.IngressAuthMode_INGRESS_AUTH_MODE_PASSTHROUGH,+ NoAnnounce: true,+ })+ if err != nil {+ t.Fatalf("ListenA2A(pass-agent): %v", err)+ }+ t.Cleanup(func() { _ = passEP.Close() })++ type whoIsObs struct {+ ok bool+ info *CallerInfo+ }+ var obsMu sync.Mutex+ var lastObs whoIsObs++ recordHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ info, ok := s.WhoIs(r.RemoteAddr)+ obsMu.Lock()+ lastObs = whoIsObs{ok: ok, info: info}+ obsMu.Unlock()+ w.Header().Set("Content-Type", "application/json")+ _, _ = w.Write([]byte(`{"name":"agent"}`))+ })++ edgePubSrv := &http.Server{Handler: recordHandler}+ passPubSrv := &http.Server{Handler: recordHandler}+ go func() { _ = edgePubSrv.Serve(edgeEP.PublicListener) }()+ go func() { _ = passPubSrv.Serve(passEP.PublicListener) }()+ t.Cleanup(func() {+ _ = edgePubSrv.Close()+ _ = passPubSrv.Close()+ })++ routerPeer := newAdmittedTestPeer(t, ctx, s, env.cpPriv, api.RoleRouter)+ routerBiscuit := mintTestBiscuit(t, env.cpPriv, routerPeer.ID().String(), api.RoleRouter, time.Now().Add(time.Hour))++ sendIngressReq := func(t *testing.T, method, rawURL string, hdr http.Header) {+ t.Helper()+ req, err := http.NewRequestWithContext(ctx, method, rawURL, strings.NewReader(`{}`))+ if err != nil {+ t.Fatalf("NewRequestWithContext: %v", err)+ }+ for k, vals := range hdr {+ for _, v := range vals {+ req.Header.Add(k, v)+ }+ }+ req.Header.Set(api.HeaderMeshBiscuit, base64.RawURLEncoding.EncodeToString(routerBiscuit))++ st, err := routerPeer.NewStream(ctx, s.node.Host.ID(), api.IngressProtocolID)+ if err != nil {+ t.Fatalf("NewStream: %v", err)+ }+ defer func() { _ = st.Close() }()+ if err := req.Write(st); err != nil {+ t.Fatalf("req.Write: %v", err)+ }+ _ = st.SetReadDeadline(time.Now().Add(5 * time.Second))+ resp, err := http.ReadResponse(bufio.NewReader(st), req)+ if err != nil {+ t.Fatalf("ReadResponse: %v", err)+ }+ respBody, readErr := io.ReadAll(resp.Body)+ _ = resp.Body.Close()+ if readErr != nil {+ t.Fatalf("io.ReadAll: %v", readErr)+ }+ if resp.StatusCode != http.StatusOK {+ t.Fatalf("status = %d (%s), want 200", resp.StatusCode, string(respBody))+ }+ }++ // 1. Anonymous GET /.well-known/agent-card.json in EDGE mode -> WhoIs returns (nil, false).+ sendIngressReq(t, http.MethodGet, "http://mesh/a2a/edge-agent/.well-known/agent-card.json", http.Header{+ api.HeaderMeshIngress: {api.IngressAuthModeStringEdge},+ })+ obsMu.Lock()+ anonObs := lastObs+ obsMu.Unlock()+ if anonObs.ok || anonObs.info != nil {+ t.Fatalf("anonymous agent-card WhoIs = (%+v, %v), want (nil, false)", anonObs.info, anonObs.ok)+ }++ // 2. PASSTHROUGH request -> WhoIs returns (nil, false), never router's PeerID.+ sendIngressReq(t, http.MethodPost, "http://mesh/a2a/pass-agent/invoke", http.Header{+ api.HeaderMeshIngress: {api.IngressAuthModeStringPassthrough},+ "Authorization": {"Bearer pair-token"},+ })+ obsMu.Lock()+ passObs := lastObs+ obsMu.Unlock()+ if passObs.ok || passObs.info != nil {+ t.Fatalf("PASSTHROUGH WhoIs = (%+v, %v), want (nil, false)", passObs.info, passObs.ok)+ }++ // 3. Verified standalone external OIDC user Biscuit from /oauth/token (actor_node bound to+ // control-plane Biscuit authority signing key, not a mesh node) -> WhoIs returns ok=true with PeerID == "".+ cpAuthorityPeerID := authorityKeyPeerID(env.cpPub)+ if cpAuthorityPeerID == "" {+ t.Fatal("authorityKeyPeerID(env.cpPub) returned empty string")+ }+ userBuilder := biscuit.NewBuilder(env.cpPriv)+ for _, f := range []biscuit.Fact{+ {Predicate: biscuit.Predicate{Name: api.FactActorNode, IDs: []biscuit.Term{biscuit.String(cpAuthorityPeerID)}}},+ {Predicate: biscuit.Predicate{Name: api.FactClientPeerID, IDs: []biscuit.Term{biscuit.String(cpAuthorityPeerID)}}},+ {Predicate: biscuit.Predicate{Name: api.FactRole, IDs: []biscuit.Term{biscuit.String(api.RoleNode)}}},+ {Predicate: biscuit.Predicate{Name: api.FactEmail, IDs: []biscuit.Term{biscuit.String("[email protected]")}}},+ {Predicate: biscuit.Predicate{Name: api.FactExpiration, IDs: []biscuit.Term{biscuit.Date(time.Now().Add(time.Hour))}}},+ api.MarkerFact(api.FactTargetUnrestricted),+ } {+ if err := userBuilder.AddAuthorityFact(f); err != nil {+ t.Fatalf("AddAuthorityFact: %v", err)+ }+ }+ userTok, err := userBuilder.Build()+ if err != nil {+ t.Fatalf("Build: %v", err)+ }+ userBytes, err := userTok.Serialize()+ if err != nil {+ t.Fatalf("Serialize: %v", err)+ }+ sendIngressReq(t, http.MethodPost, "http://mesh/a2a/edge-agent/invoke", http.Header{+ api.HeaderMeshIngress: {api.IngressAuthModeStringEdge},+ api.HeaderMeshIngressCallerBiscuit: {base64.RawURLEncoding.EncodeToString(userBytes)},+ })+ obsMu.Lock()+ userObs := lastObs+ obsMu.Unlock()+ if !userObs.ok || userObs.info == nil {+ t.Fatal("verified external OIDC user WhoIs returned (nil, false), want verified CallerInfo")+ }+ if userObs.info.Principal != "[email protected]" {+ t.Errorf("user WhoIs Principal = %q, want [email protected]", userObs.info.Principal)+ }+ if userObs.info.PeerID != "" {+ t.Errorf("user WhoIs PeerID = %q, want empty string (must not leak router %q or authority key %q)", userObs.info.PeerID, routerPeer.ID(), cpAuthorityPeerID)+ }++ // 4. Delegated Biscuit exchanged by an actual mesh actor node -> WhoIs preserves actor node PeerID.+ actorHost, err := libp2p.New(libp2p.NoListenAddrs)+ if err != nil {+ t.Fatalf("libp2p.New(actorHost): %v", err)+ }+ t.Cleanup(func() { _ = actorHost.Close() })+ actorBuilder := biscuit.NewBuilder(env.cpPriv)+ for _, f := range []biscuit.Fact{+ {Predicate: biscuit.Predicate{Name: api.FactActorNode, IDs: []biscuit.Term{biscuit.String(actorHost.ID().String())}}},+ {Predicate: biscuit.Predicate{Name: api.FactClientPeerID, IDs: []biscuit.Term{biscuit.String(actorHost.ID().String())}}},+ {Predicate: biscuit.Predicate{Name: api.FactRole, IDs: []biscuit.Term{biscuit.String(api.RoleNode)}}},+ {Predicate: biscuit.Predicate{Name: api.FactEmail, IDs: []biscuit.Term{biscuit.String("[email protected]")}}},+ {Predicate: biscuit.Predicate{Name: api.FactExpiration, IDs: []biscuit.Term{biscuit.Date(time.Now().Add(time.Hour))}}},+ api.MarkerFact(api.FactTargetUnrestricted),+ } {+ if err := actorBuilder.AddAuthorityFact(f); err != nil {+ t.Fatalf("AddAuthorityFact(actor): %v", err)+ }+ }+ actorTok, err := actorBuilder.Build()+ if err != nil {+ t.Fatalf("Build(actor): %v", err)+ }+ actorBytes, err := actorTok.Serialize()+ if err != nil {+ t.Fatalf("Serialize(actor): %v", err)+ }+ sendIngressReq(t, http.MethodPost, "http://mesh/a2a/edge-agent/invoke", http.Header{+ api.HeaderMeshIngress: {api.IngressAuthModeStringEdge},+ api.HeaderMeshIngressCallerBiscuit: {base64.RawURLEncoding.EncodeToString(actorBytes)},+ })+ obsMu.Lock()+ actorObs := lastObs+ obsMu.Unlock()+ if !actorObs.ok || actorObs.info == nil {+ t.Fatal("actor-delegated WhoIs returned (nil, false), want verified CallerInfo")+ }+ if actorObs.info.PeerID != actorHost.ID().String() {+ t.Errorf("actor-delegated WhoIs PeerID = %q, want %q", actorObs.info.PeerID, actorHost.ID().String())+ }+}++func TestHTTPClientCanonicalizesCIDv1PeerIDAndRecoversAbort(t *testing.T) {+ t.Run("canonicalizes_cidv1_peer_id_on_egress", func(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)+ defer cancel()++ mesh := newInProcessMeshPair(t, ctx)+ ep, err := mesh.provider.ListenA2A(ctx, A2AOptions{Name: "coder"})+ if err != nil {+ t.Fatalf("ListenA2A: %v", err)+ }+ srv := &http.Server{+ Handler: http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {+ _, _ = w.Write([]byte("cidv1-ok"))+ }),+ }+ go func() { _ = srv.Serve(ep.PrivateListener) }()+ t.Cleanup(func() {+ _ = srv.Close()+ _ = ep.Close()+ })++ cidv1Provider := peer.ToCid(mesh.provider.peerID).String()+ if cidv1Provider == mesh.provider.peerID.String() {+ t.Fatalf("peer.ToCid(%s) did not produce distinct CIDv1 string", mesh.provider.peerID)+ }++ resp, err := mesh.caller.HTTPClient().Get(fmt.Sprintf("http://agentmesh.local/mesh/%s/a2a/coder/ping", cidv1Provider))+ if err != nil {+ t.Fatalf("GET with CIDv1 peer ID: %v", err)+ }+ body, readErr := io.ReadAll(resp.Body)+ _ = resp.Body.Close()+ if readErr != nil {+ t.Fatalf("io.ReadAll: %v", readErr)+ }+ if resp.StatusCode != http.StatusOK || string(body) != "cidv1-ok" {+ t.Fatalf("CIDv1 egress = %d %q, want 200 cidv1-ok", resp.StatusCode, string(body))+ }+ })++ t.Run("recovers_ErrAbortHandler_before_headers", func(t *testing.T) {+ req := httptest.NewRequest(http.MethodGet, "http://agentmesh.local/mesh/test", nil)+ pr, pw := io.Pipe()+ rw := newPipeResponseWriter(req, pw)++ go serveEgressSafe(&Server{}, req.URL.String(), req, rw, http.HandlerFunc(func(http.ResponseWriter, *http.Request) {+ panic(http.ErrAbortHandler)+ }))++ resp, err := rw.waitResponse(pr)+ if !errors.Is(err, io.ErrUnexpectedEOF) {+ t.Fatalf("waitResponse err = %v (resp=%v), want io.ErrUnexpectedEOF", err, resp)+ }+ })++ t.Run("recovers_ErrAbortHandler_mid_stream", func(t *testing.T) {+ req := httptest.NewRequest(http.MethodGet, "http://agentmesh.local/mesh/test", nil)+ pr, pw := io.Pipe()+ rw := newPipeResponseWriter(req, pw)++ go serveEgressSafe(&Server{}, req.URL.String(), req, rw, http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {+ w.WriteHeader(http.StatusOK)+ _, _ = w.Write([]byte("partial-chunk"))+ panic(http.ErrAbortHandler)+ }))++ resp, err := rw.waitResponse(pr)+ if err != nil {+ t.Fatalf("waitResponse after headers: %v", err)+ }+ body, readErr := io.ReadAll(resp.Body)+ _ = resp.Body.Close()+ if string(body) != "partial-chunk" {+ t.Errorf("partial body = %q, want partial-chunk", string(body))+ }+ if !errors.Is(readErr, io.ErrUnexpectedEOF) {+ t.Fatalf("io.ReadAll after mid-stream abort = %v, want io.ErrUnexpectedEOF", readErr)+ }+ })++ t.Run("recovers_unexpected_handler_panic", func(t *testing.T) {+ req := httptest.NewRequest(http.MethodGet, "http://agentmesh.local/mesh/test", nil)+ pr, pw := io.Pipe()+ rw := newPipeResponseWriter(req, pw)++ go serveEgressSafe(&Server{}, req.URL.String(), req, rw, http.HandlerFunc(func(http.ResponseWriter, *http.Request) {+ panic("unexpected-boom")+ }))++ resp, err := rw.waitResponse(pr)+ if err == nil || !strings.Contains(err.Error(), "unexpected-boom") {+ t.Fatalf("waitResponse on unexpected panic = (%v, %v), want error containing unexpected-boom", resp, err)+ }+ })+}++func TestServerConcurrentStatusMetricsAndClose(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)+ defer cancel()++ env := newStartedTestServer(t, ctx)+ reg := prometheus.NewRegistry()+ s := &Server{+ ControlPlaneURL: env.cp.srv.URL,+ BootstrapToken: "boot-secret",+ ListenAddrs: []string{"/ip4/127.0.0.1/tcp/0"},+ AllowInsecure: true,+ Registerer: reg,+ }+ if _, err := s.Up(ctx); err != nil {+ t.Fatalf("s.Up: %v", err)+ }+ t.Cleanup(func() { _ = s.Close() })++ var wg sync.WaitGroup+ for range 4 {+ wg.Add(1)+ go func() {+ defer wg.Done()+ for range 50 {+ _ = s.Status()+ _, _ = reg.Gather()+ _, _, _ = s.AttenuateToken(&api.TaskAuthorizationRule{Name: "race-check"}, time.Minute, false)+ s.onStreamRefused("unauthenticated", string(api.IngressProtocolID))+ }+ }()+ }+ wg.Add(1)+ go func() {+ defer wg.Done()+ time.Sleep(5 * time.Millisecond)+ _ = s.Close()+ }()+ wg.Wait()+}+type inProcessMeshPair struct {provider *Servercaller *Serverdiff --git a/sdk/meshnet/metrics.go b/sdk/meshnet/metrics.goindex 3f8d7732..af9e62ec 100644--- a/sdk/meshnet/metrics.go+++ b/sdk/meshnet/metrics.go@@ -127,14 +127,15 @@ func (c *serverStateCollector) Collect(ch chan<- prometheus.Metric) {if s.credentialLapsed.Load() {lapsedVal = 1}- if s.node != nil && s.node.Host != nil {- peersVal = float64(len(s.node.Host.Network().Peers()))- attachedVal = float64(len(s.node.AttachedRouterIDs()))- if s.node.IsConnected() && lapsedVal == 0 {+ n, store, _, _ := s.stateSnapshot()+ if n != nil && n.Host != nil {+ peersVal = float64(len(n.Host.Network().Peers()))+ attachedVal = float64(len(n.AttachedRouterIDs()))+ if n.IsConnected() && lapsedVal == 0 {readyVal = 1}- if s.node.Services() != nil {- servicesVal = float64(len(s.node.ListLocalServices(api.ServiceType_SERVICE_TYPE_UNSPECIFIED)))+ if n.Services() != nil {+ servicesVal = float64(len(n.ListLocalServices(api.ServiceType_SERVICE_TYPE_UNSPECIFIED)))}}@@ -143,7 +144,7 @@ func (c *serverStateCollector) Collect(ch chan<- prometheus.Metric) {ch <- prometheus.MustNewConstMetric(c.routers, prometheus.GaugeValue, attachedVal, "attached")ch <- prometheus.MustNewConstMetric(c.services, prometheus.GaugeValue, servicesVal)ch <- prometheus.MustNewConstMetric(c.credentialLapse, prometheus.GaugeValue, lapsedVal)- if exp := s.credentialExpireTime(); !exp.IsZero() {+ if exp := credentialExpireTimeFromStore(store); !exp.IsZero() {ch <- prometheus.MustNewConstMetric(c.biscuitExpiry, prometheus.GaugeValue, float64(exp.Unix()))}}@@ -156,8 +157,8 @@ func (s *Server) onStreamRefused(scope, proto string) {}s.refusedStreams[key]++s.refusedMu.Unlock()- if s.metrics != nil {- s.metrics.streamsRefused.WithLabelValues(scope, proto).Inc()+ if m := s.metrics.Load(); m != nil {+ m.streamsRefused.WithLabelValues(scope, proto).Inc()}}diff --git a/sdk/meshnet/server.go b/sdk/meshnet/server.goindex bb7ca14d..40998a9d 100644--- a/sdk/meshnet/server.go+++ b/sdk/meshnet/server.go@@ -59,8 +59,11 @@ var ErrCredentialLapsed = errors.New("meshnet: member credential expired and ren// the mesh in-process.type Server struct {// StateDir persists identity.key and credential.json (0700/0600, byte-for-byte- // compatible with sdk/js, sdk/python, and `agentmesh-node state export|import`).- // When empty, Server runs with an ephemeral in-memory Ed25519 identity.+ // compatible with sdk/js, sdk/python, and `agentmesh-node state export|import`),+ // guarded by an exclusive non-blocking directory lock (ErrStateDirLocked) so two+ // Servers cannot clobber the same state directory concurrently.+ // When empty, Server runs with an ephemeral in-memory Ed25519 identity that+ // creates no files on disk.StateDir string// ControlPlaneURL is the Agent Mesh hub URL (e.g. "https://bananas.sam-mesh.dev").@@ -137,13 +140,13 @@ type Server struct {closed boolrunCancel context.CancelFunc- privKey crypto.PrivKey- peerID peer.ID- cpURL string- store *node.Store- storeTmpDir string- node *node.AgentMeshNode- metrics *serverMetrics+ privKey crypto.PrivKey+ peerID peer.ID+ cpURL string+ store *node.Store+ stateDirLock *os.File+ node *node.AgentMeshNode+ metrics atomic.Pointer[serverMetrics]refusedMu sync.MutexrefusedStreams map[string]uint64@@ -208,10 +211,10 @@ func (s *Server) signalDone() {})}-func (s *Server) handleCredentialLapse(err error) {+func (s *Server) handleCredentialLapse(peerID peer.ID, err error) {s.credentialLapsed.Store(true)- if s.metrics != nil {- s.metrics.credentialLapses.Inc()+ if m := s.metrics.Load(); m != nil {+ m.credentialLapses.Inc()}wrapped := fmt.Errorf("%w: %w", ErrCredentialLapsed, err)s.errMu.Lock()@@ -219,7 +222,7 @@ func (s *Server) handleCredentialLapse(err error) {s.errMu.Unlock()s.signalDone()s.log().Error("member credential expired and renewal failed; retrying with exponential backoff",- "peer_id", s.peerID.String(),+ "peer_id", peerID.String(),"error", err,)if s.OnCredentialLapse != nil {@@ -227,9 +230,9 @@ func (s *Server) handleCredentialLapse(err error) {}}-func (s *Server) handleCredentialRenewed() {+func (s *Server) handleCredentialRenewed(peerID peer.ID) {s.credentialLapsed.Store(false)- s.log().Info("member credential renewed", "peer_id", s.peerID.String())+ s.log().Info("member credential renewed", "peer_id", peerID.String())}// Done returns a channel that is closed when either the member's credential@@ -249,17 +252,35 @@ func (s *Server) Err() error {return s.lapseErr}-func (s *Server) isRunning() bool {+func (s *Server) runningNode() *node.AgentMeshNode {s.lifecycleMu.Lock()defer s.lifecycleMu.Unlock()- return s.started && !s.closed && s.node != nil && s.node.Host != nil+ if !s.started || s.closed || s.node == nil || s.node.Host == nil {+ return nil+ }+ return s.node}-func (s *Server) cleanupStoreTmpDir() {- if s.storeTmpDir != "" {- _ = os.RemoveAll(s.storeTmpDir)- s.storeTmpDir = ""+func (s *Server) stateSnapshot() (*node.AgentMeshNode, *node.Store, peer.ID, string) {+ s.lifecycleMu.Lock()+ defer s.lifecycleMu.Unlock()+ if s.closed {+ return nil, nil, "", ""}+ return s.node, s.store, s.peerID, s.cpURL+}++func (s *Server) closeStoreLocked() error {+ var err error+ if s.store != nil {+ err = s.store.Close()+ s.store = nil+ }+ if s.stateDirLock != nil {+ unlockStateDir(s.stateDirLock)+ s.stateDirLock = nil+ }+ return err}// Up loads or enrolls the member credential, syncs mesh trust material and@@ -277,7 +298,7 @@ func (s *Server) Up(ctx context.Context) (*Status, error) {return nil, errors.New("meshnet: Server has been closed")}if s.started {- st := s.Status()+ st := s.statusLocked()return &st, nil}@@ -310,13 +331,11 @@ func (s *Server) Up(ctx context.Context) (*Status, error) {if s.Registerer != nil {m, err := newServerMetrics(s, s.Registerer)if err != nil {- _ = s.store.Close()- s.cleanupStoreTmpDir()+ _ = s.closeStoreLocked()s.node = nil- s.store = nilreturn nil, fmt.Errorf("meshnet: register metrics: %w", err)}- s.metrics = m+ s.metrics.Store(m)}runCtx, runCancel := context.WithCancel(context.Background())@@ -325,28 +344,23 @@ func (s *Server) Up(ctx context.Context) (*Status, error) {abortedByCaller := !stopAbort()if startErr != nil {runCancel()- if s.metrics != nil {- s.metrics.unregister()- s.metrics = nil+ if m := s.metrics.Swap(nil); m != nil {+ m.unregister()}_ = s.node.Teardown()- if s.store != nil {- _ = s.store.Close()- }- s.cleanupStoreTmpDir()+ _ = s.closeStoreLocked()s.node = nil- s.store = nilif abortedByCaller && ctx.Err() != nil {return nil, ctx.Err()}return nil, startErr}- s.startRenewalLoop(runCtx)+ s.startRenewalLoop(runCtx, s.node, s.store, s.peerID, s.cpURL)s.runCancel = runCancels.started = true- st := s.Status()+ st := s.statusLocked()return &st, nil}@@ -356,6 +370,12 @@ func (s *Server) Up(ctx context.Context) (*Status, error) {// router admission while additional router dials finish in the background,// AttachedRouters may list fewer than Server.Routers immediately after Up.func (s *Server) Status() Status {+ s.lifecycleMu.Lock()+ defer s.lifecycleMu.Unlock()+ return s.statusLocked()+}++func (s *Server) statusLocked() Status {var peerIDStr stringif s.peerID != "" {peerIDStr = s.peerID.String()@@ -363,7 +383,7 @@ func (s *Server) Status() Status {var routerStrs []stringvar ingressURLs []string- if s.node != nil {+ if s.node != nil && !s.closed {if s.node.Host != nil {routers := s.node.AttachedRouterIDs()routerStrs = make([]string, 0, len(routers))@@ -380,7 +400,7 @@ func (s *Server) Status() Status {ControlPlaneURL: s.cpURL,AttachedRouters: routerStrs,IngressURLs: ingressURLs,- ExpireTime: s.credentialExpireTime(),+ ExpireTime: credentialExpireTimeFromStore(s.store),CredentialLapsed: s.credentialLapsed.Load(),}}@@ -398,9 +418,12 @@ func (s *Server) Close() error {s.closed = trues.signalDone()- if s.metrics != nil {- s.metrics.unregister()- s.metrics = nil+ if m := s.metrics.Swap(nil); m != nil {+ m.unregister()+ }++ if s.runCancel != nil {+ s.runCancel()}var errs []error@@ -408,16 +431,11 @@ func (s *Server) Close() error {if err := s.node.Teardown(); err != nil {errs = append(errs, err)}+ s.node = nil}- if s.runCancel != nil {- s.runCancel()- }- if s.store != nil {- if err := s.store.Close(); err != nil {- errs = append(errs, err)- }+ if err := s.closeStoreLocked(); err != nil {+ errs = append(errs, err)}- s.cleanupStoreTmpDir()return errors.Join(errs...)}@@ -426,7 +444,8 @@ func (s *Server) Close() error {// type via parallel catalog fan-out). Returned DiscoveredProvider.LocalProxyUrl// values can be passed directly to Server.HTTPClient.func (s *Server) Discover(ctx context.Context, serviceType api.ServiceType, serviceName string) ([]*api.DiscoveredProvider, error) {- if !s.isRunning() {+ n := s.runningNode()+ if n == nil {return nil, errors.New("meshnet: Server is not running (call Up first)")}if serviceType == api.ServiceType_SERVICE_TYPE_UNSPECIFIED {@@ -435,7 +454,7 @@ func (s *Server) Discover(ctx context.Context, serviceType api.ServiceType, servif _, err := api.ServiceTypeToString(serviceType); err != nil {return nil, err}- return s.node.DiscoverRemoteServices(ctx, serviceType, serviceName)+ return n.DiscoverRemoteServices(ctx, serviceType, serviceName)}// MCPServer returns an in-process *mcp.Server exposing the 6 progressive mesh@@ -443,17 +462,19 @@ func (s *Server) Discover(ctx context.Context, serviceType api.ServiceType, serv// discover_remote_services, find_remote_tools, describe_remote_tool,// call_remote_tool). Returns nil if Up has not succeeded.func (s *Server) MCPServer() *mcp.Server {- if !s.isRunning() {+ n := s.runningNode()+ if n == nil {return nil}- return node.NewMCPServer(s.node)+ return node.NewMCPServer(n)}// MCPHandler returns a stateless StreamableHTTP handler serving Server.MCPServer// at "/", "/mcp", and "/mcp/".func (s *Server) MCPHandler() http.Handler {return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {- if !s.isRunning() {+ n := s.runningNode()+ if n == nil {http.Error(w, "meshnet: Server is not running", http.StatusServiceUnavailable)return}@@ -464,7 +485,7 @@ func (s *Server) MCPHandler() http.Handler {return}s.mcpHandlerOnce.Do(func() {- s.mcpHTTPHandler = node.NewMCPHandler(s.node)+ s.mcpHTTPHandler = node.NewMCPHandler(n)})if r.URL.Path != "/mcp" {r2 := r.Clone(r.Context())@@ -481,12 +502,13 @@ func (s *Server) MCPHandler() http.Handler {// inference://* providers on the mesh.func (s *Server) InferenceHandler() http.Handler {return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {- if !s.isRunning() {+ n := s.runningNode()+ if n == nil {http.Error(w, "meshnet: Server is not running", http.StatusServiceUnavailable)return}s.infHandlerOnce.Do(func() {- s.infHTTPHandler = s.node.InferenceHandler()+ s.infHTTPHandler = n.InferenceHandler()})s.infHTTPHandler.ServeHTTP(w, r)})@@ -497,20 +519,21 @@ func (s *Server) InferenceHandler() http.Handler {// a control-plane round trip, optionally sealing it so downstream holders// cannot append further blocks.func (s *Server) AttenuateToken(rule *api.TaskAuthorizationRule, ttl time.Duration, seal bool) (string, time.Time, error) {- if s.node == nil {+ n, store, _, _ := s.stateSnapshot()+ if n == nil {return "", time.Time{}, errors.New("meshnet: Server is not running (call Up first)")}if rule == nil {return "", time.Time{}, errors.New("TaskAuthorizationRule is required")}- raw := s.node.GetIdentity()+ raw := n.GetIdentity()if len(raw) == 0 {return "", time.Time{}, errors.New("no member credential loaded")}cloned := proto.Clone(rule).(*api.TaskAuthorizationRule)now := time.Now()- exp := s.credentialExpireTime()+ exp := credentialExpireTimeFromStore(store)if exp.IsZero() {exp = now.Add(api.BiscuitTokenTTL)}diff --git a/sdk/meshnet/statedir.go b/sdk/meshnet/statedir.goindex 48446485..fe4e25e4 100644--- a/sdk/meshnet/statedir.go+++ b/sdk/meshnet/statedir.go@@ -50,6 +50,10 @@ const minResumeLifetime = 5 * time.Minute// and no join token or TokenSource was supplied to re-enroll.var ErrCredentialRetired = errors.New("meshnet: saved credential signing key retired; provide a join token to re-enroll")+// ErrStateDirLocked reports that another running Server already holds an+// exclusive lock on StateDir.+var ErrStateDirLocked = errors.New("meshnet: state directory is already locked by another Server")+// WriteStateDir writes a member's libp2p private key and MemberCredential to// dir with owner-only permissions (0700 directory, 0600 files).func WriteStateDir(dir string, privKey []byte, c *api.MemberCredential) error {@@ -212,73 +216,73 @@ func (s *Server) newAgentMeshNode(store *node.Store, privKey crypto.PrivKey) (*nreturn n, nil}-func (s *Server) openStore() (*node.Store, error) {- tmpDir, err := os.MkdirTemp("", "meshnet-store-*")- if err != nil {- return nil, fmt.Errorf("create store workspace: %w", err)+func (s *Server) openStore() (*node.Store, *os.File, error) {+ store := node.NewMemoryStore()++ dir := strings.TrimSpace(s.StateDir)+ if dir == "" {+ return store, nil, nil}- store, err := node.NewStore(tmpDir)++ if err := os.MkdirAll(dir, 0o700); err != nil {+ _ = store.Close()+ return nil, nil, fmt.Errorf("create state directory %s: %w", dir, err)+ }+ lockFile, err := lockStateDir(dir)if err != nil {- _ = os.RemoveAll(tmpDir)- return nil, err+ _ = store.Close()+ return nil, nil, err}- cleanupOnErr := func(e error) (*node.Store, error) {+ cleanupOnErr := func(e error) (*node.Store, *os.File, error) {_ = store.Close()- _ = os.RemoveAll(tmpDir)- return nil, e+ unlockStateDir(lockFile)+ return nil, nil, e}- dir := strings.TrimSpace(s.StateDir)- if dir != "" {- if err := os.MkdirAll(dir, 0o700); err != nil {- return cleanupOnErr(fmt.Errorf("create state directory %s: %w", dir, err))- }- keyPath := filepath.Join(dir, StateDirIdentityFile)- if rawKey, rErr := os.ReadFile(keyPath); rErr == nil {- if _, uErr := crypto.UnmarshalPrivateKey(rawKey); uErr != nil {- return cleanupOnErr(fmt.Errorf("%s: not a libp2p private key: %w", keyPath, uErr))- }- if sErr := store.SaveKey(rawKey); sErr != nil {- return cleanupOnErr(sErr)- }- } else if !errors.Is(rErr, os.ErrNotExist) {- return cleanupOnErr(rErr)+ keyPath := filepath.Join(dir, StateDirIdentityFile)+ if rawKey, rErr := os.ReadFile(keyPath); rErr == nil {+ if _, uErr := crypto.UnmarshalPrivateKey(rawKey); uErr != nil {+ return cleanupOnErr(fmt.Errorf("%s: not a libp2p private key: %w", keyPath, uErr))}-- credPath := filepath.Join(dir, StateDirCredentialFile)- if rawCred, rErr := os.ReadFile(credPath); rErr == nil {- cred := &api.MemberCredential{}- if uErr := protojson.Unmarshal(rawCred, cred); uErr != nil {- return cleanupOnErr(fmt.Errorf("%s: %w", credPath, uErr))- }- if sErr := store.SetCredential(cred); sErr != nil {- return cleanupOnErr(sErr)- }- } else if !errors.Is(rErr, os.ErrNotExist) {- return cleanupOnErr(rErr)+ if sErr := store.SaveKey(rawKey); sErr != nil {+ return cleanupOnErr(sErr)}+ } else if !errors.Is(rErr, os.ErrNotExist) {+ return cleanupOnErr(rErr)+ }- var persistMu sync.Mutex- store.SetOnChange(func() error {- persistMu.Lock()- defer persistMu.Unlock()- kb, lErr := store.LoadKey()- if lErr != nil {- return lErr- }- if len(kb) == 0 {- return nil- }- c, cErr := store.Credential()- if cErr != nil {- return cErr- }- return WriteStateDir(dir, kb, c)- })+ credPath := filepath.Join(dir, StateDirCredentialFile)+ if rawCred, rErr := os.ReadFile(credPath); rErr == nil {+ cred := &api.MemberCredential{}+ if uErr := protojson.Unmarshal(rawCred, cred); uErr != nil {+ return cleanupOnErr(fmt.Errorf("%s: %w", credPath, uErr))+ }+ if sErr := store.SetCredential(cred); sErr != nil {+ return cleanupOnErr(sErr)+ }+ } else if !errors.Is(rErr, os.ErrNotExist) {+ return cleanupOnErr(rErr)}- s.storeTmpDir = tmpDir- return store, nil+ var persistMu sync.Mutex+ store.SetOnChange(func() error {+ persistMu.Lock()+ defer persistMu.Unlock()+ kb, lErr := store.LoadKey()+ if lErr != nil {+ return lErr+ }+ if len(kb) == 0 {+ return nil+ }+ c, cErr := store.Credential()+ if cErr != nil {+ return cErr+ }+ return WriteStateDir(dir, kb, c)+ })++ return store, lockFile, nil}func (s *Server) verifyOwnBiscuit(raw []byte) bool {@@ -323,15 +327,15 @@ func (s *Server) identityPredatesRotation() bool {}func (s *Server) ensureCredential(ctx context.Context) (retErr error) {- store, err := s.openStore()+ store, lockFile, err := s.openStore()if err != nil {return err}+ s.store = store+ s.stateDirLock = lockFiledefer func() {if retErr != nil {- _ = store.Close()- s.cleanupStoreTmpDir()- s.store = nil+ _ = s.closeStoreLocked()s.node = nil}}()@@ -346,7 +350,6 @@ func (s *Server) ensureCredential(ctx context.Context) (retErr error) {}s.privKey = privKeys.peerID = peerID- s.store = storecred, err := store.Credential()if err != nil {@@ -420,7 +423,7 @@ func (s *Server) ensureCredential(ctx context.Context) (retErr error) {if cred != nil && len(cred.GetBiscuit()) > 0 {keysErr := n.SyncTrustedKeys(ctx, s.cpURL)roleOK := s.verifyOwnBiscuit(cred.GetBiscuit())- remaining := time.Until(s.credentialExpireTime())+ remaining := time.Until(credentialExpireTimeFromStore(store))if keysErr == nil && roleOK && remaining > minResumeLifetime && !s.identityPredatesRotation() {s.log().Info("resumed mesh credential from state directory",@@ -507,7 +510,25 @@ func isRetiredVerificationError(err error) bool {return err != nil && strings.Contains(err.Error(), "/keys response rejected")}-func (s *Server) startRenewalLoop(ctx context.Context) {+func nextRenewalDelay(now, expTime time.Time, failureBackoff, initBackoff time.Duration) time.Duration {+ if failureBackoff > 0 {+ return failureBackoff+ }+ if expTime.IsZero() {+ return node.DefaultRenewalFallback+ }+ duration := expTime.Sub(now)+ switch {+ case duration > node.RenewalThreshold:+ return duration - node.RenewalBuffer+ case duration > 0:+ return max(duration/2, initBackoff)+ default:+ return min(time.Second, initBackoff)+ }+}++func (s *Server) startRenewalLoop(ctx context.Context, n *node.AgentMeshNode, store *node.Store, peerID peer.ID, cpURL string) {initBackoff := s.renewalInitialBackoffif initBackoff <= 0 {initBackoff = 2 * time.Second@@ -519,28 +540,7 @@ func (s *Server) startRenewalLoop(ctx context.Context) {go func() {var failureBackoff time.Durationfor {- renewAfter := node.DefaultRenewalFallback- if expTime := s.credentialExpireTime(); !expTime.IsZero() {- duration := time.Until(expTime)- switch {- case duration > node.RenewalThreshold:- renewAfter = duration - node.RenewalBuffer- if failureBackoff > renewAfter {- renewAfter = min(failureBackoff, duration/2)- }- case duration > 0:- renewAfter = max(duration/2, initBackoff)- if failureBackoff > renewAfter {- renewAfter = min(failureBackoff, max(duration/2, initBackoff))- }- case failureBackoff > 0:- renewAfter = failureBackoff- default:- renewAfter = min(time.Second, initBackoff)- }- } else if failureBackoff > 0 {- renewAfter = failureBackoff- }+ renewAfter := nextRenewalDelay(time.Now(), credentialExpireTimeFromStore(store), failureBackoff, initBackoff)timer := time.NewTimer(renewAfter)select {@@ -548,17 +548,17 @@ func (s *Server) startRenewalLoop(ctx context.Context) {timer.Stop()returncase <-timer.C:- refreshErr := s.node.RefreshEnrollment(ctx)+ refreshErr := n.RefreshEnrollment(ctx)if refreshErr == nil {failureBackoff = 0- s.handleCredentialRenewed()+ s.handleCredentialRenewed(peerID)continue}- renewErr := s.fallbackReEnroll(ctx)+ renewErr := s.fallbackReEnroll(ctx, n, store, cpURL)if renewErr == nil {failureBackoff = 0- s.handleCredentialRenewed()+ s.handleCredentialRenewed(peerID)continue}@@ -568,35 +568,35 @@ func (s *Server) startRenewalLoop(ctx context.Context) {failureBackoff = min(failureBackoff*2, maxBackoff)}s.log().Warn("credential renewal failed; scheduled backoff retry",- "peer_id", s.peerID.String(),+ "peer_id", peerID.String(),"next_backoff", failureBackoff,"refresh_error", refreshErr,"reenroll_error", renewErr,)- if exp := s.credentialExpireTime(); !exp.IsZero() && !time.Now().Before(exp) {- s.handleCredentialLapse(errors.Join(refreshErr, renewErr))+ if exp := credentialExpireTimeFromStore(store); !exp.IsZero() && !time.Now().Before(exp) {+ s.handleCredentialLapse(peerID, errors.Join(refreshErr, renewErr))}}}}()}-func (s *Server) fallbackReEnroll(ctx context.Context) error {+func (s *Server) fallbackReEnroll(ctx context.Context, n *node.AgentMeshNode, store *node.Store, cpURL string) error {var errs []errorif ts := s.tokenSource(); ts != nil {newJWT, fetchErr := ts.FetchToken(ctx)if fetchErr != nil {errs = append(errs, fetchErr)- } else if enrollErr := s.node.Enroll(ctx, s.cpURL, newJWT); enrollErr == nil {+ } else if enrollErr := n.Enroll(ctx, cpURL, newJWT); enrollErr == nil {return nil} else {errs = append(errs, enrollErr)}}- if s.store != nil {- if rt, _ := s.store.LoadRefreshToken(); rt != "" {- if oidcErr := s.node.ReEnrollWithRefreshToken(ctx); oidcErr == nil {+ if store != nil {+ if rt, _ := store.LoadRefreshToken(); rt != "" {+ if oidcErr := n.ReEnrollWithRefreshToken(ctx); oidcErr == nil {return nil} else {errs = append(errs, oidcErr)@@ -615,7 +615,7 @@ func (s *Server) fallbackReEnroll(ctx context.Context) error {bootToken = strings.TrimSpace(s.BootstrapToken)}if bootToken != "" {- if _, bErr := s.node.EnrollBootstrapCredential(ctx, s.cpURL, bootToken, true); bErr == nil {+ if _, bErr := n.EnrollBootstrapCredential(ctx, cpURL, bootToken, true); bErr == nil {return nil} else {errs = append(errs, bErr)@@ -632,21 +632,22 @@ func (s *Server) fallbackReEnroll(ctx context.Context) error {// configured), verifies the returned credential, persists it to StateDir, and// re-authenticates attached router sessions.func (s *Server) Refresh(ctx context.Context) error {- if s.node == nil {+ n, _, _, _ := s.stateSnapshot()+ if n == nil {return errors.New("meshnet: Server is not running (call Up first)")}- err := s.node.RefreshEnrollment(ctx)+ err := n.RefreshEnrollment(ctx)if err == nil {s.credentialLapsed.Store(false)}return err}-func (s *Server) credentialExpireTime() time.Time {- if s.store == nil {+func credentialExpireTimeFromStore(store *node.Store) time.Time {+ if store == nil {return time.Time{}}- expUnix, err := s.store.LoadIdentityExpiration()+ expUnix, err := store.LoadIdentityExpiration()if err != nil || expUnix <= 0 {return time.Time{}}diff --git a/sdk/meshnet/statedir_lock_unix.go b/sdk/meshnet/statedir_lock_unix.gonew file mode 100644index 00000000..b582b10e--- /dev/null+++ b/sdk/meshnet/statedir_lock_unix.go@@ -0,0 +1,47 @@+// Copyright 2026 Google LLC+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++//go:build unix++package meshnet++import (+ "errors"+ "fmt"+ "os"+ "syscall"+)++func lockStateDir(dir string) (*os.File, error) {+ f, err := os.Open(dir)+ if err != nil {+ return nil, err+ }+ if err := syscall.Flock(int(f.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil {+ _ = f.Close()+ if errors.Is(err, syscall.EWOULDBLOCK) || errors.Is(err, syscall.EAGAIN) {+ return nil, fmt.Errorf("%w: %s", ErrStateDirLocked, dir)+ }+ return nil, fmt.Errorf("lock state directory %s: %w", dir, err)+ }+ return f, nil+}++func unlockStateDir(f *os.File) {+ if f == nil {+ return+ }+ _ = syscall.Flock(int(f.Fd()), syscall.LOCK_UN)+ _ = f.Close()+}diff --git a/sdk/meshnet/statedir_lock_windows.go b/sdk/meshnet/statedir_lock_windows.gonew file mode 100644index 00000000..c37b960d--- /dev/null+++ b/sdk/meshnet/statedir_lock_windows.go@@ -0,0 +1,53 @@+// Copyright 2026 Google LLC+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++//go:build windows++package meshnet++import (+ "errors"+ "fmt"+ "os"+ "path/filepath"++ "golang.org/x/sys/windows"+)++func lockStateDir(dir string) (*os.File, error) {+ lockPath := filepath.Join(dir, ".lock")+ f, err := os.OpenFile(lockPath, os.O_CREATE|os.O_RDWR, 0o600)+ if err != nil {+ return nil, err+ }+ var ol windows.Overlapped+ flags := uint32(windows.LOCKFILE_EXCLUSIVE_LOCK | windows.LOCKFILE_FAIL_IMMEDIATELY)+ if err := windows.LockFileEx(windows.Handle(f.Fd()), flags, 0, 1, 0, &ol); err != nil {+ _ = f.Close()+ if errors.Is(err, windows.ERROR_LOCK_VIOLATION) {+ return nil, fmt.Errorf("%w: %s", ErrStateDirLocked, dir)+ }+ return nil, fmt.Errorf("lock state directory %s: %w", dir, err)+ }+ return f, nil+}++func unlockStateDir(f *os.File) {+ if f == nil {+ return+ }+ var ol windows.Overlapped+ _ = windows.UnlockFileEx(windows.Handle(f.Fd()), 0, 1, 0, &ol)+ _ = f.Close()+}diff --git a/sdk/meshnet/statedir_test.go b/sdk/meshnet/statedir_test.goindex 00a2acfe..fb351d90 100644--- a/sdk/meshnet/statedir_test.go+++ b/sdk/meshnet/statedir_test.go@@ -35,6 +35,7 @@ import ("github.com/eclipse-biscuit/biscuit-go/v2""github.com/google/agentmesh/api"+ "github.com/google/agentmesh/internal/node""github.com/libp2p/go-libp2p/core/crypto""github.com/libp2p/go-libp2p/core/peer""google.golang.org/protobuf/proto"@@ -184,10 +185,12 @@ func TestEnrollmentAndRenewal(t *testing.T) {BootstrapToken: "boot-secret",AllowInsecure: true,}- t.Cleanup(func() { _ = seed.Close() })if err := seed.ensureCredential(ctx); err != nil {t.Fatalf("seed ensureCredential: %v", err)}+ if err := seed.Close(); err != nil {+ t.Fatalf("seed.Close: %v", err)+ }enrollsBefore := cp.bootstrapEnrolls.Load()refreshesBefore := cp.refreshes.Load()@@ -196,10 +199,12 @@ func TestEnrollmentAndRenewal(t *testing.T) {StateDir: stateDir,AllowInsecure: true,}- t.Cleanup(func() { _ = online.Close() })if err := online.ensureCredential(ctx); err != nil {t.Fatalf("online resume ensureCredential: %v", err)}+ if err := online.Close(); err != nil {+ t.Fatalf("online.Close: %v", err)+ }if cp.bootstrapEnrolls.Load() != enrollsBefore || cp.refreshes.Load() != refreshesBefore {t.Errorf("online resume called enroll/refresh (%d/%d), want (%d/%d)",cp.bootstrapEnrolls.Load(), cp.refreshes.Load(), enrollsBefore, refreshesBefore)@@ -348,10 +353,12 @@ func TestEnrollmentAndRenewal(t *testing.T) {BootstrapToken: "boot-secret",AllowInsecure: true,}- t.Cleanup(func() { _ = seed.Close() })if err := seed.ensureCredential(ctx); err != nil {t.Fatalf("seed ensureCredential: %v", err)}+ if err := seed.Close(); err != nil {+ t.Fatalf("seed.Close: %v", err)+ }// Rotate control-plane key completely (dropping old key from KeysResponse).cp.rotateKeyHard(t)@@ -380,6 +387,148 @@ func TestEnrollmentAndRenewal(t *testing.T) {})}+func TestStateDirLockAndEphemeralMemoryIdentity(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)+ defer cancel()++ env := newStartedTestServer(t, ctx)++ t.Run("ephemeral_empty_StateDir_writes_zero_files_to_TMPDIR", func(t *testing.T) {+ emptyTmp := t.TempDir()+ t.Setenv("TMPDIR", emptyTmp)++ ephemeral := &Server{+ ControlPlaneURL: env.cp.srv.URL,+ BootstrapToken: "boot-secret",+ ListenAddrs: []string{"/ip4/127.0.0.1/tcp/0"},+ AllowInsecure: true,+ }+ st, err := ephemeral.Up(ctx)+ if err != nil {+ t.Fatalf("ephemeral.Up: %v", err)+ }+ if st.PeerID == "" {+ t.Fatal("ephemeral PeerID is empty")+ }++ duringEntries, err := os.ReadDir(emptyTmp)+ if err != nil {+ t.Fatalf("ReadDir(TMPDIR) while running: %v", err)+ }+ if len(duringEntries) != 0 {+ t.Fatalf("ephemeral Server wrote %d entries to TMPDIR while running (%v), want 0", len(duringEntries), duringEntries)+ }++ if err := ephemeral.Close(); err != nil {+ t.Fatalf("ephemeral.Close: %v", err)+ }+ afterEntries, err := os.ReadDir(emptyTmp)+ if err != nil {+ t.Fatalf("ReadDir(TMPDIR) after Close: %v", err)+ }+ if len(afterEntries) != 0 {+ t.Fatalf("ephemeral Server left %d entries in TMPDIR (%v), want 0", len(afterEntries), afterEntries)+ }+ })++ t.Run("concurrent_Servers_on_same_StateDir_return_ErrStateDirLocked", func(t *testing.T) {+ sharedDir := t.TempDir()+ s1 := &Server{+ StateDir: sharedDir,+ ControlPlaneURL: env.cp.srv.URL,+ BootstrapToken: "boot-secret",+ ListenAddrs: []string{"/ip4/127.0.0.1/tcp/0"},+ AllowInsecure: true,+ }+ st1, err := s1.Up(ctx)+ if err != nil {+ t.Fatalf("s1.Up: %v", err)+ }+ t.Cleanup(func() { _ = s1.Close() })++ s2 := &Server{+ StateDir: sharedDir,+ ControlPlaneURL: env.cp.srv.URL,+ BootstrapToken: "boot-secret",+ ListenAddrs: []string{"/ip4/127.0.0.1/tcp/0"},+ AllowInsecure: true,+ }+ if _, err := s2.Up(ctx); !errors.Is(err, ErrStateDirLocked) {+ _ = s2.Close()+ t.Fatalf("s2.Up on locked StateDir = %v, want ErrStateDirLocked", err)+ }++ if err := s1.Close(); err != nil {+ t.Fatalf("s1.Close: %v", err)+ }++ st2, err := s2.Up(ctx)+ if err != nil {+ t.Fatalf("s2.Up after s1.Close: %v", err)+ }+ t.Cleanup(func() { _ = s2.Close() })+ if st2.PeerID != st1.PeerID {+ t.Fatalf("s2 PeerID = %q, want %q from shared StateDir", st2.PeerID, st1.PeerID)+ }+ })+}++func TestNextRenewalDelay(t *testing.T) {+ now := time.Date(2026, 11, 1, 12, 0, 0, 0, time.UTC)+ initBackoff := 2 * time.Second++ for _, tc := range []struct {+ name string+ expTime time.Time+ failureBackoff time.Duration+ want time.Duration+ }{+ {+ name: "unknown_expiry_uses_fallback",+ expTime: time.Time{},+ want: node.DefaultRenewalFallback,+ },+ {+ name: "healthy_long_lived_renews_before_expiry_buffer",+ expTime: now.Add(24 * time.Hour),+ want: 24*time.Hour - node.RenewalBuffer,+ },+ {+ name: "short_lived_renews_at_half_lifetime",+ expTime: now.Add(4 * time.Minute),+ want: 2 * time.Minute,+ },+ {+ name: "very_short_lived_floored_at_initBackoff",+ expTime: now.Add(500 * time.Millisecond),+ want: initBackoff,+ },+ {+ name: "already_expired_without_prior_failure_retries_within_1s",+ expTime: now.Add(-time.Minute),+ want: time.Second,+ },+ {+ name: "failure_backoff_honored_before_expiry",+ expTime: now.Add(2 * time.Minute),+ failureBackoff: 8 * time.Second,+ want: 8 * time.Second,+ },+ {+ name: "failure_backoff_honored_after_expiry",+ expTime: now.Add(-time.Minute),+ failureBackoff: 16 * time.Second,+ want: 16 * time.Second,+ },+ } {+ t.Run(tc.name, func(t *testing.T) {+ if got := nextRenewalDelay(now, tc.expTime, tc.failureBackoff, initBackoff); got != tc.want {+ t.Fatalf("nextRenewalDelay() = %v, want %v", got, tc.want)+ }+ })+ }+}+type fakeControlPlaneHTTP struct {mu sync.MutexcpPub ed25519.PublicKeydiff --git a/site/content/docs/guides/go-sdk.md b/site/content/docs/guides/go-sdk.mdindex bce41865..96651ec5 100644--- a/site/content/docs/guides/go-sdk.md+++ b/site/content/docs/guides/go-sdk.md@@ -243,7 +243,11 @@ Public A2A: http://127.0.0.1:33776/ingress/12D3KooW.../a2a/coder- **Unspoofable caller identity (`Server.WhoIs`):** Inside your HTTP handler,`srv.WhoIs(r.RemoteAddr)` returns the cryptographically verified `*CallerInfo`(`PeerID`, `Principal`, `Roles`, `Labels`, `Task`) bound to that in-memory- connection—never trusting spoofable HTTP headers.+ connection—never trusting spoofable HTTP headers. For unauthenticated external+ requests (`INGRESS_AUTH_MODE_PASSTHROUGH` and anonymous+ `GET /.well-known/agent-card.json` discovery), `WhoIs` returns `(nil, false)`.+ For verified external `EDGE` callers without a mesh node identity (such as OIDC+ users), `CallerInfo.PeerID` is empty (`""`).- **`EDGE` vs. `PASSTHROUGH` ingress:** `INGRESS_AUTH_MODE_EDGE` (the default)verifies Delegated Session Biscuits at both the router and the `meshnet`server (`identity.AuthorizeMeshRequest` + `IngressAllow`), while still serving@@ -447,7 +451,10 @@ Code, Gemini CLI, or a sandboxed coding tool), mount `srv.MCPHandler()` and`Server.StateDir` persists `identity.key` and `credential.json` with owner-onlypermissions (`0700` directory, `0600` files) using the exact protobuf/protojson-layout shared by `agentmesh-node`, `sdk/js`, and `sdk/python`.+layout shared by `agentmesh-node`, `sdk/js`, and `sdk/python`, guarded by an+exclusive non-blocking directory lock (`meshnet.ErrStateDirLocked`) so two+`Server` instances cannot clobber the same directory concurrently. When+`StateDir` is empty, `Server` runs purely in memory with zero files on disk.An operator can enroll interactively once with `agentmesh-node` and pass theexported state directory directly to a Go binary built on `sdk/meshnet` (with nodiff --git a/tests/integration/meshnet_test.go b/tests/integration/meshnet_test.goindex 0a768002..765c5262 100644--- a/tests/integration/meshnet_test.go+++ b/tests/integration/meshnet_test.go@@ -430,7 +430,8 @@ func TestMeshnetEndToEndIntegration(t *testing.T) {pairedMux := http.NewServeMux()pairedMux.Handle(a2asrv.WellKnownAgentCardPath, a2asrv.NewStaticAgentCardHandler(pairedCard))pairedMux.Handle("/invoke", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {- if r.Header.Get(api.HeaderMeshPrincipal) == "" && r.Header.Get(api.HeaderMeshIngress) == api.IngressAuthModeStringPassthrough {+ _, hasWhoIs := srvA.WhoIs(r.RemoteAddr)+ if !hasWhoIs && r.Header.Get(api.HeaderMeshPrincipal) == "" && r.Header.Get(api.HeaderMeshIngress) == api.IngressAuthModeStringPassthrough {passthroughClean.Store(true)}if r.Header.Get("Authorization") != "Bearer pair-secret-123" {@@ -567,6 +568,9 @@ func TestMeshnetEndToEndIntegration(t *testing.T) {if gotPubWhoIs.Principal != "[email protected]" {t.Fatalf("public WhoIs Principal = %q, want [email protected]", gotPubWhoIs.Principal)}+ if gotPubWhoIs.PeerID != "" {+ t.Fatalf("public OIDC user WhoIs PeerID = %q, want empty string", gotPubWhoIs.PeerID)+ }// 5b. External OIDC user excluded by IngressAllow gets denied.bobJWT := mintJWT(map[string]any{"sub": "bob-sub", "email": "[email protected]"})@@ -609,6 +613,9 @@ func TestMeshnetEndToEndIntegration(t *testing.T) {if gotTARWhoIs.Task != "audit-pr-99" {t.Fatalf("TAR WhoIs Task = %q, want audit-pr-99", gotTARWhoIs.Task)}+ if gotTARWhoIs.PeerID != stB.PeerID {+ t.Fatalf("TAR WhoIs PeerID = %q, want %q", gotTARWhoIs.PeerID, stB.PeerID)+ }// 5d. PASSTHROUGH service preserves upstream Bearer pair token and strips spoofed X-Mesh-*.pubPairedCard, err := agentcard.NewResolver(extHTTP).Resolve(ctx, pairedEP.PublicURL.String())