Skip to content

Agent Mesh

internal/node, sdk/meshnet: revert Store changes and reuse bbolt StateDir lock

Antonio Ojea9d877aaa5f22
diff --git a/internal/node/store.go b/internal/node/store.go
index 84c7b981..de4a937d 100644
--- a/internal/node/store.go
+++ b/internal/node/store.go
@@ -23,7 +23,6 @@ import (
"path/filepath"
"strconv"
"strings"
- "sync"
"time"
"github.com/google/agentmesh/api"
@@ -65,38 +64,9 @@ var legacyKeys = []string{
// Store is the node's persistent state: its private key and its
// api.MemberCredential, in a bbolt database whose file lock also tells a
-// second agentmesh-node that this data directory is in use (or purely in memory
-// when constructed with NewMemoryStore).
+// second agentmesh-node that this data directory is in use.
type Store struct {
- db *bbolt.DB
- memMu sync.RWMutex
- memKey []byte
- memCred *api.MemberCredential
- onChange func() error
-}
-
-// NewMemoryStore returns an ephemeral in-memory Store that creates no files on
-// disk.
-func NewMemoryStore() *Store {
- return &Store{memCred: &api.MemberCredential{}}
-}
-
-// SetOnChange registers an optional post-commit callback invoked after key or
-// credential mutations complete.
-func (s *Store) SetOnChange(fn func() error) {
- s.memMu.Lock()
- s.onChange = fn
- s.memMu.Unlock()
-}
-
-func (s *Store) notifyChange() error {
- s.memMu.RLock()
- fn := s.onChange
- s.memMu.RUnlock()
- if fn != nil {
- return fn()
- }
- return nil
+ db *bbolt.DB
}
// ErrStoreLocked reports that another process already holds the data
@@ -239,12 +209,6 @@ func getCredential(b *bbolt.Bucket) (*api.MemberCredential, error) {
// view reads the credential; an empty message stands for a node that has not enrolled.
func (s *Store) view(fn func(c *api.MemberCredential) error) error {
- if s.db == nil {
- s.memMu.RLock()
- cloned := proto.Clone(s.memCred).(*api.MemberCredential)
- s.memMu.RUnlock()
- return fn(cloned)
- }
return s.db.View(func(tx *bbolt.Tx) error {
c, err := getCredential(tx.Bucket([]byte(bucketIdentity)))
if err != nil {
@@ -256,20 +220,7 @@ func (s *Store) view(fn func(c *api.MemberCredential) error) error {
// update applies fn to the credential and writes it back in one transaction.
func (s *Store) update(fn func(c *api.MemberCredential) error) error {
- if s.db == nil {
- s.memMu.Lock()
- next := proto.Clone(s.memCred).(*api.MemberCredential)
- err := fn(next)
- if err == nil {
- s.memCred = next
- }
- s.memMu.Unlock()
- if err != nil {
- return err
- }
- return s.notifyChange()
- }
- if err := s.db.Update(func(tx *bbolt.Tx) error {
+ return s.db.Update(func(tx *bbolt.Tx) error {
b := tx.Bucket([]byte(bucketIdentity))
c, err := getCredential(b)
if err != nil {
@@ -279,10 +230,7 @@ func (s *Store) update(fn func(c *api.MemberCredential) error) error {
return err
}
return putCredential(b, c)
- }); err != nil {
- return err
- }
- return s.notifyChange()
+ })
}
// Credential returns a copy of everything the node persists besides its
@@ -299,21 +247,9 @@ func (s *Store) Credential() (*api.MemberCredential, error) {
// SetCredential replaces everything the node persists besides its private
// key; `agentmesh-node state import` uses it.
func (s *Store) SetCredential(c *api.MemberCredential) error {
- if c == nil {
- c = &api.MemberCredential{}
- }
- if s.db == nil {
- s.memMu.Lock()
- s.memCred = proto.Clone(c).(*api.MemberCredential)
- s.memMu.Unlock()
- return s.notifyChange()
- }
- if err := s.db.Update(func(tx *bbolt.Tx) error {
+ return s.db.Update(func(tx *bbolt.Tx) error {
return putCredential(tx.Bucket([]byte(bucketIdentity)), c)
- }); err != nil {
- return err
- }
- return s.notifyChange()
+ })
}
func (s *Store) SaveIdentity(biscuit []byte) error {
@@ -402,27 +338,12 @@ func (s *Store) LoadOIDCConfig() (string, string, string, error) {
}
func (s *Store) SaveKey(key []byte) error {
- if s.db == nil {
- s.memMu.Lock()
- s.memKey = append([]byte(nil), key...)
- s.memMu.Unlock()
- return s.notifyChange()
- }
- if err := s.db.Update(func(tx *bbolt.Tx) error {
+ return s.db.Update(func(tx *bbolt.Tx) error {
return tx.Bucket([]byte(bucketIdentity)).Put([]byte(keyPrivKey), key)
- }); err != nil {
- return err
- }
- return s.notifyChange()
+ })
}
func (s *Store) LoadKey() ([]byte, error) {
- if s.db == nil {
- s.memMu.RLock()
- val := append([]byte(nil), s.memKey...)
- s.memMu.RUnlock()
- return val, nil
- }
var val []byte
_ = s.db.View(func(tx *bbolt.Tx) error {
val = append([]byte(nil), tx.Bucket([]byte(bucketIdentity)).Get([]byte(keyPrivKey))...)
@@ -534,18 +455,9 @@ func (s *Store) LoadControlPlaneURL() (string, error) {
// a different mesh. It keeps the long-lived libp2p key (node_private_key), so
// the node's PeerID survives the switch.
func (s *Store) ResetMeshIdentity() error {
- if s.db == nil {
- s.memMu.Lock()
- s.memCred = &api.MemberCredential{}
- s.memMu.Unlock()
- return s.notifyChange()
- }
- if err := s.db.Update(func(tx *bbolt.Tx) error {
+ return s.db.Update(func(tx *bbolt.Tx) error {
return tx.Bucket([]byte(bucketIdentity)).Delete([]byte(keyCredential))
- }); err != nil {
- return err
- }
- return s.notifyChange()
+ })
}
func trustedKeysToProto(keys []TrustedKey) []*api.TrustedSigningKey {
@@ -596,9 +508,6 @@ func containsKeyBytes(keys [][]byte, want []byte) bool {
}
func (s *Store) Close() error {
- if s.db == nil {
- return nil
- }
return s.db.Close()
}
diff --git a/internal/node/store_test.go b/internal/node/store_test.go
index bbce4585..10f76fb7 100644
--- a/internal/node/store_test.go
+++ b/internal/node/store_test.go
@@ -578,118 +578,3 @@ func TestGetDefaultDataDir(t *testing.T) {
t.Fatalf("GetDefaultDataDir = %q, want %q", got, custom)
}
}
-
-func TestStore_SetOnChange(t *testing.T) {
- store, err := NewStore(t.TempDir())
- if err != nil {
- t.Fatalf("NewStore: %v", err)
- }
- defer func() { _ = store.Close() }()
-
- calls := 0
- var hookErr error
- store.SetOnChange(func() error {
- calls++
- return hookErr
- })
-
- if err := store.SaveKey([]byte("priv-key")); err != nil {
- t.Fatalf("SaveKey: %v", err)
- }
- if err := store.SaveIdentity([]byte("biscuit")); err != nil {
- t.Fatalf("SaveIdentity: %v", err)
- }
- if err := store.SaveRefreshToken("rt-rotated"); err != nil {
- t.Fatalf("SaveRefreshToken: %v", err)
- }
- if err := store.SetCredential(&api.MemberCredential{OidcSession: &api.OIDCSession{RefreshToken: "rt-2"}}); err != nil {
- t.Fatalf("SetCredential: %v", err)
- }
- if err := store.ResetMeshIdentity(); err != nil {
- t.Fatalf("ResetMeshIdentity: %v", err)
- }
- if calls != 5 {
- t.Fatalf("onChange calls = %d, want 5", calls)
- }
-
- hookErr = errors.New("sync disk full")
- if err := store.SaveRefreshToken("rt-3"); !errors.Is(err, hookErr) {
- t.Fatalf("SaveRefreshToken with failing onChange = %v, want %v", err, hookErr)
- }
-}
-
-func TestStore_NewMemoryStore(t *testing.T) {
- emptyTmp := t.TempDir()
- t.Setenv("TMPDIR", emptyTmp)
-
- store := NewMemoryStore()
- defer func() {
- if err := store.Close(); err != nil {
- t.Errorf("Close: %v", err)
- }
- }()
-
- var mirroredKey []byte
- var mirroredCred *api.MemberCredential
- store.SetOnChange(func() error {
- k, err := store.LoadKey()
- if err != nil {
- return err
- }
- c, err := store.Credential()
- if err != nil {
- return err
- }
- mirroredKey = k
- mirroredCred = c
- return nil
- })
-
- key := []byte("mem-private-key")
- if err := store.SaveKey(key); err != nil {
- t.Fatalf("SaveKey: %v", err)
- }
- key[0] = 'X' // verify defensive copy
- gotKey, err := store.LoadKey()
- if err != nil || string(gotKey) != "mem-private-key" {
- t.Fatalf("LoadKey = %q, %v; want mem-private-key", gotKey, err)
- }
- if string(mirroredKey) != "mem-private-key" {
- t.Fatalf("mirroredKey = %q, want mem-private-key", mirroredKey)
- }
-
- cpPub := bytes.Repeat([]byte{9}, ed25519.PublicKeySize)
- if err := store.SaveIdentity([]byte("mem-biscuit")); err != nil {
- t.Fatalf("SaveIdentity: %v", err)
- }
- if err := store.SaveIdentityExpiration(1790000000); err != nil {
- t.Fatalf("SaveIdentityExpiration: %v", err)
- }
- if err := store.SaveMeshConfig(cpPub, []string{"/ip4/127.0.0.1/tcp/4001"}); err != nil {
- t.Fatalf("SaveMeshConfig: %v", err)
- }
- if err := store.SaveRefreshToken("rt-mem"); err != nil {
- t.Fatalf("SaveRefreshToken: %v", err)
- }
- if mirroredCred == nil || mirroredCred.GetOidcSession().GetRefreshToken() != "rt-mem" {
- t.Fatalf("mirroredCred refresh_token = %q, want rt-mem", mirroredCred.GetOidcSession().GetRefreshToken())
- }
-
- if err := store.ResetMeshIdentity(); err != nil {
- t.Fatalf("ResetMeshIdentity: %v", err)
- }
- if _, err := store.LoadIdentity(); err == nil {
- t.Fatal("LoadIdentity succeeded after ResetMeshIdentity, want error")
- }
- if afterResetKey, _ := store.LoadKey(); string(afterResetKey) != "mem-private-key" {
- t.Fatalf("LoadKey after ResetMeshIdentity = %q, want mem-private-key", afterResetKey)
- }
-
- entries, err := os.ReadDir(emptyTmp)
- if err != nil {
- t.Fatalf("ReadDir(TMPDIR): %v", err)
- }
- if len(entries) != 0 {
- t.Fatalf("NewMemoryStore created %d entries in TMPDIR (%v), want 0", len(entries), entries)
- }
-}
diff --git a/sdk/meshnet/README.md b/sdk/meshnet/README.md
index 31696032..af5120ba 100644
--- a/sdk/meshnet/README.md
+++ b/sdk/meshnet/README.md
@@ -6,7 +6,7 @@ For a full walkthrough, see the [Embedded Go SDK guide](../../site/content/docs/
## Capabilities
-- **Lifecycle & State Compatibility (`Server.Up`, `WriteStateDir`, `ReadStateDir`):** Built directly on `internal/node.AgentMeshNode` so embedded Go applications and `agentmesh-node` share one PEP. Holds an exclusive non-blocking lock (`ErrStateDirLocked`) on `StateDir` while running (or runs purely in memory with zero disk files when `StateDir` is empty), resumes cached credentials from `StateDir` (including public-client `OIDCSession` recovery that persists rotated `refresh_token`s before `POST /register`), enrolls via `BootstrapTokenPath` / `BootstrapToken` / `JWTPath` / `TokenSource`, returns from `Up` as soon as the first router admits the member while remaining dials up to `Routers` finish in the background, refreshes credentials proactively, backs off exponentially (`2s` to `5m`) and signals `Done()` / `Err()` / `OnCredentialLapse` (`ErrCredentialLapsed`, `Status.CredentialLapsed`) if a credential lapses, verifies signed `/keys` sets, and enforces live `/mesh/events/v1` FloodSub revocations and bans.
+- **Lifecycle & State Compatibility (`Server.Up`, `WriteStateDir`, `ReadStateDir`):** Built directly on `internal/node.AgentMeshNode` so embedded Go applications and `agentmesh-node` share one PEP. Holds the `bbolt` store lock (`ErrStateDirLocked`) on `StateDir` while running (or uses a temporary `0700` directory removed on `Close` when `StateDir` is empty), resumes cached credentials from `StateDir` (including public-client `OIDCSession` recovery that persists rotated `refresh_token`s before `POST /register`), enrolls via `BootstrapTokenPath` / `BootstrapToken` / `JWTPath` / `TokenSource`, returns from `Up` as soon as the first router admits the member while remaining dials up to `Routers` finish in the background, refreshes credentials proactively, backs off exponentially (`2s` to `5m`) and signals `Done()` / `Err()` / `OnCredentialLapse` (`ErrCredentialLapsed`, `Status.CredentialLapsed`) if a credential lapses, verifies signed `/keys` sets, and enforces live `/mesh/events/v1` FloodSub revocations and bans.
- **In-Process A2A Serving (`Server.ListenA2A`):** Announces `a2a://<name>` to the router directory (unless `NoAnnounce: true`) and returns in-memory `net.Listener`s (`PrivateListener` over `/libp2p-http` and optional `PublicListener` over `/mesh/ingress/1.0.0` in `EDGE` or `PASSTHROUGH` mode) plus canonical `PrivateURL` and `PublicURL` endpoints (`PublicURL` is `nil` when `NoAnnounce: true`, when no router advertises an `ingress_url`, or when the hub policy ceiling withholds public exposure). Automatically relativizes and rewrites A2A `AgentCard` `supportedInterfaces` URLs per caller.
- **Per-Server Prometheus Metrics (`Server.Registerer`):** Optionally registers `peer_id`-labeled state gauges and refusal/ingress counters (`agentmesh_meshnet_*`) so multiple `Server` instances can share a `prometheus.Registerer` without collision, unregistering all collectors on `Close`.
- **Unspoofable Caller Inspection (`Server.WhoIs`):** Maps `http.Request.RemoteAddr` on `PrivateListener` and `PublicListener` to the cryptographically verified `*CallerInfo` (`PeerID`, `Principal`, `Roles`, `Labels`, `Task`) without trusting HTTP headers. Returns `(nil, false)` for unauthenticated external requests (`INGRESS_AUTH_MODE_PASSTHROUGH` and anonymous `GET /.well-known/agent-card.json` discovery), and sets `PeerID` to `""` for verified external `EDGE` callers without a mesh node identity (such as OIDC users).
diff --git a/sdk/meshnet/server.go b/sdk/meshnet/server.go
index 40998a9d..a1dba0fc 100644
--- a/sdk/meshnet/server.go
+++ b/sdk/meshnet/server.go
@@ -59,11 +59,11 @@ var ErrCredentialLapsed = errors.New("meshnet: member credential expired and ren
// the mesh in-process.
type Server struct {
// StateDir persists identity.key and credential.json (0700/0600, byte-for-byte
- // compatible with sdk/js, sdk/python, and `agentmesh-node state export|import`),
- // guarded by an exclusive non-blocking directory lock (ErrStateDirLocked) so two
- // Servers cannot clobber the same state directory concurrently.
- // When empty, Server runs with an ephemeral in-memory Ed25519 identity that
- // creates no files on disk.
+ // compatible with sdk/js, sdk/python, and `agentmesh-node state export|import`)
+ // alongside the local bbolt store lock (agent.db), returning ErrStateDirLocked
+ // if another Server or agentmesh-node instance already holds the directory.
+ // When empty, Up creates an ephemeral Ed25519 identity in a temporary 0700
+ // directory that is removed on Close (or if Up fails).
StateDir string
// ControlPlaneURL is the Agent Mesh hub URL (e.g. "https://bananas.sam-mesh.dev").
@@ -140,13 +140,14 @@ type Server struct {
closed bool
runCancel context.CancelFunc
- privKey crypto.PrivKey
- peerID peer.ID
- cpURL string
- store *node.Store
- stateDirLock *os.File
- node *node.AgentMeshNode
- metrics atomic.Pointer[serverMetrics]
+ privKey crypto.PrivKey
+ peerID peer.ID
+ cpURL string
+ store *node.Store
+ storeTmpDir string
+ stateDirSyncMu sync.Mutex
+ node *node.AgentMeshNode
+ metrics atomic.Pointer[serverMetrics]
refusedMu sync.Mutex
refusedStreams map[string]uint64
@@ -271,16 +272,25 @@ func (s *Server) stateSnapshot() (*node.AgentMeshNode, *node.Store, peer.ID, str
}
func (s *Server) closeStoreLocked() error {
- var err error
+ var errs []error
if s.store != nil {
- err = s.store.Close()
+ if s.started {
+ if err := s.syncStateDir(s.store); err != nil {
+ errs = append(errs, err)
+ }
+ }
+ if err := s.store.Close(); err != nil {
+ errs = append(errs, err)
+ }
s.store = nil
}
- if s.stateDirLock != nil {
- unlockStateDir(s.stateDirLock)
- s.stateDirLock = nil
+ if s.storeTmpDir != "" {
+ if err := os.RemoveAll(s.storeTmpDir); err != nil {
+ errs = append(errs, err)
+ }
+ s.storeTmpDir = ""
}
- return err
+ return errors.Join(errs...)
}
// Up loads or enrolls the member credential, syncs mesh trust material and
@@ -356,6 +366,9 @@ func (s *Server) Up(ctx context.Context) (*Status, error) {
return nil, startErr
}
+ if err := s.syncStateDir(s.store); err != nil {
+ s.log().Warn("failed to sync state directory after startup", "error", err)
+ }
s.startRenewalLoop(runCtx, s.node, s.store, s.peerID, s.cpURL)
s.runCancel = runCancel
s.started = true
diff --git a/sdk/meshnet/statedir.go b/sdk/meshnet/statedir.go
index fe4e25e4..defc1abd 100644
--- a/sdk/meshnet/statedir.go
+++ b/sdk/meshnet/statedir.go
@@ -24,7 +24,6 @@ import (
"path/filepath"
"slices"
"strings"
- "sync"
"time"
"github.com/google/agentmesh/api"
@@ -50,9 +49,9 @@ const minResumeLifetime = 5 * time.Minute
// and no join token or TokenSource was supplied to re-enroll.
var ErrCredentialRetired = errors.New("meshnet: saved credential signing key retired; provide a join token to re-enroll")
-// ErrStateDirLocked reports that another running Server already holds an
-// exclusive lock on StateDir.
-var ErrStateDirLocked = errors.New("meshnet: state directory is already locked by another Server")
+// ErrStateDirLocked reports that another running Server or agentmesh-node
+// instance already holds the bbolt file lock on StateDir.
+var ErrStateDirLocked = errors.New("meshnet: state directory is already locked by another instance")
// WriteStateDir writes a member's libp2p private key and MemberCredential to
// dir with owner-only permissions (0700 directory, 0600 files).
@@ -216,27 +215,31 @@ func (s *Server) newAgentMeshNode(store *node.Store, privKey crypto.PrivKey) (*n
return n, nil
}
-func (s *Server) openStore() (*node.Store, *os.File, error) {
- store := node.NewMemoryStore()
-
+func (s *Server) openStore() (*node.Store, string, error) {
dir := strings.TrimSpace(s.StateDir)
if dir == "" {
- return store, nil, nil
+ tmpDir, err := os.MkdirTemp("", "meshnet-store-*")
+ if err != nil {
+ return nil, "", fmt.Errorf("create temporary store directory: %w", err)
+ }
+ store, err := node.NewStore(tmpDir)
+ if err != nil {
+ _ = os.RemoveAll(tmpDir)
+ return nil, "", err
+ }
+ return store, tmpDir, nil
}
- if err := os.MkdirAll(dir, 0o700); err != nil {
- _ = store.Close()
- return nil, nil, fmt.Errorf("create state directory %s: %w", dir, err)
- }
- lockFile, err := lockStateDir(dir)
+ store, err := node.NewStore(dir)
if err != nil {
- _ = store.Close()
- return nil, nil, err
+ if errors.Is(err, node.ErrStoreLocked) {
+ return nil, "", fmt.Errorf("%w: %w", ErrStateDirLocked, err)
+ }
+ return nil, "", err
}
- cleanupOnErr := func(e error) (*node.Store, *os.File, error) {
+ cleanupOnErr := func(e error) (*node.Store, string, error) {
_ = store.Close()
- unlockStateDir(lockFile)
- return nil, nil, e
+ return nil, "", e
}
keyPath := filepath.Join(dir, StateDirIdentityFile)
@@ -264,25 +267,29 @@ func (s *Server) openStore() (*node.Store, *os.File, error) {
return cleanupOnErr(rErr)
}
- var persistMu sync.Mutex
- store.SetOnChange(func() error {
- persistMu.Lock()
- defer persistMu.Unlock()
- kb, lErr := store.LoadKey()
- if lErr != nil {
- return lErr
- }
- if len(kb) == 0 {
- return nil
- }
- c, cErr := store.Credential()
- if cErr != nil {
- return cErr
- }
- return WriteStateDir(dir, kb, c)
- })
+ return store, "", nil
+}
- return store, lockFile, nil
+func (s *Server) syncStateDir(store *node.Store) error {
+ dir := strings.TrimSpace(s.StateDir)
+ if dir == "" || store == nil {
+ return nil
+ }
+ s.stateDirSyncMu.Lock()
+ defer s.stateDirSyncMu.Unlock()
+
+ kb, err := store.LoadKey()
+ if err != nil {
+ return err
+ }
+ if len(kb) == 0 {
+ return nil
+ }
+ cred, err := store.Credential()
+ if err != nil {
+ return err
+ }
+ return WriteStateDir(dir, kb, cred)
}
func (s *Server) verifyOwnBiscuit(raw []byte) bool {
@@ -327,12 +334,12 @@ func (s *Server) identityPredatesRotation() bool {
}
func (s *Server) ensureCredential(ctx context.Context) (retErr error) {
- store, lockFile, err := s.openStore()
+ store, tmpDir, err := s.openStore()
if err != nil {
return err
}
s.store = store
- s.stateDirLock = lockFile
+ s.storeTmpDir = tmpDir
defer func() {
if retErr != nil {
_ = s.closeStoreLocked()
@@ -405,6 +412,9 @@ func (s *Server) ensureCredential(ctx context.Context) (retErr error) {
s.node = n
finalize := func() error {
+ if err := s.syncStateDir(store); err != nil {
+ return err
+ }
rebuilt, rErr := s.newAgentMeshNode(store, privKey)
if rErr != nil {
return rErr
@@ -441,18 +451,25 @@ func (s *Server) ensureCredential(ctx context.Context) (retErr error) {
}
// Try OIDC refresh_token recovery if stored in credential.json.
- // ReEnrollWithRefreshToken persists any rotated refresh_token immediately
- // before calling POST /register so one-time refresh tokens are never lost.
+ // ReEnrollWithRefreshToken saves any rotated refresh_token to store before
+ // calling POST /register; syncStateDir flushes it immediately so one-time
+ // refresh tokens survive even if POST /register fails.
if cred.GetOidcSession().GetRefreshToken() != "" {
- if oidcErr := n.ReEnrollWithRefreshToken(ctx); oidcErr == nil {
- s.log().Info("re-enrolled via saved OIDC session", "peer_id", s.peerID.String())
- return finalize()
- } else {
- s.log().Warn("re-enrollment via saved OIDC session failed on startup",
+ oidcErr := n.ReEnrollWithRefreshToken(ctx)
+ if syncErr := s.syncStateDir(store); syncErr != nil {
+ s.log().Warn("failed to sync OIDC session state to state directory",
"peer_id", s.peerID.String(),
- "error", oidcErr,
+ "error", syncErr,
)
}
+ if oidcErr == nil {
+ s.log().Info("re-enrolled via saved OIDC session", "peer_id", s.peerID.String())
+ return finalize()
+ }
+ s.log().Warn("re-enrollment via saved OIDC session failed on startup",
+ "peer_id", s.peerID.String(),
+ "error", oidcErr,
+ )
}
// Try operator-supplied join token / TokenSource re-enrollment.
@@ -550,6 +567,12 @@ func (s *Server) startRenewalLoop(ctx context.Context, n *node.AgentMeshNode, st
case <-timer.C:
refreshErr := n.RefreshEnrollment(ctx)
if refreshErr == nil {
+ if syncErr := s.syncStateDir(store); syncErr != nil {
+ s.log().Warn("failed to sync renewed credential to state directory",
+ "peer_id", peerID.String(),
+ "error", syncErr,
+ )
+ }
failureBackoff = 0
s.handleCredentialRenewed(peerID)
continue
@@ -557,6 +580,12 @@ func (s *Server) startRenewalLoop(ctx context.Context, n *node.AgentMeshNode, st
renewErr := s.fallbackReEnroll(ctx, n, store, cpURL)
if renewErr == nil {
+ if syncErr := s.syncStateDir(store); syncErr != nil {
+ s.log().Warn("failed to sync re-enrolled credential to state directory",
+ "peer_id", peerID.String(),
+ "error", syncErr,
+ )
+ }
failureBackoff = 0
s.handleCredentialRenewed(peerID)
continue
@@ -596,11 +625,14 @@ func (s *Server) fallbackReEnroll(ctx context.Context, n *node.AgentMeshNode, st
}
if store != nil {
if rt, _ := store.LoadRefreshToken(); rt != "" {
- if oidcErr := n.ReEnrollWithRefreshToken(ctx); oidcErr == nil {
+ oidcErr := n.ReEnrollWithRefreshToken(ctx)
+ if syncErr := s.syncStateDir(store); syncErr != nil {
+ s.log().Warn("failed to sync OIDC session state to state directory", "error", syncErr)
+ }
+ if oidcErr == nil {
return nil
- } else {
- errs = append(errs, oidcErr)
}
+ errs = append(errs, oidcErr)
}
}
var bootToken string
@@ -632,15 +664,15 @@ func (s *Server) fallbackReEnroll(ctx context.Context, n *node.AgentMeshNode, st
// configured), verifies the returned credential, persists it to StateDir, and
// re-authenticates attached router sessions.
func (s *Server) Refresh(ctx context.Context) error {
- n, _, _, _ := s.stateSnapshot()
+ n, store, _, _ := s.stateSnapshot()
if n == nil {
return errors.New("meshnet: Server is not running (call Up first)")
}
- err := n.RefreshEnrollment(ctx)
- if err == nil {
- s.credentialLapsed.Store(false)
+ if err := n.RefreshEnrollment(ctx); err != nil {
+ return err
}
- return err
+ s.credentialLapsed.Store(false)
+ return s.syncStateDir(store)
}
func credentialExpireTimeFromStore(store *node.Store) time.Time {
diff --git a/sdk/meshnet/statedir_lock_unix.go b/sdk/meshnet/statedir_lock_unix.go
deleted file mode 100644
index b582b10e..00000000
--- a/sdk/meshnet/statedir_lock_unix.go
+++ /dev/null
@@ -1,47 +0,0 @@
-// Copyright 2026 Google LLC
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-//go:build unix
-
-package meshnet
-
-import (
- "errors"
- "fmt"
- "os"
- "syscall"
-)
-
-func lockStateDir(dir string) (*os.File, error) {
- f, err := os.Open(dir)
- if err != nil {
- return nil, err
- }
- if err := syscall.Flock(int(f.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil {
- _ = f.Close()
- if errors.Is(err, syscall.EWOULDBLOCK) || errors.Is(err, syscall.EAGAIN) {
- return nil, fmt.Errorf("%w: %s", ErrStateDirLocked, dir)
- }
- return nil, fmt.Errorf("lock state directory %s: %w", dir, err)
- }
- return f, nil
-}
-
-func unlockStateDir(f *os.File) {
- if f == nil {
- return
- }
- _ = syscall.Flock(int(f.Fd()), syscall.LOCK_UN)
- _ = f.Close()
-}
diff --git a/sdk/meshnet/statedir_lock_windows.go b/sdk/meshnet/statedir_lock_windows.go
deleted file mode 100644
index c37b960d..00000000
--- a/sdk/meshnet/statedir_lock_windows.go
+++ /dev/null
@@ -1,53 +0,0 @@
-// Copyright 2026 Google LLC
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-//go:build windows
-
-package meshnet
-
-import (
- "errors"
- "fmt"
- "os"
- "path/filepath"
-
- "golang.org/x/sys/windows"
-)
-
-func lockStateDir(dir string) (*os.File, error) {
- lockPath := filepath.Join(dir, ".lock")
- f, err := os.OpenFile(lockPath, os.O_CREATE|os.O_RDWR, 0o600)
- if err != nil {
- return nil, err
- }
- var ol windows.Overlapped
- flags := uint32(windows.LOCKFILE_EXCLUSIVE_LOCK | windows.LOCKFILE_FAIL_IMMEDIATELY)
- if err := windows.LockFileEx(windows.Handle(f.Fd()), flags, 0, 1, 0, &ol); err != nil {
- _ = f.Close()
- if errors.Is(err, windows.ERROR_LOCK_VIOLATION) {
- return nil, fmt.Errorf("%w: %s", ErrStateDirLocked, dir)
- }
- return nil, fmt.Errorf("lock state directory %s: %w", dir, err)
- }
- return f, nil
-}
-
-func unlockStateDir(f *os.File) {
- if f == nil {
- return
- }
- var ol windows.Overlapped
- _ = windows.UnlockFileEx(windows.Handle(f.Fd()), 0, 1, 0, &ol)
- _ = f.Close()
-}
diff --git a/sdk/meshnet/statedir_test.go b/sdk/meshnet/statedir_test.go
index fb351d90..a6448c78 100644
--- a/sdk/meshnet/statedir_test.go
+++ b/sdk/meshnet/statedir_test.go
@@ -387,16 +387,34 @@ func TestEnrollmentAndRenewal(t *testing.T) {
})
}
-func TestStateDirLockAndEphemeralMemoryIdentity(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
+func TestStateDirLockAndEphemeralTempCleanup(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
env := newStartedTestServer(t, ctx)
- t.Run("ephemeral_empty_StateDir_writes_zero_files_to_TMPDIR", func(t *testing.T) {
+ t.Run("ephemeral_empty_StateDir_uses_0700_temp_dir_and_cleans_up", func(t *testing.T) {
emptyTmp := t.TempDir()
t.Setenv("TMPDIR", emptyTmp)
+ failed := &Server{
+ ControlPlaneURL: env.cp.srv.URL,
+ BootstrapToken: "wrong-token",
+ ListenAddrs: []string{"/ip4/127.0.0.1/tcp/0"},
+ AllowInsecure: true,
+ }
+ if _, err := failed.Up(ctx); err == nil {
+ _ = failed.Close()
+ t.Fatal("failed.Up succeeded with wrong token, want error")
+ }
+ failedEntries, err := os.ReadDir(emptyTmp)
+ if err != nil {
+ t.Fatalf("ReadDir(TMPDIR) after failed Up: %v", err)
+ }
+ if len(failedEntries) != 0 {
+ t.Fatalf("failed ephemeral Up left %d entries in TMPDIR (%v), want 0", len(failedEntries), failedEntries)
+ }
+
ephemeral := &Server{
ControlPlaneURL: env.cp.srv.URL,
BootstrapToken: "boot-secret",
@@ -415,8 +433,15 @@ func TestStateDirLockAndEphemeralMemoryIdentity(t *testing.T) {
if err != nil {
t.Fatalf("ReadDir(TMPDIR) while running: %v", err)
}
- if len(duringEntries) != 0 {
- t.Fatalf("ephemeral Server wrote %d entries to TMPDIR while running (%v), want 0", len(duringEntries), duringEntries)
+ if len(duringEntries) != 1 {
+ t.Fatalf("TMPDIR entries while running = %d, want 1", len(duringEntries))
+ }
+ info, err := duringEntries[0].Info()
+ if err != nil {
+ t.Fatalf("entry.Info: %v", err)
+ }
+ if perm := info.Mode().Perm(); perm != 0o700 {
+ t.Fatalf("ephemeral temp dir permissions = %04o, want 0700", perm)
}
if err := ephemeral.Close(); err != nil {
@@ -453,9 +478,9 @@ func TestStateDirLockAndEphemeralMemoryIdentity(t *testing.T) {
ListenAddrs: []string{"/ip4/127.0.0.1/tcp/0"},
AllowInsecure: true,
}
- if _, err := s2.Up(ctx); !errors.Is(err, ErrStateDirLocked) {
+ if _, err := s2.Up(ctx); !errors.Is(err, ErrStateDirLocked) || !errors.Is(err, node.ErrStoreLocked) {
_ = s2.Close()
- t.Fatalf("s2.Up on locked StateDir = %v, want ErrStateDirLocked", err)
+ t.Fatalf("s2.Up on locked StateDir = %v, want ErrStateDirLocked + node.ErrStoreLocked", err)
}
if err := s1.Close(); err != nil {
diff --git a/site/content/docs/guides/go-sdk.md b/site/content/docs/guides/go-sdk.md
index 96651ec5..246cd8ba 100644
--- a/site/content/docs/guides/go-sdk.md
+++ b/site/content/docs/guides/go-sdk.md
@@ -451,10 +451,11 @@ Code, Gemini CLI, or a sandboxed coding tool), mount `srv.MCPHandler()` and
`Server.StateDir` persists `identity.key` and `credential.json` with owner-only
permissions (`0700` directory, `0600` files) using the exact protobuf/protojson
-layout shared by `agentmesh-node`, `sdk/js`, and `sdk/python`, guarded by an
-exclusive non-blocking directory lock (`meshnet.ErrStateDirLocked`) so two
-`Server` instances cannot clobber the same directory concurrently. When
-`StateDir` is empty, `Server` runs purely in memory with zero files on disk.
+layout shared by `agentmesh-node`, `sdk/js`, and `sdk/python`, alongside the
+`bbolt` store lock (`agent.db`, returning `meshnet.ErrStateDirLocked`) so
+concurrent `Server` or `agentmesh-node` instances cannot clobber the same
+directory. When `StateDir` is empty, `Up` creates an ephemeral identity in a
+temporary `0700` directory that is removed on `Close` (or if `Up` fails).
An operator can enroll interactively once with `agentmesh-node` and pass the
exported state directory directly to a Go binary built on `sdk/meshnet` (with no