Skip to content

The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu

fix: migrate existing forwarder log paths on upgrade (#916)

Rudy Mizrahi Celekli373cf0620d68
diff --git a/cli/beacon/internal/endpoint/asymptote/connect.go b/cli/beacon/internal/endpoint/asymptote/connect.go
index 2cd58253..a2dc1704 100644
--- a/cli/beacon/internal/endpoint/asymptote/connect.go
+++ b/cli/beacon/internal/endpoint/asymptote/connect.go
@@ -25,6 +25,7 @@ type Forwarder interface {
Label() string
UnitPath() (string, error)
WriteUnit(vectorBin, configPath string) (string, error)
+ UnitCurrent(vectorBin, configPath string) (bool, error)
Load() error
Unload() error
RemoveUnits()
diff --git a/cli/beacon/internal/endpoint/asymptote/connect_test.go b/cli/beacon/internal/endpoint/asymptote/connect_test.go
index d0194268..1abd12c5 100644
--- a/cli/beacon/internal/endpoint/asymptote/connect_test.go
+++ b/cli/beacon/internal/endpoint/asymptote/connect_test.go
@@ -36,6 +36,9 @@ func (f *fakeForwarder) Supported() bool { return f.supported }
func (f *fakeForwarder) UnsupportedReason() string { return "fake forwarder unsupported" }
func (f *fakeForwarder) Label() string { return "fake.forwarder" }
func (f *fakeForwarder) UnitPath() (string, error) { return f.unitPath, nil }
+func (f *fakeForwarder) UnitCurrent(string, string) (bool, error) {
+ return true, nil
+}
func (f *fakeForwarder) WriteUnit(vectorBin, configPath string) (string, error) {
f.written = append(f.written, vectorBin+" --config "+configPath)
if f.writeErr != nil {
diff --git a/cli/beacon/internal/endpoint/asymptote/reconcile.go b/cli/beacon/internal/endpoint/asymptote/reconcile.go
index 8b486cd6..c1b80f3c 100644
--- a/cli/beacon/internal/endpoint/asymptote/reconcile.go
+++ b/cli/beacon/internal/endpoint/asymptote/reconcile.go
@@ -28,14 +28,16 @@ type ReconcileResult struct {
// Skipped names why nothing was checked: the endpoint is not connected, the forwarder is not
// installed, or the service manager cannot run it here.
Skipped string `json:"skipped,omitempty"`
- // Unchanged is true when the rendered config already matched the one on disk.
+ // Unchanged is true when both the config and the service log paths are current.
Unchanged bool `json:"unchanged,omitempty"`
// Updated is true when a validated config replaced the old one.
Updated bool `json:"updated,omitempty"`
- // Restarted is true when the forwarder was restarted to load the updated config.
+ // UnitUpdated is true when a stale service unit was rewritten.
+ UnitUpdated bool `json:"unit_updated,omitempty"`
+ // Restarted is true when the forwarder was restarted to load an updated config or unit.
Restarted bool `json:"restarted,omitempty"`
- // Warning says why a connected forwarder kept its old config, or why an updated one was not
- // restarted. The old config stays in place and keeps running in either case.
+ // Warning explains a refused check/update or a failed restart. Updated and UnitUpdated
+ // report any changes already written; a warning does not imply those were rolled back.
Warning string `json:"warning,omitempty"`
}
@@ -44,6 +46,15 @@ func (r ReconcileResult) Message() string {
switch {
case r.Warning != "":
return "Beacon Cloud forwarder: " + r.Warning
+ case r.UnitUpdated:
+ what := "service unit"
+ if r.Updated {
+ what = "config and service unit"
+ }
+ if r.Restarted {
+ return "Beacon Cloud forwarder " + what + " updated and the forwarder restarted."
+ }
+ return "Beacon Cloud forwarder " + what + " updated; it takes effect when the forwarder next starts."
case r.Updated && r.Restarted:
return "Beacon Cloud forwarder config updated and the forwarder restarted (" + r.VectorConfig + ")."
case r.Updated:
@@ -66,7 +77,7 @@ func (r ReconcileResult) Message() string {
// endpoint's runtime log. It never enrolls, never reads the device key or rewrites the secrets
// file, makes no network request, and leaves the Vector data directory and its checkpoints
// alone. The new config is validated by Vector exactly as connect validates it before it
-// replaces the old one, and the forwarder is restarted only when the content changed. A config
+// replaces the old one, and the forwarder is restarted only when the config or launchd log paths changed. A config
// Vector rejects is reported as a warning and the running forwarder keeps the old one.
//
// The returned error is reserved for a failure that leaves something to fix by hand (the
@@ -162,24 +173,41 @@ func ReconcileForwarder(opts ReconcileOptions) (ReconcileResult, error) {
result.Warning = fmt.Sprintf("kept the existing config: %v", err)
return result, nil
}
- if bytes.Equal(current, []byte(rendered)) {
- result.Unchanged = true
- return result, nil
- }
- // Vector refuses a data_dir that does not exist. Creating a missing one loses nothing: its
- // checkpoints were already gone. An existing one is never cleared here.
- if err := os.MkdirAll(dataDir, 0o700); err != nil {
- result.Warning = fmt.Sprintf("kept the existing config: %v", err)
+ configCurrent := bytes.Equal(current, []byte(rendered))
+ unitCurrent, err := manager.UnitCurrent(vector.Path, configPath)
+ if err != nil {
+ result.Warning = fmt.Sprintf("kept the existing config and service unit: could not read the forwarder service unit: %v", err)
return result, nil
}
- if err := preValidateVectorConfig(vector.Path, configPath, []byte(rendered)); err != nil {
- result.Warning = fmt.Sprintf("kept the existing config because Vector rejected the new one: %v", err)
+ if configCurrent && unitCurrent {
+ result.Unchanged = true
return result, nil
}
- if err := atomicfile.Write(configPath, []byte(rendered), 0o644); err != nil {
- return result, fmt.Errorf("could not write the validated forwarder config %s: %w", configPath, err)
+ if !configCurrent {
+ // Creating a missing data directory loses no existing checkpoints.
+ if err := os.MkdirAll(dataDir, 0o700); err != nil {
+ result.Warning = fmt.Sprintf("kept the existing config: %v", err)
+ return result, nil
+ }
+ if err := preValidateVectorConfig(vector.Path, configPath, []byte(rendered)); err != nil {
+ result.Warning = fmt.Sprintf("kept the existing config because Vector rejected the new one: %v", err)
+ return result, nil
+ }
+ if err := atomicfile.Write(configPath, []byte(rendered), 0o644); err != nil {
+ return result, fmt.Errorf("could not write the validated forwarder config %s: %w", configPath, err)
+ }
+ result.Updated = true
+ }
+ if !unitCurrent {
+ if _, err := manager.WriteUnit(vector.Path, configPath); err != nil {
+ result.Warning = fmt.Sprintf("could not update the forwarder service unit: %v", err)
+ if result.Updated {
+ result.Warning = fmt.Sprintf("config updated (%s) but the forwarder service unit could not be updated: %v", configPath, err)
+ }
+ return result, nil
+ }
+ result.UnitUpdated = true
}
- result.Updated = true
// Restart only a forwarder that is running, so reconcile never starts one an operator stopped.
// Loaded is not enough: on systemd an enabled unit that was stopped still reports Loaded. A
// forwarder that is not running reads the new config when it is next started.
@@ -187,7 +215,7 @@ func ReconcileForwarder(opts ReconcileOptions) (ReconcileResult, error) {
return result, nil
}
if err := manager.Load(); err != nil {
- result.Warning = fmt.Sprintf("config updated (%s) but the forwarder could not be restarted: %v", configPath, err)
+ result.Warning = fmt.Sprintf("forwarder updated (%s) but it could not be restarted: %v", configPath, err)
return result, nil
}
result.Restarted = true
diff --git a/cli/beacon/internal/endpoint/asymptote/reconcile_unit_test.go b/cli/beacon/internal/endpoint/asymptote/reconcile_unit_test.go
new file mode 100644
index 00000000..f8888244
--- /dev/null
+++ b/cli/beacon/internal/endpoint/asymptote/reconcile_unit_test.go
@@ -0,0 +1,187 @@
+package asymptote
+
+import (
+ "bytes"
+ "errors"
+ "os"
+ "path/filepath"
+ "runtime"
+ "strings"
+ "testing"
+
+ "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/service"
+)
+
+// Keep service operations fake, while checking and writing the actual production
+// launchd plist in an isolated HOME.
+type unitReconcileForwarder struct {
+ *fakeForwarder
+ manager service.ForwarderManager
+ unitWrites int
+}
+
+func (f *unitReconcileForwarder) UnitCurrent(vector, config string) (bool, error) {
+ return f.manager.UnitCurrent(vector, config)
+}
+func (f *unitReconcileForwarder) WriteUnit(vector, config string) (string, error) {
+ f.unitWrites++
+ if f.writeErr != nil {
+ return "", f.writeErr
+ }
+ return f.manager.WriteUnit(vector, config)
+}
+
+func legacyUnitFixture(t *testing.T) (*reconcileFixture, *unitReconcileForwarder) {
+ t.Helper()
+ if runtime.GOOS != "darwin" {
+ t.Skip("launchd unit writes require macOS")
+ }
+ f := newReconcileFixture(t, "standard")
+ manager := service.ForwarderManager{UserMode: true, Kind: service.KindLaunchd}
+ path, err := manager.UnitPath()
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
+ t.Fatal(err)
+ }
+ legacy := `<?xml version="1.0"?><plist version="1.0"><dict>
+ <key>StandardOutPath</key><string>/tmp/com.beacon.endpoint.asymptote-forwarder.out</string>
+ <key>StandardErrorPath</key><string>/tmp/com.beacon.endpoint.asymptote-forwarder.err</string>
+ </dict></plist>`
+ if err := os.WriteFile(path, []byte(legacy), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ f.fwd.unitPath = path
+ return f, &unitReconcileForwarder{fakeForwarder: f.fwd, manager: manager}
+}
+
+func TestReconcileMigratesLegacyUnitEvenWhenConfigCurrent(t *testing.T) {
+ f, manager := legacyUnitFixture(t)
+ before, _ := os.ReadFile(VectorConfigPath(true))
+ beforeInfo, _ := os.Stat(VectorConfigPath(true))
+ loads := f.fwd.loads
+ result, err := ReconcileForwarder(ReconcileOptions{UserMode: true, Forwarder: manager})
+ if err != nil {
+ t.Fatal(err)
+ }
+ if result.Unchanged || !result.Restarted || result.Warning != "" || manager.unitWrites != 1 || f.fwd.loads != loads+1 {
+ t.Fatalf("result=%+v unitWrites=%d loads=%d->%d; want one unit rewrite and one restart", result, manager.unitWrites, loads, f.fwd.loads)
+ }
+ after, _ := os.ReadFile(VectorConfigPath(true))
+ afterInfo, _ := os.Stat(VectorConfigPath(true))
+ if !bytes.Equal(before, after) || !beforeInfo.ModTime().Equal(afterInfo.ModTime()) {
+ t.Fatal("unit-only migration rewrote config")
+ }
+ unit, _ := os.ReadFile(f.fwd.unitPath)
+ if bytes.Contains(unit, []byte("/tmp/com.beacon.endpoint.asymptote-forwarder")) || !bytes.Contains(unit, []byte("Library/Logs/Beacon/")) {
+ t.Fatalf("unit was not migrated: %s", unit)
+ }
+ f.assertCredentialsAndCheckpointsUntouched(t)
+ result, err = ReconcileForwarder(ReconcileOptions{UserMode: true, Forwarder: manager})
+ if err != nil || !result.Unchanged || manager.unitWrites != 1 || f.fwd.loads != loads+1 {
+ t.Fatalf("repeat result=%+v err=%v writes=%d loads=%d", result, err, manager.unitWrites, f.fwd.loads)
+ }
+}
+
+func TestReconcileUnitMigrationKeepsStoppedServicesStopped(t *testing.T) {
+ for _, loaded := range []bool{false, true} {
+ t.Run(fmtBool(loaded), func(t *testing.T) {
+ f, manager := legacyUnitFixture(t)
+ f.fwd.stopped = true
+ if !loaded {
+ f.fwd.loads = 0
+ }
+ loads := f.fwd.loads
+ result, err := ReconcileForwarder(ReconcileOptions{UserMode: true, Forwarder: manager})
+ if err != nil || !result.UnitUpdated || result.Updated || result.Restarted || result.Warning != "" || manager.unitWrites != 1 || f.fwd.loads != loads {
+ t.Fatalf("result=%+v err=%v writes=%d loads=%d->%d", result, err, manager.unitWrites, loads, f.fwd.loads)
+ }
+ f.assertCredentialsAndCheckpointsUntouched(t)
+ })
+ }
+}
+func fmtBool(loaded bool) string {
+ if loaded {
+ return "loaded"
+ }
+ return "unloaded"
+}
+
+func TestReconcileUnitMigrationReportsWriteFailure(t *testing.T) {
+ f, manager := legacyUnitFixture(t)
+ before, _ := os.ReadFile(f.fwd.unitPath)
+ config, _ := os.ReadFile(VectorConfigPath(true))
+ loads := f.fwd.loads
+ f.fwd.writeErr = errors.New("unit write refused")
+ result, err := ReconcileForwarder(ReconcileOptions{UserMode: true, Forwarder: manager})
+ if err != nil || result.UnitUpdated || result.Updated || result.Restarted || !strings.Contains(result.Warning, "unit write refused") || f.fwd.loads != loads {
+ t.Fatalf("result=%+v err=%v loads=%d->%d", result, err, loads, f.fwd.loads)
+ }
+ after, _ := os.ReadFile(f.fwd.unitPath)
+ afterConfig, _ := os.ReadFile(VectorConfigPath(true))
+ if !bytes.Equal(before, after) || !bytes.Equal(config, afterConfig) {
+ t.Fatal("failed unit migration changed a file")
+ }
+ f.assertCredentialsAndCheckpointsUntouched(t)
+}
+
+func TestReconcileUnitMigrationReportsReloadFailure(t *testing.T) {
+ f, manager := legacyUnitFixture(t)
+ f.fwd.loadErr = errors.New("reload refused")
+ result, err := ReconcileForwarder(ReconcileOptions{UserMode: true, Forwarder: manager})
+ if err != nil || !result.UnitUpdated || result.Restarted || !strings.Contains(result.Warning, "reload refused") {
+ t.Fatalf("result=%+v err=%v", result, err)
+ }
+ f.assertCredentialsAndCheckpointsUntouched(t)
+}
+
+// A unit as WriteUnit writes it is current: reconcile neither rewrites nor restarts it, so an
+// upgrade restarts the forwarder only when the unit Beacon writes has changed.
+func TestReconcileLeavesACurrentUnitAlone(t *testing.T) {
+ f, manager := legacyUnitFixture(t)
+ if _, err := manager.manager.WriteUnit(f.vector, VectorConfigPath(true)); err != nil {
+ t.Fatal(err)
+ }
+ unit, err := os.ReadFile(f.fwd.unitPath)
+ if err != nil {
+ t.Fatal(err)
+ }
+ loads := f.fwd.loads
+ result, err := ReconcileForwarder(ReconcileOptions{UserMode: true, Forwarder: manager})
+ if err != nil || !result.Unchanged || result.UnitUpdated || result.Restarted || manager.unitWrites != 0 || f.fwd.loads != loads {
+ t.Fatalf("result=%+v err=%v writes=%d loads=%d", result, err, manager.unitWrites, f.fwd.loads)
+ }
+ if after, _ := os.ReadFile(f.fwd.unitPath); !bytes.Equal(after, unit) {
+ t.Fatal("current unit was rewritten")
+ }
+ f.assertCredentialsAndCheckpointsUntouched(t)
+}
+
+func TestReconcileUnitMigrationReportsConfigAlreadyWrittenBeforeUnitFailure(t *testing.T) {
+ f, manager := legacyUnitFixture(t)
+ configPath := VectorConfigPath(true)
+ current, err := os.ReadFile(configPath)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(configPath, append(current, []byte("\n# older pack\n")...), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ unit, err := os.ReadFile(f.fwd.unitPath)
+ if err != nil {
+ t.Fatal(err)
+ }
+ loads := f.fwd.loads
+ f.fwd.writeErr = errors.New("unit write refused")
+ result, err := ReconcileForwarder(ReconcileOptions{UserMode: true, Forwarder: manager})
+ if err != nil || !result.Updated || result.UnitUpdated || result.Restarted || !strings.Contains(result.Warning, "unit write refused") || !strings.Contains(result.Warning, "config updated") || f.fwd.loads != loads {
+ t.Fatalf("result=%+v err=%v loads=%d->%d", result, err, loads, f.fwd.loads)
+ }
+ after, _ := os.ReadFile(configPath)
+ afterUnit, _ := os.ReadFile(f.fwd.unitPath)
+ if !bytes.Equal(after, current) || !bytes.Equal(afterUnit, unit) {
+ t.Fatal("partial-update result does not match files")
+ }
+ f.assertCredentialsAndCheckpointsUntouched(t)
+}
diff --git a/cli/beacon/internal/endpoint/service/forwarder.go b/cli/beacon/internal/endpoint/service/forwarder.go
index b0d7e280..6623a874 100644
--- a/cli/beacon/internal/endpoint/service/forwarder.go
+++ b/cli/beacon/internal/endpoint/service/forwarder.go
@@ -112,18 +112,14 @@ func (m ForwarderManager) WriteUnit(vectorBin, configPath string) (string, error
if err := ensureDir(path); err != nil {
return "", err
}
- var content string
- if m.resolvedKind() == KindSystemd {
- content = forwarderUnitFile(vectorBin, configPath, m.UserMode)
- } else {
+ if m.resolvedKind() == KindLaunchd {
if err := ensureLaunchdLogDir(m.UserMode, ForwarderLabel); err != nil {
return "", err
}
- stdout, stderr, err := launchdLogPaths(m.UserMode, ForwarderLabel)
- if err != nil {
- return "", err
- }
- content = forwarderPlist(ForwarderLabel, vectorBin, configPath, stdout, stderr)
+ }
+ content, err := m.unitContent(vectorBin, configPath)
+ if err != nil {
+ return "", err
}
if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
return "", err
@@ -136,6 +132,36 @@ func (m ForwarderManager) WriteUnit(vectorBin, configPath string) (string, error
return path, nil
}
+// UnitCurrent reports whether the installed unit is the one WriteUnit would write now. Reconcile
+// rewrites one that is not, so a unit an earlier release wrote, such as one logging to /tmp, is
+// brought up to date by an upgrade rather than only by a fresh connect.
+func (m ForwarderManager) UnitCurrent(vectorBin, configPath string) (bool, error) {
+ path, err := m.UnitPath()
+ if err != nil {
+ return false, err
+ }
+ installed, err := os.ReadFile(path)
+ if err != nil {
+ return false, err
+ }
+ want, err := m.unitContent(vectorBin, configPath)
+ if err != nil {
+ return false, err
+ }
+ return string(installed) == want, nil
+}
+
+func (m ForwarderManager) unitContent(vectorBin, configPath string) (string, error) {
+ if m.resolvedKind() == KindSystemd {
+ return forwarderUnitFile(vectorBin, configPath, m.UserMode), nil
+ }
+ stdout, stderr, err := launchdLogPaths(m.UserMode, ForwarderLabel)
+ if err != nil {
+ return "", err
+ }
+ return forwarderPlist(ForwarderLabel, vectorBin, configPath, stdout, stderr), nil
+}
+
// Load registers and starts the forwarder. An already-loaded launchd job is re-bootstrapped so a
// rewritten config takes effect.
func (m ForwarderManager) Load() error {