The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu
Go agent-ai-cliagent-harnessesagent-securitybrowser-extensionciclaude-codecloudcodexcursordetection-engineeringjevknowledge-basememorymobile-device-managementobservabilitysecuritysecurity-information-and-event-managementsecurity-toolstelemetrytraces
sentinel: keep the forwarder's Vector and service across a rejected reconcile
shukieshah2b81b59d071a
diff --git a/cli/beacon/internal/endpoint/sentinel/forwarder.go b/cli/beacon/internal/endpoint/sentinel/forwarder.goindex 1e287130..a3dd1c57 100644--- a/cli/beacon/internal/endpoint/sentinel/forwarder.go+++ b/cli/beacon/internal/endpoint/sentinel/forwarder.go@@ -79,7 +79,10 @@ type Settings struct {AuthorityHost string `json:"authority_host,omitempty"`ReadFrom string `json:"read_from"`LogPath string `json:"log_path"`- InstalledAt string `json:"installed_at"`+ // VectorBin is the Vector binary install was told to run, so reconcile keeps running it; empty+ // means the one Beacon's package installed, found again each time.+ VectorBin string `json:"vector_bin,omitempty"`+ InstalledAt string `json:"installed_at"`}// InstallOptions is what `forwarder install` was given.@@ -154,6 +157,7 @@ func (o InstallOptions) settings() (Settings, error) {AuthorityHost: strings.TrimRight(strings.TrimSpace(o.AuthorityHost), "/"),ReadFrom: strings.TrimSpace(o.ReadFrom),LogPath: o.LogPath,+ VectorBin: strings.TrimSpace(o.VectorBin),}if err := httpsURL("--endpoint", s.Endpoint, true); err != nil {return s, err@@ -334,7 +338,7 @@ func Install(o InstallOptions) (InstallResult, error) {if !manager.Supported() {return InstallResult{}, errors.New(manager.UnsupportedReason())}- vector, err := findVector(o.VectorBin)+ vector, err := findVector(settings.VectorBin)if err != nil {return InstallResult{}, err}@@ -560,7 +564,7 @@ func Reconcile(opts ReconcileOptions) (ReconcileResult, error) {if !manager.Supported() {return ReconcileResult{Skipped: manager.UnsupportedReason()}, nil}- vector, err := findVector("")+ vector, err := findVector(settings.VectorBin)if err != nil {return ReconcileResult{Warning: err.Error()}, err}@@ -592,8 +596,9 @@ func Reconcile(opts ReconcileOptions) (ReconcileResult, error) {LogDir: logDir,HasCertificatePassword: hasPassword,})- current, _ := os.ReadFile(configPath)+ current, readErr := os.ReadFile(configPath)configChanged := string(current) != rendered+ var rejected stringif configChanged {candidate := configPath + ".new"if err := atomicfile.Write(candidate, []byte(rendered), 0o600); err != nil {@@ -601,9 +606,15 @@ func Reconcile(opts ReconcileOptions) (ReconcileResult, error) {}if err := validateVector(vector.Path, candidate); err != nil {_ = os.Remove(candidate)- return ReconcileResult{Warning: "kept the previous config: " + err.Error()}, nil- }- if err := os.Rename(candidate, configPath); err != nil {+ if readErr != nil {+ // Nothing to fall back to.+ return ReconcileResult{Warning: "Vector rejected the config, and there is no previous one: " + err.Error()}, nil+ }+ // The previous config still runs. The service is put back regardless, because the MSI+ // upgrade removed it before this install, and an old config running beats none.+ rejected = "kept the previous config, which Vector still runs: " + err.Error()+ configChanged = false+ } else if err := os.Rename(candidate, configPath); err != nil {return ReconcileResult{Warning: err.Error()}, err}}@@ -614,12 +625,12 @@ func Reconcile(opts ReconcileOptions) (ReconcileResult, error) {}}if !opts.Restart && !configChanged && unitCurrent && manager.Status().Running {- return ReconcileResult{Unchanged: true}, nil+ return ReconcileResult{Unchanged: rejected == "", Warning: rejected}, nil}if err := manager.Load(); err != nil {return ReconcileResult{Warning: err.Error()}, err}- return ReconcileResult{Restarted: true}, nil+ return ReconcileResult{Restarted: true, Warning: rejected}, nil}// Status is what `forwarder status` and `endpoint status` show.diff --git a/cli/beacon/internal/endpoint/sentinel/forwarder_e2e_test.go b/cli/beacon/internal/endpoint/sentinel/forwarder_e2e_test.goindex 35f19a56..15a10bba 100644--- a/cli/beacon/internal/endpoint/sentinel/forwarder_e2e_test.go+++ b/cli/beacon/internal/endpoint/sentinel/forwarder_e2e_test.go@@ -2,7 +2,10 @@ package sentinelimport ("bufio"+ "crypto/sha1"+ "encoding/hex""encoding/json"+ "encoding/pem""fmt""os""os/exec"@@ -111,23 +114,27 @@ func TestForwarderConfigDeliversThroughRealVector(t *testing.T) {// The restricted-mode switch replaces the log with a copy under the same name. Vector recognizes// a file by its first line, so it keeps reading the replaced one; install and repair, which make// the switch, then restart the forwarder (Reconcile with Restart), and it resumes in the copy.- data, err := os.ReadFile(settings.LogPath)- if err != nil {- t.Fatal(err)- }- if err := os.WriteFile(settings.LogPath+".copy", data, 0o600); err != nil {- t.Fatal(err)- }- if err := os.Rename(settings.LogPath+".copy", settings.LogPath); err != nil {- t.Fatal(err)- }- log.append(5) // between the switch and the restart- if !vector.stop() {- graceful = false+ // On Windows a plain rename cannot replace a file Vector holds open (Beacon's writer renames with+ // POSIX semantics instead), so there the MSI smoke covers it, with the real switch.+ if runtime.GOOS != "windows" {+ data, err := os.ReadFile(settings.LogPath)+ if err != nil {+ t.Fatal(err)+ }+ if err := os.WriteFile(settings.LogPath+".copy", data, 0o600); err != nil {+ t.Fatal(err)+ }+ if err := os.Rename(settings.LogPath+".copy", settings.LogPath); err != nil {+ t.Fatal(err)+ }+ log.append(5) // between the switch and the restart+ if !vector.stop() {+ graceful = false+ }+ vector = startVector(t, vectorBin, configPath, mock.URL, caFile)+ log.append(5)+ waitForRows(t, mock, log.ids, 60*time.Second)}- vector = startVector(t, vectorBin, configPath, mock.URL, caFile)- log.append(5)- waitForRows(t, mock, log.ids, 60*time.Second)time.Sleep(3 * time.Second) // let anything resent arrive before countingvector.stop()@@ -158,7 +165,7 @@ func TestForwarderConfigDeliversThroughRealVector(t *testing.T) {}if !t.Failed() {tokens, ingests, _ := mock.Counts()- t.Logf("%d events delivered exactly once across a rotation, a restart and a log copy; %d token requests, %d ingest requests", len(log.ids), tokens, ingests)+ t.Logf("%d events delivered exactly once across retried 403s, a rotation, a restart and (off Windows) a log copy; %d token requests, %d ingest requests", len(log.ids), tokens, ingests)}}@@ -279,12 +286,21 @@ func trustCA(t *testing.T, caFile string) {t.Fatalf("certutil -addstore Root: %v\n%s", err, out)}t.Cleanup(func() {- // The mock's CA has its own random name; delete exactly that certificate.- out, _ := exec.Command("powershell", "-NoProfile", "-Command",- `$c = New-Object System.Security.Cryptography.X509Certificates.X509Certificate2('`+caFile+`'); `+- `Get-ChildItem Cert:\LocalMachine\Root | Where-Object Thumbprint -eq $c.Thumbprint | Remove-Item`).CombinedOutput()- if len(out) > 0 {- t.Logf("removing the mock CA: %s", out)+ // The mock's CA has its own random name; delete exactly that certificate, by its SHA-1+ // thumbprint, which is how certutil names one.+ data, err := os.ReadFile(caFile)+ if err != nil {+ t.Logf("removing the mock CA: %v", err)+ return+ }+ block, _ := pem.Decode(data)+ if block == nil {+ t.Log("removing the mock CA: not PEM")+ return+ }+ thumbprint := sha1.Sum(block.Bytes)+ if out, err := exec.Command("certutil", "-delstore", "Root", hex.EncodeToString(thumbprint[:])).CombinedOutput(); err != nil {+ t.Logf("removing the mock CA: %v\n%s", err, out)}})}diff --git a/cli/beacon/internal/endpoint/sentinel/forwarder_test.go b/cli/beacon/internal/endpoint/sentinel/forwarder_test.goindex 023e69ac..7df64e11 100644--- a/cli/beacon/internal/endpoint/sentinel/forwarder_test.go+++ b/cli/beacon/internal/endpoint/sentinel/forwarder_test.go@@ -311,6 +311,18 @@ func TestForwarderReconcile(t *testing.T) {if readFile(t, configPath) != "# old\n" {t.Fatal("a rejected reconcile replaced the config")}+ // After an MSI upgrade removed the service, a rejected config still gets the service back, on+ // the previous config.+ if err := Remove(true); err != nil {+ t.Fatal(err)+ }+ f.svc.current = false+ if got, err := Reconcile(ReconcileOptions{Restart: true}); err != nil || got.Warning == "" || !got.Restarted || !f.svc.running {+ t.Fatalf("rejected after upgrade: %+v %v running=%t", got, err, f.svc.running)+ }+ if readFile(t, configPath) != "# old\n" {+ t.Fatal("a rejected reconcile replaced the config")+ }f.reject = nil// The MSI upgrade removes the service and keeps the state; the install that follows puts the// service back.@@ -388,3 +400,24 @@ func TestForwarderInstallRemovesAServiceThatNeverStarted(t *testing.T) {t.Fatalf("the settings must stay for the next install or reconcile: %v", err)}}++// The Vector install was told to run is the one reconcile keeps running.+func TestForwarderReconcileKeepsTheVectorInstallWasGiven(t *testing.T) {+ newForwarderFixture(t)+ var asked []string+ findVector = func(explicit string) (asymptote.VectorInfo, error) {+ asked = append(asked, explicit)+ return asymptote.VectorInfo{Path: "/custom/vector", Version: "0.56.0"}, nil+ }+ o := validInstall()+ o.VectorBin = "/custom/vector"+ if _, err := Install(o); err != nil {+ t.Fatal(err)+ }+ if _, err := Reconcile(ReconcileOptions{Restart: true}); err != nil {+ t.Fatal(err)+ }+ if len(asked) != 2 || asked[0] != "/custom/vector" || asked[1] != "/custom/vector" {+ t.Fatalf("findVector was asked for %q", asked)+ }+}diff --git a/packaging/windows/smoke-sentinel-forwarder.ps1 b/packaging/windows/smoke-sentinel-forwarder.ps1index 54593fe2..1fe9c78e 100644--- a/packaging/windows/smoke-sentinel-forwarder.ps1+++ b/packaging/windows/smoke-sentinel-forwarder.ps1@@ -140,12 +140,55 @@ switch ($Phase) {New-Item -ItemType Directory -Force -Path $work | Out-Nullif (-not $MockExe -or -not (Test-Path -LiteralPath $MockExe)) { throw "-MockExe must name the built sentinel-mock.exe" }- # A certificate credential, the one the docs lead with, exported the way OpenSSL 3 reads it.- $cert = New-SelfSignedCertificate -Subject 'CN=beacon-sentinel-smoke' -CertStoreLocation Cert:\LocalMachine\My `+ # A certificate credential, the one the docs lead with, made with `openssl pkcs12 -export`.+ $openssl = (Get-Command openssl -ErrorAction SilentlyContinue).Source+ if (-not $openssl) { $openssl = Join-Path $env:ProgramFiles 'Git\usr\bin\openssl.exe' }+ $key = Join-Path $work 'app-key.pem'+ $certPem = Join-Path $work 'app-cert.pem'+ & $openssl req -x509 -newkey rsa:2048 -nodes -keyout $key -out $certPem -days 2 -subj '/CN=beacon-sentinel-smoke' 2>&1 | Out-Null+ & $openssl pkcs12 -export -inkey $key -in $certPem -out $pfx -passout "pass:$pfxPassword" 2>&1 | Out-Null+ if ($LASTEXITCODE -ne 0 -or -not (Test-Path -LiteralPath $pfx)) { throw "openssl could not make the test PFX" }+ Remove-Item -LiteralPath $key++ # Which PFX encodings the bundled Vector can read, recorded for the docs: Windows'+ # Export-PfxCertificate with each algorithm, and OpenSSL's default.+ $cert = New-SelfSignedCertificate -Subject 'CN=beacon-sentinel-smoke-export' -CertStoreLocation Cert:\LocalMachine\My `-KeyAlgorithm RSA -KeyLength 2048 -KeyExportPolicy Exportable -NotAfter (Get-Date).AddDays(2)- $secure = ConvertTo-SecureString $pfxPassword -AsPlainText -Force- Export-PfxCertificate -Cert $cert -FilePath $pfx -Password $secure -CryptoAlgorithmOption AES256_SHA256 | Out-Null$cert.Thumbprint | Set-Content -LiteralPath (Join-Path $work 'cert-thumbprint')+ $secure = ConvertTo-SecureString $pfxPassword -AsPlainText -Force+ $variants = [ordered]@{ 'openssl pkcs12 -export' = $pfx }+ foreach ($algorithm in @('AES256_SHA256', 'TripleDES_SHA1')) {+ $path = Join-Path $work "export-$algorithm.pfx"+ Export-PfxCertificate -Cert $cert -FilePath $path -Password $secure -CryptoAlgorithmOption $algorithm | Out-Null+ $variants["Export-PfxCertificate -CryptoAlgorithmOption $algorithm"] = $path+ }+ $vectorExe = Join-Path $env:ProgramFiles 'Beacon\bin\vector.exe'+ foreach ($name in $variants.Keys) {+ $probe = Join-Path $work 'pfx-probe.toml'+ @"+data_dir = '$work'+[sources.probe]+type = 'demo_logs'+format = 'json'+count = 1+[sinks.probe]+type = 'azure_logs_ingestion'+inputs = ['probe']+endpoint = 'https://127.0.0.1:9'+dcr_immutable_id = '$dcr'+stream_name = 'Custom-BeaconRuntimeRaw'+[sinks.probe.auth]+azure_credential_kind = 'client_certificate_credential'+azure_tenant_id = '$tenant'+azure_client_id = '$client'+certificate_path = '$($variants[$name])'+certificate_password = '$pfxPassword'+"@ | Set-Content -LiteralPath $probe -Encoding ascii+ $out = & $vectorExe validate --skip-healthchecks $probe 2>&1 | Out-String+ $verdict = if ($LASTEXITCODE -eq 0) { 'accepted' } else { 'REJECTED: ' + (($out -split "`n" | Where-Object { $_ -match 'PKCS12|Sink' }) -join ' ').Trim() }+ Write-Output "pfx probe: $name -> $verdict"+ $global:LASTEXITCODE = 0+ }# The mock outlives this step: it keeps answering through the upgrade and the log access# switches, until the cleanup phase stops it.