Skip to content

The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu

sentinel: keep the forwarder's Vector and service across a rejected reconcile

shukieshah2b81b59d071a
diff --git a/cli/beacon/internal/endpoint/sentinel/forwarder.go b/cli/beacon/internal/endpoint/sentinel/forwarder.go
index 1e287130..a3dd1c57 100644
--- a/cli/beacon/internal/endpoint/sentinel/forwarder.go
+++ b/cli/beacon/internal/endpoint/sentinel/forwarder.go
@@ -79,7 +79,10 @@ type Settings struct {
AuthorityHost string `json:"authority_host,omitempty"`
ReadFrom string `json:"read_from"`
LogPath string `json:"log_path"`
- InstalledAt string `json:"installed_at"`
+ // VectorBin is the Vector binary install was told to run, so reconcile keeps running it; empty
+ // means the one Beacon's package installed, found again each time.
+ VectorBin string `json:"vector_bin,omitempty"`
+ InstalledAt string `json:"installed_at"`
}
// InstallOptions is what `forwarder install` was given.
@@ -154,6 +157,7 @@ func (o InstallOptions) settings() (Settings, error) {
AuthorityHost: strings.TrimRight(strings.TrimSpace(o.AuthorityHost), "/"),
ReadFrom: strings.TrimSpace(o.ReadFrom),
LogPath: o.LogPath,
+ VectorBin: strings.TrimSpace(o.VectorBin),
}
if err := httpsURL("--endpoint", s.Endpoint, true); err != nil {
return s, err
@@ -334,7 +338,7 @@ func Install(o InstallOptions) (InstallResult, error) {
if !manager.Supported() {
return InstallResult{}, errors.New(manager.UnsupportedReason())
}
- vector, err := findVector(o.VectorBin)
+ vector, err := findVector(settings.VectorBin)
if err != nil {
return InstallResult{}, err
}
@@ -560,7 +564,7 @@ func Reconcile(opts ReconcileOptions) (ReconcileResult, error) {
if !manager.Supported() {
return ReconcileResult{Skipped: manager.UnsupportedReason()}, nil
}
- vector, err := findVector("")
+ vector, err := findVector(settings.VectorBin)
if err != nil {
return ReconcileResult{Warning: err.Error()}, err
}
@@ -592,8 +596,9 @@ func Reconcile(opts ReconcileOptions) (ReconcileResult, error) {
LogDir: logDir,
HasCertificatePassword: hasPassword,
})
- current, _ := os.ReadFile(configPath)
+ current, readErr := os.ReadFile(configPath)
configChanged := string(current) != rendered
+ var rejected string
if configChanged {
candidate := configPath + ".new"
if err := atomicfile.Write(candidate, []byte(rendered), 0o600); err != nil {
@@ -601,9 +606,15 @@ func Reconcile(opts ReconcileOptions) (ReconcileResult, error) {
}
if err := validateVector(vector.Path, candidate); err != nil {
_ = os.Remove(candidate)
- return ReconcileResult{Warning: "kept the previous config: " + err.Error()}, nil
- }
- if err := os.Rename(candidate, configPath); err != nil {
+ if readErr != nil {
+ // Nothing to fall back to.
+ return ReconcileResult{Warning: "Vector rejected the config, and there is no previous one: " + err.Error()}, nil
+ }
+ // The previous config still runs. The service is put back regardless, because the MSI
+ // upgrade removed it before this install, and an old config running beats none.
+ rejected = "kept the previous config, which Vector still runs: " + err.Error()
+ configChanged = false
+ } else if err := os.Rename(candidate, configPath); err != nil {
return ReconcileResult{Warning: err.Error()}, err
}
}
@@ -614,12 +625,12 @@ func Reconcile(opts ReconcileOptions) (ReconcileResult, error) {
}
}
if !opts.Restart && !configChanged && unitCurrent && manager.Status().Running {
- return ReconcileResult{Unchanged: true}, nil
+ return ReconcileResult{Unchanged: rejected == "", Warning: rejected}, nil
}
if err := manager.Load(); err != nil {
return ReconcileResult{Warning: err.Error()}, err
}
- return ReconcileResult{Restarted: true}, nil
+ return ReconcileResult{Restarted: true, Warning: rejected}, nil
}
// Status is what `forwarder status` and `endpoint status` show.
diff --git a/cli/beacon/internal/endpoint/sentinel/forwarder_e2e_test.go b/cli/beacon/internal/endpoint/sentinel/forwarder_e2e_test.go
index 35f19a56..15a10bba 100644
--- a/cli/beacon/internal/endpoint/sentinel/forwarder_e2e_test.go
+++ b/cli/beacon/internal/endpoint/sentinel/forwarder_e2e_test.go
@@ -2,7 +2,10 @@ package sentinel
import (
"bufio"
+ "crypto/sha1"
+ "encoding/hex"
"encoding/json"
+ "encoding/pem"
"fmt"
"os"
"os/exec"
@@ -111,23 +114,27 @@ func TestForwarderConfigDeliversThroughRealVector(t *testing.T) {
// The restricted-mode switch replaces the log with a copy under the same name. Vector recognizes
// a file by its first line, so it keeps reading the replaced one; install and repair, which make
// the switch, then restart the forwarder (Reconcile with Restart), and it resumes in the copy.
- data, err := os.ReadFile(settings.LogPath)
- if err != nil {
- t.Fatal(err)
- }
- if err := os.WriteFile(settings.LogPath+".copy", data, 0o600); err != nil {
- t.Fatal(err)
- }
- if err := os.Rename(settings.LogPath+".copy", settings.LogPath); err != nil {
- t.Fatal(err)
- }
- log.append(5) // between the switch and the restart
- if !vector.stop() {
- graceful = false
+ // On Windows a plain rename cannot replace a file Vector holds open (Beacon's writer renames with
+ // POSIX semantics instead), so there the MSI smoke covers it, with the real switch.
+ if runtime.GOOS != "windows" {
+ data, err := os.ReadFile(settings.LogPath)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(settings.LogPath+".copy", data, 0o600); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.Rename(settings.LogPath+".copy", settings.LogPath); err != nil {
+ t.Fatal(err)
+ }
+ log.append(5) // between the switch and the restart
+ if !vector.stop() {
+ graceful = false
+ }
+ vector = startVector(t, vectorBin, configPath, mock.URL, caFile)
+ log.append(5)
+ waitForRows(t, mock, log.ids, 60*time.Second)
}
- vector = startVector(t, vectorBin, configPath, mock.URL, caFile)
- log.append(5)
- waitForRows(t, mock, log.ids, 60*time.Second)
time.Sleep(3 * time.Second) // let anything resent arrive before counting
vector.stop()
@@ -158,7 +165,7 @@ func TestForwarderConfigDeliversThroughRealVector(t *testing.T) {
}
if !t.Failed() {
tokens, ingests, _ := mock.Counts()
- t.Logf("%d events delivered exactly once across a rotation, a restart and a log copy; %d token requests, %d ingest requests", len(log.ids), tokens, ingests)
+ t.Logf("%d events delivered exactly once across retried 403s, a rotation, a restart and (off Windows) a log copy; %d token requests, %d ingest requests", len(log.ids), tokens, ingests)
}
}
@@ -279,12 +286,21 @@ func trustCA(t *testing.T, caFile string) {
t.Fatalf("certutil -addstore Root: %v\n%s", err, out)
}
t.Cleanup(func() {
- // The mock's CA has its own random name; delete exactly that certificate.
- out, _ := exec.Command("powershell", "-NoProfile", "-Command",
- `$c = New-Object System.Security.Cryptography.X509Certificates.X509Certificate2('`+caFile+`'); `+
- `Get-ChildItem Cert:\LocalMachine\Root | Where-Object Thumbprint -eq $c.Thumbprint | Remove-Item`).CombinedOutput()
- if len(out) > 0 {
- t.Logf("removing the mock CA: %s", out)
+ // The mock's CA has its own random name; delete exactly that certificate, by its SHA-1
+ // thumbprint, which is how certutil names one.
+ data, err := os.ReadFile(caFile)
+ if err != nil {
+ t.Logf("removing the mock CA: %v", err)
+ return
+ }
+ block, _ := pem.Decode(data)
+ if block == nil {
+ t.Log("removing the mock CA: not PEM")
+ return
+ }
+ thumbprint := sha1.Sum(block.Bytes)
+ if out, err := exec.Command("certutil", "-delstore", "Root", hex.EncodeToString(thumbprint[:])).CombinedOutput(); err != nil {
+ t.Logf("removing the mock CA: %v\n%s", err, out)
}
})
}
diff --git a/cli/beacon/internal/endpoint/sentinel/forwarder_test.go b/cli/beacon/internal/endpoint/sentinel/forwarder_test.go
index 023e69ac..7df64e11 100644
--- a/cli/beacon/internal/endpoint/sentinel/forwarder_test.go
+++ b/cli/beacon/internal/endpoint/sentinel/forwarder_test.go
@@ -311,6 +311,18 @@ func TestForwarderReconcile(t *testing.T) {
if readFile(t, configPath) != "# old\n" {
t.Fatal("a rejected reconcile replaced the config")
}
+ // After an MSI upgrade removed the service, a rejected config still gets the service back, on
+ // the previous config.
+ if err := Remove(true); err != nil {
+ t.Fatal(err)
+ }
+ f.svc.current = false
+ if got, err := Reconcile(ReconcileOptions{Restart: true}); err != nil || got.Warning == "" || !got.Restarted || !f.svc.running {
+ t.Fatalf("rejected after upgrade: %+v %v running=%t", got, err, f.svc.running)
+ }
+ if readFile(t, configPath) != "# old\n" {
+ t.Fatal("a rejected reconcile replaced the config")
+ }
f.reject = nil
// The MSI upgrade removes the service and keeps the state; the install that follows puts the
// service back.
@@ -388,3 +400,24 @@ func TestForwarderInstallRemovesAServiceThatNeverStarted(t *testing.T) {
t.Fatalf("the settings must stay for the next install or reconcile: %v", err)
}
}
+
+// The Vector install was told to run is the one reconcile keeps running.
+func TestForwarderReconcileKeepsTheVectorInstallWasGiven(t *testing.T) {
+ newForwarderFixture(t)
+ var asked []string
+ findVector = func(explicit string) (asymptote.VectorInfo, error) {
+ asked = append(asked, explicit)
+ return asymptote.VectorInfo{Path: "/custom/vector", Version: "0.56.0"}, nil
+ }
+ o := validInstall()
+ o.VectorBin = "/custom/vector"
+ if _, err := Install(o); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := Reconcile(ReconcileOptions{Restart: true}); err != nil {
+ t.Fatal(err)
+ }
+ if len(asked) != 2 || asked[0] != "/custom/vector" || asked[1] != "/custom/vector" {
+ t.Fatalf("findVector was asked for %q", asked)
+ }
+}
diff --git a/packaging/windows/smoke-sentinel-forwarder.ps1 b/packaging/windows/smoke-sentinel-forwarder.ps1
index 54593fe2..1fe9c78e 100644
--- a/packaging/windows/smoke-sentinel-forwarder.ps1
+++ b/packaging/windows/smoke-sentinel-forwarder.ps1
@@ -140,12 +140,55 @@ switch ($Phase) {
New-Item -ItemType Directory -Force -Path $work | Out-Null
if (-not $MockExe -or -not (Test-Path -LiteralPath $MockExe)) { throw "-MockExe must name the built sentinel-mock.exe" }
- # A certificate credential, the one the docs lead with, exported the way OpenSSL 3 reads it.
- $cert = New-SelfSignedCertificate -Subject 'CN=beacon-sentinel-smoke' -CertStoreLocation Cert:\LocalMachine\My `
+ # A certificate credential, the one the docs lead with, made with `openssl pkcs12 -export`.
+ $openssl = (Get-Command openssl -ErrorAction SilentlyContinue).Source
+ if (-not $openssl) { $openssl = Join-Path $env:ProgramFiles 'Git\usr\bin\openssl.exe' }
+ $key = Join-Path $work 'app-key.pem'
+ $certPem = Join-Path $work 'app-cert.pem'
+ & $openssl req -x509 -newkey rsa:2048 -nodes -keyout $key -out $certPem -days 2 -subj '/CN=beacon-sentinel-smoke' 2>&1 | Out-Null
+ & $openssl pkcs12 -export -inkey $key -in $certPem -out $pfx -passout "pass:$pfxPassword" 2>&1 | Out-Null
+ if ($LASTEXITCODE -ne 0 -or -not (Test-Path -LiteralPath $pfx)) { throw "openssl could not make the test PFX" }
+ Remove-Item -LiteralPath $key
+
+ # Which PFX encodings the bundled Vector can read, recorded for the docs: Windows'
+ # Export-PfxCertificate with each algorithm, and OpenSSL's default.
+ $cert = New-SelfSignedCertificate -Subject 'CN=beacon-sentinel-smoke-export' -CertStoreLocation Cert:\LocalMachine\My `
-KeyAlgorithm RSA -KeyLength 2048 -KeyExportPolicy Exportable -NotAfter (Get-Date).AddDays(2)
- $secure = ConvertTo-SecureString $pfxPassword -AsPlainText -Force
- Export-PfxCertificate -Cert $cert -FilePath $pfx -Password $secure -CryptoAlgorithmOption AES256_SHA256 | Out-Null
$cert.Thumbprint | Set-Content -LiteralPath (Join-Path $work 'cert-thumbprint')
+ $secure = ConvertTo-SecureString $pfxPassword -AsPlainText -Force
+ $variants = [ordered]@{ 'openssl pkcs12 -export' = $pfx }
+ foreach ($algorithm in @('AES256_SHA256', 'TripleDES_SHA1')) {
+ $path = Join-Path $work "export-$algorithm.pfx"
+ Export-PfxCertificate -Cert $cert -FilePath $path -Password $secure -CryptoAlgorithmOption $algorithm | Out-Null
+ $variants["Export-PfxCertificate -CryptoAlgorithmOption $algorithm"] = $path
+ }
+ $vectorExe = Join-Path $env:ProgramFiles 'Beacon\bin\vector.exe'
+ foreach ($name in $variants.Keys) {
+ $probe = Join-Path $work 'pfx-probe.toml'
+ @"
+data_dir = '$work'
+[sources.probe]
+type = 'demo_logs'
+format = 'json'
+count = 1
+[sinks.probe]
+type = 'azure_logs_ingestion'
+inputs = ['probe']
+endpoint = 'https://127.0.0.1:9'
+dcr_immutable_id = '$dcr'
+stream_name = 'Custom-BeaconRuntimeRaw'
+[sinks.probe.auth]
+azure_credential_kind = 'client_certificate_credential'
+azure_tenant_id = '$tenant'
+azure_client_id = '$client'
+certificate_path = '$($variants[$name])'
+certificate_password = '$pfxPassword'
+"@ | Set-Content -LiteralPath $probe -Encoding ascii
+ $out = & $vectorExe validate --skip-healthchecks $probe 2>&1 | Out-String
+ $verdict = if ($LASTEXITCODE -eq 0) { 'accepted' } else { 'REJECTED: ' + (($out -split "`n" | Where-Object { $_ -match 'PKCS12|Sink' }) -join ' ').Trim() }
+ Write-Output "pfx probe: $name -> $verdict"
+ $global:LASTEXITCODE = 0
+ }
# The mock outlives this step: it keeps answering through the upgrade and the log access
# switches, until the cleanup phase stops it.