Skip to content

The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu

fix(siem): preserve log paths in rendered Vector TOML (#962)

Divyam Talwar12e09c817c0a
diff --git a/cli/beacon/cmd/endpoint_destinations_characterization_test.go b/cli/beacon/cmd/endpoint_destinations_characterization_test.go
index 9a22f1dc..5c654a0f 100644
--- a/cli/beacon/cmd/endpoint_destinations_characterization_test.go
+++ b/cli/beacon/cmd/endpoint_destinations_characterization_test.go
@@ -8,6 +8,8 @@ import (
"testing"
"github.com/spf13/cobra"
+
+ "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/siempack"
)
// These characterization tests pin the externally observable behavior of every
@@ -144,7 +146,8 @@ func TestDestinationCommandsCharacterization(t *testing.T) {
if err != nil {
t.Fatalf("%s print-config returned error: %v", tc.group, err)
}
- if !strings.Contains(out, logPath) {
+ // TOML configs write the path escaped; on Windows that doubles its backslashes.
+ if !strings.Contains(out, logPath) && !strings.Contains(out, siempack.TOMLEscape(logPath)) {
t.Fatalf("%s print-config did not wire --log-path %q through to output:\n%s", tc.group, logPath, out)
}
})
diff --git a/cli/beacon/cmd/endpoint_test.go b/cli/beacon/cmd/endpoint_test.go
index e82b1749..0e7a4a9b 100644
--- a/cli/beacon/cmd/endpoint_test.go
+++ b/cli/beacon/cmd/endpoint_test.go
@@ -22,6 +22,7 @@ import (
"github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/lifecycle"
"github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/selfupdate"
"github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/service"
+ "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/siempack"
"github.com/asymptote-labs/agent-beacon/cli/beacon/internal/testenv"
"github.com/asymptote-labs/agent-beacon/cli/beacon/internal/updatecheck"
@@ -2244,7 +2245,7 @@ func TestEndpointCloudWatchPrintConfigPrintsVectorConfig(t *testing.T) {
"aws_cloudwatch_logs",
"BEACON_CLOUDWATCH_LOG_GROUP",
"BEACON_CLOUDWATCH_LOG_STREAM_PREFIX",
- endpointOpts.logPath,
+ siempack.TOMLEscape(endpointOpts.logPath),
} {
if !strings.Contains(output, want) {
t.Fatalf("CloudWatch print-config output missing %q: %s", want, output)
diff --git a/cli/beacon/internal/endpoint/cloudwatch/cloudwatch_test.go b/cli/beacon/internal/endpoint/cloudwatch/cloudwatch_test.go
index ace8ec4e..275762c7 100644
--- a/cli/beacon/internal/endpoint/cloudwatch/cloudwatch_test.go
+++ b/cli/beacon/internal/endpoint/cloudwatch/cloudwatch_test.go
@@ -9,6 +9,7 @@ import (
"testing"
"testing/fstest"
+ "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/siempack"
"github.com/asymptote-labs/agent-beacon/cli/beacon/internal/testenv"
)
@@ -212,7 +213,7 @@ func TestConfigSnippet_DefaultLogPath(t *testing.T) {
if err != nil {
t.Fatalf("ConfigSnippet with empty path returned error: %v", err)
}
- if !strings.Contains(got, DefaultLogPath) {
+ if !strings.Contains(got, siempack.TOMLEscape(DefaultLogPath)) {
t.Fatalf("ConfigSnippet with empty logPath should use DefaultLogPath %q, got: %s", DefaultLogPath, got)
}
}
diff --git a/cli/beacon/internal/endpoint/falcon/falcon_test.go b/cli/beacon/internal/endpoint/falcon/falcon_test.go
index 91c122bd..7776f991 100644
--- a/cli/beacon/internal/endpoint/falcon/falcon_test.go
+++ b/cli/beacon/internal/endpoint/falcon/falcon_test.go
@@ -9,6 +9,7 @@ import (
"testing"
"testing/fstest"
+ "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/siempack"
"github.com/asymptote-labs/agent-beacon/cli/beacon/internal/testenv"
)
@@ -243,7 +244,7 @@ func TestConfigSnippet_DefaultLogPath(t *testing.T) {
if err != nil {
t.Fatalf("ConfigSnippet with empty path returned error: %v", err)
}
- if !strings.Contains(got, DefaultLogPath) {
+ if !strings.Contains(got, siempack.TOMLEscape(DefaultLogPath)) {
t.Fatalf("ConfigSnippet with empty logPath should use DefaultLogPath %q, got: %s", DefaultLogPath, got)
}
}
diff --git a/cli/beacon/internal/endpoint/siempack/siempack.go b/cli/beacon/internal/endpoint/siempack/siempack.go
index 35264cad..d3a6502d 100644
--- a/cli/beacon/internal/endpoint/siempack/siempack.go
+++ b/cli/beacon/internal/endpoint/siempack/siempack.go
@@ -39,6 +39,21 @@ func RenderLogPath(content, logPath string) string {
return strings.ReplaceAll(content, backfillLogPathToken, BackfillLogPath(logPath))
}
+func renderTOMLLogPath(content, logPath string) string {
+ return strings.NewReplacer(
+ logPathToken, TOMLEscape(logPath),
+ inventoryLogPathToken, TOMLEscape(InventoryLogPath(logPath)),
+ backfillLogPathToken, TOMLEscape(BackfillLogPath(logPath)),
+ ).Replace(content)
+}
+
+// TOMLEscape is value as it is written inside a TOML basic string. JSON string escapes are also
+// valid there, but TOML additionally forbids a literal DEL. Sibling paths are derived before
+// encoding, so escapes never reach filepath.Dir.
+func TOMLEscape(value string) string {
+ return strings.ReplaceAll(JSONEscapeForString(value), "\x7f", `\u007f`)
+}
+
// BackfillLogPath is BackfillLogName beside logPath.
func BackfillLogPath(logPath string) string {
if logPath == "" {
@@ -68,11 +83,14 @@ func Install(outputDir string, files []File, logPath string) error {
for _, file := range files {
content := file.Content
if file.TemplateLogPath {
- lp := logPath
- if file.JSONEscape {
- lp = JSONEscapeForString(lp)
+ switch {
+ case file.JSONEscape:
+ content = RenderLogPath(content, JSONEscapeForString(logPath))
+ case isTOMLAsset(file.Name):
+ content = renderTOMLLogPath(content, logPath)
+ default:
+ content = RenderLogPath(content, logPath)
}
- content = RenderLogPath(content, lp)
}
if err := os.WriteFile(filepath.Join(outputDir, file.Name), []byte(content), Mode(file.Name)); err != nil {
return err
@@ -167,9 +185,18 @@ func (p Pack) Render(assetPath, logPath string) (string, error) {
if err != nil {
return "", err
}
+ if isTOMLAsset(assetPath) {
+ return renderTOMLLogPath(content, logPath), nil
+ }
return RenderLogPath(content, logPath), nil
}
+// isTOMLAsset reports whether name, with or without its .tmpl suffix, is a TOML file, whose log
+// paths must be escaped for a basic string.
+func isTOMLAsset(name string) bool {
+ return filepath.Ext(strings.TrimSuffix(name, ".tmpl")) == ".toml"
+}
+
// Install writes the pack files to outputDir (defaulting to DefaultOutputDir),
// substituting logPath (defaulting to DefaultLogPath) into templated files.
func (p Pack) Install(outputDir, logPath string) error {
diff --git a/cli/beacon/internal/endpoint/siempack/vector_paths_test.go b/cli/beacon/internal/endpoint/siempack/vector_paths_test.go
new file mode 100644
index 00000000..021d7a53
--- /dev/null
+++ b/cli/beacon/internal/endpoint/siempack/vector_paths_test.go
@@ -0,0 +1,180 @@
+package siempack_test
+
+import (
+ "encoding/json"
+ "os"
+ "path/filepath"
+ "reflect"
+ "strings"
+ "testing"
+ "testing/fstest"
+
+ "github.com/pelletier/go-toml/v2"
+
+ "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/asymptote"
+ "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/cloudwatch"
+ "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/falcon"
+ "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/gcs"
+ "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/rapid7"
+ "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/s3"
+ "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/sentinel"
+ "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/siempack"
+ "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/sumo"
+)
+
+func TestVectorPackLogPathsRoundTrip(t *testing.T) {
+ paths := []struct{ name, path string }{
+ {"posix", "/var/log/beacon-agent/runtime.jsonl"},
+ {"windows", `C:\ProgramData\Beacon\Endpoint\logs\runtime.jsonl`},
+ {"escape_sequences", `C:\new\temp\runtime.jsonl`},
+ {"quotes", `/tmp/beacon"quoted/runtime.jsonl`},
+ {"spaces_unicode", "/tmp/telemetry logs/日本語/runtime.jsonl"},
+ {"backslash", `/tmp/beacon\temp/runtime.jsonl`},
+ {"controls", "/tmp/beacon\n\r\t\b\f\x01\x1f\x7f/runtime.jsonl"},
+ }
+ packs := []struct {
+ name string
+ install func(string, string) error
+ render func(string) (string, error)
+ inventory bool
+ backfill bool
+ }{
+ {name: "asymptote", install: asymptote.InstallPack, render: asymptote.VectorConfig, inventory: true, backfill: true},
+ {name: "cloudwatch", install: cloudwatch.InstallPack, render: cloudwatch.ConfigSnippet},
+ {name: "falcon", install: falcon.InstallPack, render: falcon.ConfigSnippet},
+ {name: "gcs", install: gcs.InstallPack, inventory: true},
+ {name: "rapid7", install: rapid7.InstallPack},
+ {name: "s3", install: s3.InstallPack, inventory: true},
+ {name: "sentinel", install: sentinel.InstallPack},
+ {name: "sumo", install: sumo.InstallPack},
+ }
+ for _, pack := range packs {
+ t.Run(pack.name, func(t *testing.T) {
+ for _, path := range paths {
+ t.Run(path.name, func(t *testing.T) {
+ want := map[string][]string{"beacon_runtime": {path.path}}
+ if pack.inventory {
+ want["beacon_inventory"] = []string{filepath.Join(filepath.Dir(path.path), "inventory_state.jsonl")}
+ }
+ if pack.backfill {
+ want["beacon_backfill"] = []string{filepath.Join(filepath.Dir(path.path), "cloud-backfill.jsonl")}
+ }
+ dir := t.TempDir()
+ if err := pack.install(dir, path.path); err != nil {
+ t.Fatal(err)
+ }
+ content, err := os.ReadFile(filepath.Join(dir, "vector.toml"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ assertSourcePaths(t, string(content), want)
+ if pack.render != nil {
+ rendered, err := pack.render(path.path)
+ if err != nil {
+ t.Fatal(err)
+ }
+ assertSourcePaths(t, rendered, want)
+ if rendered != string(content) {
+ t.Fatal("rendered snippet and installed config differ")
+ }
+ }
+ })
+ }
+ })
+ }
+}
+
+func assertSourcePaths(t *testing.T, content string, want map[string][]string) {
+ t.Helper()
+ var config struct {
+ Sources map[string]struct {
+ Include []string `toml:"include"`
+ } `toml:"sources"`
+ }
+ if err := toml.Unmarshal([]byte(content), &config); err != nil {
+ t.Fatalf("invalid rendered TOML: %v", err)
+ }
+ for source, paths := range want {
+ if got := config.Sources[source].Include; !reflect.DeepEqual(got, paths) {
+ t.Errorf("%s.include = %q, want %q", source, got, paths)
+ }
+ }
+}
+
+func TestPackTOMLRenderingPreservesSiblingPathsAndArchiveGlob(t *testing.T) {
+ const content = "[sources.beacon_runtime]\ninclude = [\"{{LOG_PATH}}\", \"{{LOG_PATH}}.*\"]\n" +
+ "[sources.beacon_inventory]\ninclude = [\"{{INVENTORY_LOG_PATH}}\"]\n" +
+ "[sources.beacon_backfill]\ninclude = [\"{{BACKFILL_LOG_PATH}}\"]\n"
+ path := `/tmp/beacon"quoted\temp/runtime.jsonl`
+ pack := siempack.Pack{
+ DefaultLogPath: path,
+ FS: fstest.MapFS{"pack/vector.toml.tmpl": {Data: []byte(content)}},
+ Assets: []siempack.Asset{{Source: "pack/vector.toml.tmpl", Name: "vector.toml", TemplateLogPath: true}},
+ }
+ want := map[string][]string{
+ "beacon_runtime": {path, path + ".*"},
+ "beacon_inventory": {filepath.Join(filepath.Dir(path), "inventory_state.jsonl")},
+ "beacon_backfill": {filepath.Join(filepath.Dir(path), "cloud-backfill.jsonl")},
+ }
+ rendered, err := pack.Render("pack/vector.toml.tmpl", "")
+ if err != nil {
+ t.Fatal(err)
+ }
+ assertSourcePaths(t, rendered, want)
+ dir := t.TempDir()
+ if err := pack.Install(dir, ""); err != nil {
+ t.Fatal(err)
+ }
+ installed, err := os.ReadFile(filepath.Join(dir, "vector.toml"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if string(installed) != rendered {
+ t.Fatal("default path differs between render and install")
+ }
+}
+
+func TestPathRenderingPreservesOtherFormats(t *testing.T) {
+ path := `/tmp/beacon"quoted\temp/runtime.jsonl`
+ for _, name := range []string{"run.sh", "conf.yaml", "README.md", "log.xml"} {
+ t.Run(name, func(t *testing.T) {
+ const content = "path={{LOG_PATH}}; inventory={{INVENTORY_LOG_PATH}}; backfill={{BACKFILL_LOG_PATH}}"
+ want := siempack.RenderLogPath(content, path)
+ pack := siempack.Pack{FS: fstest.MapFS{"pack/" + name + ".tmpl": {Data: []byte(content)}}}
+ rendered, err := pack.Render("pack/"+name+".tmpl", path)
+ if err != nil || rendered != want {
+ t.Fatalf("Render = %q, %v; want %q", rendered, err, want)
+ }
+ dir := t.TempDir()
+ if err := siempack.Install(dir, []siempack.File{{Name: name, Content: content, TemplateLogPath: true}}, path); err != nil {
+ t.Fatal(err)
+ }
+ installed, err := os.ReadFile(filepath.Join(dir, name))
+ if err != nil || string(installed) != want {
+ t.Fatalf("Install = %q, %v; want %q", installed, err, want)
+ }
+ })
+ }
+ t.Run("json", func(t *testing.T) {
+ dir := t.TempDir()
+ if err := siempack.Install(dir, []siempack.File{{Name: "conf.json", Content: `{"path":"{{LOG_PATH}}"}`, TemplateLogPath: true, JSONEscape: true}}, path); err != nil {
+ t.Fatal(err)
+ }
+ content, err := os.ReadFile(filepath.Join(dir, "conf.json"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ var got struct{ Path string }
+ if err := json.Unmarshal(content, &got); err != nil || got.Path != path {
+ t.Fatalf("JSON path = %q, %v; want %q", got.Path, err, path)
+ }
+ })
+ t.Run("ordinary_posix", func(t *testing.T) {
+ const content = "include = [\"{{LOG_PATH}}\"]\n"
+ pack := siempack.Pack{FS: fstest.MapFS{"pack/vector.toml.tmpl": {Data: []byte(content)}}}
+ rendered, err := pack.Render("pack/vector.toml.tmpl", "/var/log/beacon-agent/runtime.jsonl")
+ if err != nil || rendered != strings.ReplaceAll(content, "{{LOG_PATH}}", "/var/log/beacon-agent/runtime.jsonl") {
+ t.Fatalf("ordinary POSIX render changed: %q, %v", rendered, err)
+ }
+ })
+}
diff --git a/docs/changelog/cli.mdx b/docs/changelog/cli.mdx
index 1a5df6be..9cdf18ef 100644
--- a/docs/changelog/cli.mdx
+++ b/docs/changelog/cli.mdx
@@ -19,6 +19,7 @@ Stay up to date with the latest changes to the Agent Beacon CLI.
- **Dashboard token totals honor the session filter** · With the dashboard's session-state filter set to captured, recurring or empty, the token view showed zero tokens and no events, and `all` or an unrecognized value hid everything. Token totals now cover the same sessions the event list shows for that filter.
- **Hermes sync keeps profiles apart** · Every Hermes home shared one sync cursor, so syncing a second profile or a custom `HERMES_HOME` on the same host could skip sessions the first sync had already passed. Each home now has its own cursor, and synced events record the home in `harness.config_path` and the profile in the new `harness.profile` field. The native `~/.hermes` root keeps its existing cursor, and the home a default sync resolves to (through `HERMES_HOME` or a sticky profile) starts its keyed cursor from the one it had been advancing, so neither re-emits history; any other home collects its retained history once more on its first sync.
- **Failed Grok writes are recorded as failures** · When a Grok `write_file` or edit failed, the hook recorded it as a completed `file.modified` with a diff built from content that was never written, and the threat scan read that content as a change. The failed attempt is now a high-severity `tool.failed` with its tool, path and call ID and no diff. Successful writes are unchanged.
+- **Vector forwarding configs work with Windows log paths** · Beacon wrote log paths into the Vector configs for Beacon Cloud and the SIEM packs without escaping them, so on Windows a path's backslashes made `vector.toml` fail to parse, or quietly point Vector at a different file. Paths are now escaped and Vector reads exactly the configured runtime, inventory and backfill logs.
- **Dashboard output tokens count reasoning once** · The local dashboard's token view added reasoning tokens to output, but `gen_ai.usage.output_tokens` already includes them, so models that report reasoning had their output and total tokens overstated. Output and the total now count reasoning once, the split bar no longer shows reasoning as a separate segment, and a note below it gives the reasoning count included in output. `beacon token-usage` already counted it this way.
---