Skip to content

The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu

fix(siem): escape log paths in the datadog and elastic yaml configs (#1078)

Rudy Mizrahi Celekli99a7e3ab8621
diff --git a/cli/beacon/cmd/endpoint_destinations.go b/cli/beacon/cmd/endpoint_destinations.go
index 3356a95e..28f4462c 100644
--- a/cli/beacon/cmd/endpoint_destinations.go
+++ b/cli/beacon/cmd/endpoint_destinations.go
@@ -104,10 +104,8 @@ var siemDestinations = []siemDestination{
name: "wazuh",
short: "Manage Wazuh integration content",
printConfig: &destPrintConfig{
- short: "Print a Wazuh localfile snippet",
- render: func(cfg endpointconfig.Config) (string, error) {
- return wazuh.LocalfileSnippet(cfg.LogPath), nil
- },
+ short: "Print a Wazuh localfile snippet",
+ render: func(cfg endpointconfig.Config) (string, error) { return wazuh.LocalfileSnippet(cfg.LogPath) },
},
installPack: &destInstallPack{
short: "Write Wazuh rules and config snippets to a directory",
@@ -121,8 +119,12 @@ var siemDestinations = []siemDestination{
message: "Beacon endpoint Wazuh validation event",
print: func(cfg endpointconfig.Config) {
fmt.Println("Expected Wazuh fields: vendor=beacon product=endpoint-agent event.kind=agent_runtime")
- fmt.Println("Wazuh localfile snippet:")
- fmt.Print(wazuh.LocalfileSnippet(cfg.LogPath))
+ if snippet, err := wazuh.LocalfileSnippet(cfg.LogPath); err != nil {
+ fmt.Println(err)
+ } else {
+ fmt.Println("Wazuh localfile snippet:")
+ fmt.Print(snippet)
+ }
fmt.Println("Expected base rule: 100500")
},
},
diff --git a/cli/beacon/cmd/endpoint_destinations_characterization_test.go b/cli/beacon/cmd/endpoint_destinations_characterization_test.go
index 9f38581e..cf1d9732 100644
--- a/cli/beacon/cmd/endpoint_destinations_characterization_test.go
+++ b/cli/beacon/cmd/endpoint_destinations_characterization_test.go
@@ -8,8 +8,6 @@ import (
"testing"
"github.com/spf13/cobra"
-
- "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/siempack"
)
// These tests drive every forwarding-destination command (print-config /
@@ -52,8 +50,8 @@ func TestDestinationCommandsCharacterization(t *testing.T) {
if err != nil {
t.Fatalf("%s print-config returned error: %v", tc.group, err)
}
- // TOML configs write the path escaped; on Windows that doubles its backslashes.
- if !strings.Contains(out, logPath) && !strings.Contains(out, siempack.TOMLEscape(logPath)) {
+ // Quoted configs write the path escaped; on Windows that doubles its backslashes.
+ if !strings.Contains(out, logPath) && !strings.Contains(out, strings.ReplaceAll(logPath, `\`, `\\`)) {
t.Fatalf("%s print-config did not wire --log-path %q through to output:\n%s", tc.group, logPath, out)
}
})
diff --git a/cli/beacon/internal/endpoint/elastic/elastic.go b/cli/beacon/internal/endpoint/elastic/elastic.go
index 9ead0000..90e0b3dc 100644
--- a/cli/beacon/internal/endpoint/elastic/elastic.go
+++ b/cli/beacon/internal/endpoint/elastic/elastic.go
@@ -7,7 +7,6 @@ import (
"io/fs"
"os"
"path/filepath"
- "strings"
"github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/siempack"
)
@@ -56,7 +55,7 @@ func inputSnippetFromFS(fsys fs.FS, logPath string) (string, error) {
if err != nil {
return "", fmt.Errorf("elastic pack asset %q: %w", "pack/filebeat.yml.tmpl", err)
}
- return strings.ReplaceAll(content, "{{LOG_PATH}}", logPath), nil
+ return siempack.RenderLogPath("filebeat.yml", content, logPath), nil
}
// standaloneAgentConfigFromFS renders the elastic-agent standalone config using the supplied FS.
@@ -68,7 +67,7 @@ func standaloneAgentConfigFromFS(fsys fs.FS, logPath string) (string, error) {
if err != nil {
return "", fmt.Errorf("elastic pack asset %q: %w", "pack/elastic-agent-standalone.yml.tmpl", err)
}
- return strings.ReplaceAll(content, "{{LOG_PATH}}", logPath), nil
+ return siempack.RenderLogPath("elastic-agent-standalone.yml", content, logPath), nil
}
// filesFromFS builds the full file list from the supplied FS, propagating any
diff --git a/cli/beacon/internal/endpoint/elastic/elastic_test.go b/cli/beacon/internal/endpoint/elastic/elastic_test.go
index 32a232aa..04bc66c6 100644
--- a/cli/beacon/internal/endpoint/elastic/elastic_test.go
+++ b/cli/beacon/internal/endpoint/elastic/elastic_test.go
@@ -160,7 +160,11 @@ func TestInputSnippet_DefaultLogPath(t *testing.T) {
if err != nil {
t.Fatalf("InputSnippet with empty path returned error: %v", err)
}
- if !strings.Contains(got, DefaultLogPath) {
+ want, err := InputSnippet(DefaultLogPath)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if got != want {
t.Fatalf("InputSnippet with empty logPath should use DefaultLogPath %q, got: %s", DefaultLogPath, got)
}
}
diff --git a/cli/beacon/internal/endpoint/sentinel/sentinel.go b/cli/beacon/internal/endpoint/sentinel/sentinel.go
index dee68e61..1432741f 100644
--- a/cli/beacon/internal/endpoint/sentinel/sentinel.go
+++ b/cli/beacon/internal/endpoint/sentinel/sentinel.go
@@ -25,7 +25,7 @@ func Files() []siempack.File {
{Name: "README.md", Content: mustRead("pack/README.md")},
{Name: "dcr-transform.kql", Content: DCRTransform()},
{Name: "table-schema.json", Content: mustRead("pack/table-schema.json")},
- {Name: "dcr-template.json", Content: renderDCRTemplate("pack/dcr-template.json"), TemplateLogPath: true, JSONEscape: true},
+ {Name: "dcr-template.json", Content: renderDCRTemplate("pack/dcr-template.json"), TemplateLogPath: true},
// The Logs Ingestion API path, for hosts without Azure Monitor Agent: Vector sends each line
// as RawData to a DCR that runs the same transform, so both paths fill the same table.
{Name: "vector.toml", Content: mustRead("pack/vector.toml.tmpl"), TemplateLogPath: true},
diff --git a/cli/beacon/internal/endpoint/siempack/log_paths_test.go b/cli/beacon/internal/endpoint/siempack/log_paths_test.go
new file mode 100644
index 00000000..ccd0df40
--- /dev/null
+++ b/cli/beacon/internal/endpoint/siempack/log_paths_test.go
@@ -0,0 +1,358 @@
+package siempack_test
+
+import (
+ "bytes"
+ "encoding/json"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "reflect"
+ "runtime"
+ "slices"
+ "strings"
+ "testing"
+ "testing/fstest"
+
+ "github.com/pelletier/go-toml/v2"
+ "gopkg.in/yaml.v3"
+ "mvdan.cc/sh/v3/syntax"
+
+ "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/asymptote"
+ "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/cloudwatch"
+ "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/datadog"
+ "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/elastic"
+ "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/falcon"
+ "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/gcs"
+ "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/rapid7"
+ "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/s3"
+ "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/sentinel"
+ "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/siempack"
+ "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/sumo"
+)
+
+// logPaths are the path shapes every pack must carry through unchanged: the system default on each
+// platform, and paths full of the syntax each format quotes.
+var logPaths = []struct{ name, path string }{
+ {"posix", "/var/log/beacon-agent/runtime.jsonl"},
+ {"windows", `C:\Users\x\runtime.jsonl`},
+ {"escape_sequences", `C:\new\temp\runtime.jsonl`},
+ {"syntax", "/tmp/a: b/\"q\"/$(touch pwned)/`x`/}{/runtime.jsonl"},
+ {"spaces_unicode", "/tmp/telemetry logs/日本語/runtime.jsonl"},
+ {"controls", "/tmp/beacon\n\r\t\b\f\x01\x1f\x7f\u0085\u009f\ufeff\uffff/runtime.jsonl"},
+}
+
+func TestTOMLLogPathsRoundTrip(t *testing.T) {
+ packs := []struct {
+ name string
+ install func(string, string) error
+ render func(string) (string, error)
+ inventory bool
+ backfill bool
+ }{
+ {name: "asymptote", install: asymptote.InstallPack, render: asymptote.VectorConfig, inventory: true, backfill: true},
+ {name: "cloudwatch", install: cloudwatch.InstallPack, render: cloudwatch.ConfigSnippet},
+ {name: "falcon", install: falcon.InstallPack, render: falcon.ConfigSnippet},
+ {name: "gcs", install: gcs.InstallPack, inventory: true},
+ {name: "rapid7", install: rapid7.InstallPack},
+ {name: "s3", install: s3.InstallPack, inventory: true},
+ {name: "sentinel", install: sentinel.InstallPack},
+ {name: "sumo", install: sumo.InstallPack},
+ }
+ for _, pack := range packs {
+ for _, path := range logPaths {
+ t.Run(pack.name+"/"+path.name, func(t *testing.T) {
+ want := map[string][]string{"beacon_runtime": {path.path}}
+ if pack.inventory {
+ want["beacon_inventory"] = []string{siempack.InventoryLogPath(path.path)}
+ }
+ if pack.backfill {
+ want["beacon_backfill"] = []string{siempack.BackfillLogPath(path.path)}
+ }
+ installed := installedFile(t, pack.install, path.path, "vector.toml")
+ assertSourcePaths(t, installed, want)
+ if pack.render != nil {
+ assertRendered(t, pack.render, path.path, installed)
+ }
+ })
+ }
+ }
+}
+
+func TestYAMLLogPathsRoundTrip(t *testing.T) {
+ files := []struct {
+ pack string
+ install func(string, string) error
+ name string
+ render func(string) (string, error)
+ }{
+ {"datadog", datadog.InstallPack, "conf.yaml", datadog.ConfigSnippet},
+ {"elastic", elastic.InstallPack, "filebeat.yml", elastic.InputSnippet},
+ {"elastic", elastic.InstallPack, "elastic-agent-standalone.yml", nil},
+ }
+ for _, file := range files {
+ for _, path := range logPaths {
+ t.Run(file.name+"/"+path.name, func(t *testing.T) {
+ installed := installedFile(t, file.install, path.path, file.name)
+ var doc any
+ if err := yaml.Unmarshal([]byte(installed), &doc); err != nil {
+ t.Fatalf("invalid rendered YAML: %v\n%s", err, installed)
+ }
+ if got := stringsUnder(doc, "path", "paths"); !reflect.DeepEqual(got, []string{path.path}) {
+ t.Fatalf("log paths = %q, want %q", got, path.path)
+ }
+ if file.render != nil {
+ assertRendered(t, file.render, path.path, installed)
+ }
+ })
+ }
+ }
+}
+
+func TestJSONLogPathsRoundTrip(t *testing.T) {
+ for _, path := range logPaths {
+ t.Run(path.name, func(t *testing.T) {
+ installed := installedFile(t, sentinel.InstallPack, path.path, "dcr-template.json")
+ var doc any
+ if err := json.Unmarshal([]byte(installed), &doc); err != nil {
+ t.Fatalf("invalid rendered JSON: %v", err)
+ }
+ if got := stringsUnder(doc, "filePatterns"); !reflect.DeepEqual(got, []string{path.path}) {
+ t.Fatalf("filePatterns = %q, want %q", got, path.path)
+ }
+ })
+ }
+}
+
+// The smoke-test scripts take the log path as the default of "${BEACON_LOG:-...}". Each script must
+// parse, and its defaults must evaluate to the paths without running anything inside them.
+func TestShellLogPathsRoundTrip(t *testing.T) {
+ // The smoke-test scripts run on macOS and Linux hosts. Git for Windows' sh drops carriage
+ // returns from its input, so it cannot read these values back exactly.
+ if runtime.GOOS == "windows" {
+ t.Skip("the smoke-test scripts are POSIX shell scripts for macOS and Linux")
+ }
+ sh, err := exec.LookPath("sh")
+ if err != nil {
+ t.Skip("no POSIX shell to run the smoke-test scripts")
+ }
+ scripts := []struct {
+ pack string
+ install func(string, string) error
+ name string
+ render func(string) (string, error)
+ inventory bool
+ }{
+ {"asymptote", asymptote.InstallPack, "asymptote-ingest-smoke-test.sh", asymptote.IngestSmokeTest, true},
+ {"falcon", falcon.InstallPack, "falcon-hec-smoke-test.sh", falcon.SmokeTest, false},
+ {"gcs", gcs.InstallPack, "gcs-upload-smoke-test.sh", gcs.UploadSmokeTest, true},
+ {"rapid7", rapid7.InstallPack, "rapid7-upload-smoke-test.sh", rapid7.UploadSmokeTest, false},
+ {"s3", s3.InstallPack, "s3-upload-smoke-test.sh", s3.UploadSmokeTest, true},
+ {"sumo", sumo.InstallPack, "sumo-upload-smoke-test.sh", sumo.UploadSmokeTest, false},
+ }
+ for _, script := range scripts {
+ for _, path := range logPaths {
+ t.Run(script.pack+"/"+path.name, func(t *testing.T) {
+ dir := t.TempDir()
+ if err := script.install(dir, path.path); err != nil {
+ t.Fatal(err)
+ }
+ scriptPath := filepath.Join(dir, script.name)
+ if out, err := exec.Command(sh, "-n", scriptPath).CombinedOutput(); err != nil {
+ t.Fatalf("sh -n: %v\n%s", err, out)
+ }
+ content, err := os.ReadFile(scriptPath)
+ if err != nil {
+ t.Fatal(err)
+ }
+ assertRendered(t, script.render, path.path, string(content))
+
+ want := map[string]string{"BEACON_LOG": path.path}
+ if script.inventory {
+ want["BEACON_INVENTORY_LOG"] = siempack.InventoryLogPath(path.path)
+ }
+ // Run only the default assignments: the rest of the script needs credentials.
+ defaults, names := defaultAssignments(t, string(content), want)
+ for _, name := range names {
+ defaults += "\nprintf '%s\\0' \"$" + name + "\""
+ }
+ cwd := t.TempDir()
+ cmd := exec.Command(sh, "-c", defaults)
+ cmd.Dir = cwd
+ cmd.Env = []string{"PATH=" + os.Getenv("PATH")}
+ var stdout, stderr bytes.Buffer
+ cmd.Stdout, cmd.Stderr = &stdout, &stderr
+ if err := cmd.Run(); err != nil || stderr.Len() > 0 {
+ t.Fatalf("defaults failed: %v\n%s\n%s", err, stderr.String(), defaults)
+ }
+ got := strings.Split(strings.TrimSuffix(stdout.String(), "\x00"), "\x00")
+ for i, name := range names {
+ if got[i] != want[name] {
+ t.Errorf("%s = %q, want %q", name, got[i], want[name])
+ }
+ }
+ if entries, _ := os.ReadDir(cwd); len(entries) > 0 {
+ t.Fatalf("evaluating the defaults ran a command that created %s", entries[0].Name())
+ }
+ })
+ }
+ }
+}
+
+// defaultAssignments returns the script's top-level assignments to the variables in want, in order.
+func defaultAssignments(t *testing.T, content string, want map[string]string) (string, []string) {
+ t.Helper()
+ file, err := syntax.NewParser().Parse(strings.NewReader(content), "")
+ if err != nil {
+ t.Fatal(err)
+ }
+ var lines, names []string
+ for _, stmt := range file.Stmts {
+ call, ok := stmt.Cmd.(*syntax.CallExpr)
+ if !ok || len(call.Args) != 0 || len(call.Assigns) != 1 {
+ continue
+ }
+ if name := call.Assigns[0].Name.Value; want[name] != "" {
+ lines = append(lines, content[stmt.Pos().Offset():stmt.End().Offset()])
+ names = append(names, name)
+ }
+ }
+ if len(names) != len(want) {
+ t.Fatalf("found assignments to %q, want %d", names, len(want))
+ }
+ return strings.Join(lines, "\n"), names
+}
+
+func installedFile(t *testing.T, install func(string, string) error, logPath, name string) string {
+ t.Helper()
+ dir := t.TempDir()
+ if err := install(dir, logPath); err != nil {
+ t.Fatal(err)
+ }
+ content, err := os.ReadFile(filepath.Join(dir, name))
+ if err != nil {
+ t.Fatal(err)
+ }
+ return string(content)
+}
+
+// assertRendered checks that the printed config is the installed file.
+func assertRendered(t *testing.T, render func(string) (string, error), logPath, installed string) {
+ t.Helper()
+ rendered, err := render(logPath)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if rendered != installed {
+ t.Fatalf("rendered config and installed file differ:\n%s\n---\n%s", rendered, installed)
+ }
+}
+
+// stringsUnder collects the strings stored under any of keys in a decoded YAML or JSON document.
+func stringsUnder(node any, keys ...string) []string {
+ var out []string
+ switch n := node.(type) {
+ case map[string]any:
+ for key, value := range n {
+ if !slices.Contains(keys, key) {
+ out = append(out, stringsUnder(value, keys...)...)
+ continue
+ }
+ switch v := value.(type) {
+ case string:
+ out = append(out, v)
+ case []any:
+ for _, item := range v {
+ if s, ok := item.(string); ok {
+ out = append(out, s)
+ }
+ }
+ }
+ }
+ case []any:
+ for _, item := range n {
+ out = append(out, stringsUnder(item, keys...)...)
+ }
+ }
+ return out
+}
+
+func assertSourcePaths(t *testing.T, content string, want map[string][]string) {
+ t.Helper()
+ var config struct {
+ Sources map[string]struct {
+ Include []string `toml:"include"`
+ } `toml:"sources"`
+ }
+ if err := toml.Unmarshal([]byte(content), &config); err != nil {
+ t.Fatalf("invalid rendered TOML: %v", err)
+ }
+ for source, paths := range want {
+ if got := config.Sources[source].Include; !reflect.DeepEqual(got, paths) {
+ t.Errorf("%s.include = %q, want %q", source, got, paths)
+ }
+ }
+}
+
+func TestPackTOMLRenderingPreservesSiblingPathsAndArchiveGlob(t *testing.T) {
+ const content = "[sources.beacon_runtime]\ninclude = [\"{{LOG_PATH}}\", \"{{LOG_PATH}}.*\"]\n" +
+ "[sources.beacon_inventory]\ninclude = [\"{{INVENTORY_LOG_PATH}}\"]\n" +
+ "[sources.beacon_backfill]\ninclude = [\"{{BACKFILL_LOG_PATH}}\"]\n"
+ path := `/tmp/beacon"quoted\temp/runtime.jsonl`
+ pack := siempack.Pack{
+ DefaultLogPath: path,
+ FS: fstest.MapFS{"pack/vector.toml.tmpl": {Data: []byte(content)}},
+ Assets: []siempack.Asset{{Source: "pack/vector.toml.tmpl", Name: "vector.toml", TemplateLogPath: true}},
+ }
+ want := map[string][]string{
+ "beacon_runtime": {path, path + ".*"},
+ "beacon_inventory": {filepath.Join(filepath.Dir(path), "inventory_state.jsonl")},
+ "beacon_backfill": {filepath.Join(filepath.Dir(path), "cloud-backfill.jsonl")},
+ }
+ rendered, err := pack.Render("pack/vector.toml.tmpl", "")
+ if err != nil {
+ t.Fatal(err)
+ }
+ assertSourcePaths(t, rendered, want)
+ dir := t.TempDir()
+ if err := pack.Install(dir, ""); err != nil {
+ t.Fatal(err)
+ }
+ installed, err := os.ReadFile(filepath.Join(dir, "vector.toml"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if string(installed) != rendered {
+ t.Fatal("default path differs between render and install")
+ }
+}
+
+func TestOtherFormatsGetTheLogPathVerbatim(t *testing.T) {
+ path := `/tmp/beacon"quoted\temp/runtime.jsonl`
+ const content = "path={{LOG_PATH}}; inventory={{INVENTORY_LOG_PATH}}; backfill={{BACKFILL_LOG_PATH}}"
+ want := "path=" + path + "; inventory=" + siempack.InventoryLogPath(path) + "; backfill=" + siempack.BackfillLogPath(path)
+ for _, name := range []string{"README.md", "log.xml"} {
+ t.Run(name, func(t *testing.T) {
+ pack := siempack.Pack{FS: fstest.MapFS{"pack/" + name + ".tmpl": {Data: []byte(content)}}}
+ rendered, err := pack.Render("pack/"+name+".tmpl", path)
+ if err != nil || rendered != want {
+ t.Fatalf("Render = %q, %v; want %q", rendered, err, want)
+ }
+ dir := t.TempDir()
+ if err := siempack.Install(dir, []siempack.File{{Name: name, Content: content, TemplateLogPath: true}}, path); err != nil {
+ t.Fatal(err)
+ }
+ installed, err := os.ReadFile(filepath.Join(dir, name))
+ if err != nil || string(installed) != want {
+ t.Fatalf("Install = %q, %v; want %q", installed, err, want)
+ }
+ })
+ }
+ t.Run("ordinary_posix_toml", func(t *testing.T) {
+ const content = "include = [\"{{LOG_PATH}}\"]\n"
+ pack := siempack.Pack{FS: fstest.MapFS{"pack/vector.toml.tmpl": {Data: []byte(content)}}}
+ rendered, err := pack.Render("pack/vector.toml.tmpl", "/var/log/beacon-agent/runtime.jsonl")
+ if err != nil || rendered != strings.ReplaceAll(content, "{{LOG_PATH}}", "/var/log/beacon-agent/runtime.jsonl") {
+ t.Fatalf("ordinary POSIX render changed: %q, %v", rendered, err)
+ }
+ })
+}
diff --git a/cli/beacon/internal/endpoint/siempack/pack_test.go b/cli/beacon/internal/endpoint/siempack/pack_test.go
index 5d305ce0..09bb4c4b 100644
--- a/cli/beacon/internal/endpoint/siempack/pack_test.go
+++ b/cli/beacon/internal/endpoint/siempack/pack_test.go
@@ -24,7 +24,7 @@ func testPack() Pack {
{Source: "pack/README.md", Name: "README.md"},
{Source: "pack/run.sh.tmpl", Name: "run.sh", TemplateLogPath: true},
{Source: "pack/sample.jsonl", Name: "sample.jsonl"},
- {Source: "pack/conf.json.tmpl", Name: "conf.json", TemplateLogPath: true, JSONEscape: true},
+ {Source: "pack/conf.json.tmpl", Name: "conf.json", TemplateLogPath: true},
},
}
}
diff --git a/cli/beacon/internal/endpoint/siempack/siempack.go b/cli/beacon/internal/endpoint/siempack/siempack.go
index 9c6736d1..427bf9a8 100644
--- a/cli/beacon/internal/endpoint/siempack/siempack.go
+++ b/cli/beacon/internal/endpoint/siempack/siempack.go
@@ -22,7 +22,6 @@ type File struct {
Name string
Content string
TemplateLogPath bool
- JSONEscape bool
}
func ReadFile(fsys fs.FS, path string) (string, error) {
@@ -33,27 +32,72 @@ func ReadFile(fsys fs.FS, path string) (string, error) {
return string(data), nil
}
-func RenderLogPath(content, logPath string) string {
- content = strings.ReplaceAll(content, logPathToken, logPath)
- content = strings.ReplaceAll(content, inventoryLogPathToken, InventoryLogPath(logPath))
- return strings.ReplaceAll(content, backfillLogPathToken, BackfillLogPath(logPath))
-}
-
-func renderTOMLLogPath(content, logPath string) string {
+// RenderLogPath substitutes the log path tokens in content, escaped for the format of the file
+// name (a trailing .tmpl is ignored). Sibling paths are derived before escaping, so escapes never
+// reach filepath.Dir.
+func RenderLogPath(name, content, logPath string) string {
+ escape := logPathEscaper(name)
return strings.NewReplacer(
- logPathToken, TOMLEscape(logPath),
- inventoryLogPathToken, TOMLEscape(InventoryLogPath(logPath)),
- backfillLogPathToken, TOMLEscape(BackfillLogPath(logPath)),
+ logPathToken, escape(logPath),
+ inventoryLogPathToken, escape(InventoryLogPath(logPath)),
+ backfillLogPathToken, escape(BackfillLogPath(logPath)),
).Replace(content)
}
-// TOMLEscape is value as it is written inside a TOML basic string. JSON string escapes are also
-// valid there, but TOML additionally forbids a literal DEL. Sibling paths are derived before
-// encoding, so escapes never reach filepath.Dir.
-func TOMLEscape(value string) string {
+// logPathEscaper returns how a path is written where the templates of that format place their
+// tokens: TOML basic strings, YAML double-quoted scalars, JSON strings, and the default of a
+// double-quoted "${VAR:-...}" shell expansion. Any other file gets the path verbatim.
+func logPathEscaper(name string) func(string) string {
+ switch filepath.Ext(strings.TrimSuffix(name, ".tmpl")) {
+ case ".toml":
+ return tomlEscape
+ case ".yaml", ".yml":
+ return yamlEscape
+ case ".json":
+ return JSONEscapeForString
+ case ".sh":
+ return shellDefaultEscaper.Replace
+ case ".xml":
+ // Wazuh's os_xml keeps element text verbatim and decodes no entities, so escaping would change the path.
+ return verbatim
+ }
+ return verbatim
+}
+
+func verbatim(value string) string { return value }
+
+// tomlEscape takes JSON string escapes, which TOML basic strings share, plus DEL, which TOML forbids.
+func tomlEscape(value string) string {
return strings.ReplaceAll(JSONEscapeForString(value), "\x7f", `\u007f`)
}
+// yamlEscape takes JSON string escapes, which YAML double-quoted scalars share, plus the characters
+// YAML forbids or folds: DEL, the C1 controls (NEL reads as a line break), the BOM, U+FFFE and U+FFFF.
+func yamlEscape(value string) string {
+ escaped := JSONEscapeForString(value)
+ if strings.IndexFunc(escaped, yamlNeedsEscape) < 0 {
+ return escaped
+ }
+ var b strings.Builder
+ for _, r := range escaped {
+ if yamlNeedsEscape(r) {
+ fmt.Fprintf(&b, `\u%04x`, r)
+ } else {
+ b.WriteRune(r)
+ }
+ }
+ return b.String()
+}
+
+func yamlNeedsEscape(r rune) bool {
+ return (r >= 0x7f && r <= 0x9f) || r == 0xfeff || r == 0xfffe || r == 0xffff
+}
+
+// shellDefaultEscaper writes a path inside "${VAR:-...}": a backslash before \ $ ` and ", which keeps
+// them literal, and } quoted rather than backslashed, because bash 3.2 (macOS /bin/sh) keeps the
+// backslash of \} there.
+var shellDefaultEscaper = strings.NewReplacer(`\`, `\\`, `$`, `\$`, "`", "\\`", `"`, `\"`, `}`, `"}"`)
+
// BackfillLogPath is BackfillLogName beside logPath.
func BackfillLogPath(logPath string) string {
if logPath == "" {
@@ -83,14 +127,7 @@ func Install(outputDir string, files []File, logPath string) error {
for _, file := range files {
content := file.Content
if file.TemplateLogPath {
- switch {
- case file.JSONEscape:
- content = RenderLogPath(content, JSONEscapeForString(logPath))
- case isTOMLAsset(file.Name):
- content = renderTOMLLogPath(content, logPath)
- default:
- content = RenderLogPath(content, logPath)
- }
+ content = RenderLogPath(file.Name, content, logPath)
}
if err := os.WriteFile(filepath.Join(outputDir, file.Name), []byte(content), Mode(file.Name)); err != nil {
return err
@@ -114,8 +151,6 @@ type Asset struct {
Name string
// TemplateLogPath substitutes the runtime log path into the content on install.
TemplateLogPath bool
- // JSONEscape escapes the substituted log path for embedding in a JSON string.
- JSONEscape bool
}
// Pack is an embedded content pack for a single forwarding destination. It
@@ -162,7 +197,6 @@ func (p Pack) Files() ([]File, error) {
Name: a.Name,
Content: content,
TemplateLogPath: a.TemplateLogPath,
- JSONEscape: a.JSONEscape,
})
}
return files, nil
@@ -178,16 +212,7 @@ func (p Pack) Render(assetPath, logPath string) (string, error) {
if err != nil {
return "", err
}
- if isTOMLAsset(assetPath) {
- return renderTOMLLogPath(content, logPath), nil
- }
- return RenderLogPath(content, logPath), nil
-}
-
-// isTOMLAsset reports whether name, with or without its .tmpl suffix, is a TOML file, whose log
-// paths must be escaped for a basic string.
-func isTOMLAsset(name string) bool {
- return filepath.Ext(strings.TrimSuffix(name, ".tmpl")) == ".toml"
+ return RenderLogPath(assetPath, content, logPath), nil
}
// Install writes the pack files to outputDir (defaulting to DefaultOutputDir),
diff --git a/cli/beacon/internal/endpoint/siempack/smoke_scripts_test.go b/cli/beacon/internal/endpoint/siempack/smoke_scripts_test.go
index 350e0c85..ac939ba4 100644
--- a/cli/beacon/internal/endpoint/siempack/smoke_scripts_test.go
+++ b/cli/beacon/internal/endpoint/siempack/smoke_scripts_test.go
@@ -11,6 +11,7 @@ import (
"github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/gcs"
"github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/rapid7"
"github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/s3"
+ "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/siempack"
"github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/sumo"
)
@@ -51,8 +52,9 @@ func TestInstalledSmokeScriptIsThePacksSmokeTest(t *testing.T) {
if string(installed) != want {
t.Fatalf("installed %s is not the pack's smoke test for %q:\n%s", pack.script, logPath, installed)
}
- if !strings.HasPrefix(want, "#!") || !strings.Contains(want, logPath) {
- t.Fatalf("%s smoke test is not a script tailing %q:\n%s", pack.name, logPath, want)
+ // The script holds the path escaped for its double-quoted default (Windows backslashes double).
+ if escaped := siempack.RenderLogPath(pack.script, "{{LOG_PATH}}", logPath); !strings.HasPrefix(want, "#!") || !strings.Contains(want, escaped) {
+ t.Fatalf("%s smoke test is not a script tailing %q:\n%s", pack.name, escaped, want)
}
})
}
diff --git a/cli/beacon/internal/endpoint/siempack/vector_paths_test.go b/cli/beacon/internal/endpoint/siempack/vector_paths_test.go
deleted file mode 100644
index f0924034..00000000
--- a/cli/beacon/internal/endpoint/siempack/vector_paths_test.go
+++ /dev/null
@@ -1,188 +0,0 @@
-package siempack_test
-
-import (
- "encoding/json"
- "os"
- "path/filepath"
- "reflect"
- "strings"
- "testing"
- "testing/fstest"
-
- "github.com/pelletier/go-toml/v2"
-
- "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/asymptote"
- "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/cloudwatch"
- "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/falcon"
- "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/gcs"
- "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/rapid7"
- "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/s3"
- "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/sentinel"
- "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/siempack"
- "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/sumo"
-)
-
-func TestVectorPackLogPathsRoundTrip(t *testing.T) {
- paths := []struct{ name, path string }{
- {"posix", "/var/log/beacon-agent/runtime.jsonl"},
- {"windows", `C:\ProgramData\Beacon\Endpoint\logs\runtime.jsonl`},
- {"escape_sequences", `C:\new\temp\runtime.jsonl`},
- {"quotes", `/tmp/beacon"quoted/runtime.jsonl`},
- {"spaces_unicode", "/tmp/telemetry logs/日本語/runtime.jsonl"},
- {"backslash", `/tmp/beacon\temp/runtime.jsonl`},
- {"controls", "/tmp/beacon\n\r\t\b\f\x01\x1f\x7f/runtime.jsonl"},
- }
- packs := []struct {
- name string
- install func(string, string) error
- render func(string) (string, error)
- inventory bool
- backfill bool
- }{
- {name: "asymptote", install: asymptote.InstallPack, render: asymptote.VectorConfig, inventory: true, backfill: true},
- {name: "cloudwatch", install: cloudwatch.InstallPack, render: cloudwatch.ConfigSnippet},
- {name: "falcon", install: falcon.InstallPack, render: falcon.ConfigSnippet},
- {name: "gcs", install: gcs.InstallPack, inventory: true},
- {name: "rapid7", install: rapid7.InstallPack},
- {name: "s3", install: s3.InstallPack, inventory: true},
- {name: "sentinel", install: sentinel.InstallPack},
- {name: "sumo", install: sumo.InstallPack},
- }
- // Escaping is shared siempack code, so every path shape runs through the asymptote pack (which
- // renders all three sources). The other packs only need the hardest shapes to prove their
- // templates place paths in TOML basic strings.
- hardest := []struct{ name, path string }{paths[1], paths[6]}
- for _, pack := range packs {
- t.Run(pack.name, func(t *testing.T) {
- cases := hardest
- if pack.name == "asymptote" {
- cases = paths
- }
- for _, path := range cases {
- t.Run(path.name, func(t *testing.T) {
- want := map[string][]string{"beacon_runtime": {path.path}}
- if pack.inventory {
- want["beacon_inventory"] = []string{filepath.Join(filepath.Dir(path.path), "inventory_state.jsonl")}
- }
- if pack.backfill {
- want["beacon_backfill"] = []string{filepath.Join(filepath.Dir(path.path), "cloud-backfill.jsonl")}
- }
- dir := t.TempDir()
- if err := pack.install(dir, path.path); err != nil {
- t.Fatal(err)
- }
- content, err := os.ReadFile(filepath.Join(dir, "vector.toml"))
- if err != nil {
- t.Fatal(err)
- }
- assertSourcePaths(t, string(content), want)
- if pack.render != nil {
- rendered, err := pack.render(path.path)
- if err != nil {
- t.Fatal(err)
- }
- assertSourcePaths(t, rendered, want)
- if rendered != string(content) {
- t.Fatal("rendered snippet and installed config differ")
- }
- }
- })
- }
- })
- }
-}
-
-func assertSourcePaths(t *testing.T, content string, want map[string][]string) {
- t.Helper()
- var config struct {
- Sources map[string]struct {
- Include []string `toml:"include"`
- } `toml:"sources"`
- }
- if err := toml.Unmarshal([]byte(content), &config); err != nil {
- t.Fatalf("invalid rendered TOML: %v", err)
- }
- for source, paths := range want {
- if got := config.Sources[source].Include; !reflect.DeepEqual(got, paths) {
- t.Errorf("%s.include = %q, want %q", source, got, paths)
- }
- }
-}
-
-func TestPackTOMLRenderingPreservesSiblingPathsAndArchiveGlob(t *testing.T) {
- const content = "[sources.beacon_runtime]\ninclude = [\"{{LOG_PATH}}\", \"{{LOG_PATH}}.*\"]\n" +
- "[sources.beacon_inventory]\ninclude = [\"{{INVENTORY_LOG_PATH}}\"]\n" +
- "[sources.beacon_backfill]\ninclude = [\"{{BACKFILL_LOG_PATH}}\"]\n"
- path := `/tmp/beacon"quoted\temp/runtime.jsonl`
- pack := siempack.Pack{
- DefaultLogPath: path,
- FS: fstest.MapFS{"pack/vector.toml.tmpl": {Data: []byte(content)}},
- Assets: []siempack.Asset{{Source: "pack/vector.toml.tmpl", Name: "vector.toml", TemplateLogPath: true}},
- }
- want := map[string][]string{
- "beacon_runtime": {path, path + ".*"},
- "beacon_inventory": {filepath.Join(filepath.Dir(path), "inventory_state.jsonl")},
- "beacon_backfill": {filepath.Join(filepath.Dir(path), "cloud-backfill.jsonl")},
- }
- rendered, err := pack.Render("pack/vector.toml.tmpl", "")
- if err != nil {
- t.Fatal(err)
- }
- assertSourcePaths(t, rendered, want)
- dir := t.TempDir()
- if err := pack.Install(dir, ""); err != nil {
- t.Fatal(err)
- }
- installed, err := os.ReadFile(filepath.Join(dir, "vector.toml"))
- if err != nil {
- t.Fatal(err)
- }
- if string(installed) != rendered {
- t.Fatal("default path differs between render and install")
- }
-}
-
-func TestPathRenderingPreservesOtherFormats(t *testing.T) {
- path := `/tmp/beacon"quoted\temp/runtime.jsonl`
- for _, name := range []string{"run.sh", "conf.yaml", "README.md", "log.xml"} {
- t.Run(name, func(t *testing.T) {
- const content = "path={{LOG_PATH}}; inventory={{INVENTORY_LOG_PATH}}; backfill={{BACKFILL_LOG_PATH}}"
- want := siempack.RenderLogPath(content, path)
- pack := siempack.Pack{FS: fstest.MapFS{"pack/" + name + ".tmpl": {Data: []byte(content)}}}
- rendered, err := pack.Render("pack/"+name+".tmpl", path)
- if err != nil || rendered != want {
- t.Fatalf("Render = %q, %v; want %q", rendered, err, want)
- }
- dir := t.TempDir()
- if err := siempack.Install(dir, []siempack.File{{Name: name, Content: content, TemplateLogPath: true}}, path); err != nil {
- t.Fatal(err)
- }
- installed, err := os.ReadFile(filepath.Join(dir, name))
- if err != nil || string(installed) != want {
- t.Fatalf("Install = %q, %v; want %q", installed, err, want)
- }
- })
- }
- t.Run("json", func(t *testing.T) {
- dir := t.TempDir()
- if err := siempack.Install(dir, []siempack.File{{Name: "conf.json", Content: `{"path":"{{LOG_PATH}}"}`, TemplateLogPath: true, JSONEscape: true}}, path); err != nil {
- t.Fatal(err)
- }
- content, err := os.ReadFile(filepath.Join(dir, "conf.json"))
- if err != nil {
- t.Fatal(err)
- }
- var got struct{ Path string }
- if err := json.Unmarshal(content, &got); err != nil || got.Path != path {
- t.Fatalf("JSON path = %q, %v; want %q", got.Path, err, path)
- }
- })
- t.Run("ordinary_posix", func(t *testing.T) {
- const content = "include = [\"{{LOG_PATH}}\"]\n"
- pack := siempack.Pack{FS: fstest.MapFS{"pack/vector.toml.tmpl": {Data: []byte(content)}}}
- rendered, err := pack.Render("pack/vector.toml.tmpl", "/var/log/beacon-agent/runtime.jsonl")
- if err != nil || rendered != strings.ReplaceAll(content, "{{LOG_PATH}}", "/var/log/beacon-agent/runtime.jsonl") {
- t.Fatalf("ordinary POSIX render changed: %q, %v", rendered, err)
- }
- })
-}
diff --git a/cli/beacon/internal/endpoint/wazuh/wazuh.go b/cli/beacon/internal/endpoint/wazuh/wazuh.go
index c72593f9..4ad721ef 100644
--- a/cli/beacon/internal/endpoint/wazuh/wazuh.go
+++ b/cli/beacon/internal/endpoint/wazuh/wazuh.go
@@ -2,10 +2,13 @@ package wazuh
import (
"embed"
- endpointconfig "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/config"
+ "fmt"
"os"
"path/filepath"
"strings"
+
+ endpointconfig "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/config"
+ "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/siempack"
)
//go:embed pack/*
@@ -20,38 +23,38 @@ type File struct {
Content string
}
-func Files() []File {
- localfile := LocalfileSnippet(DefaultLogPath)
- rules := mustRead("pack/beacon-rules.xml")
- sample := mustRead("pack/sample-event.jsonl")
- readme := mustRead("pack/README.md")
- dashboardColumns := mustRead("pack/apply-dashboard-default-columns.sh")
+func files(localfile string) []File {
return []File{
{Name: "ossec-localfile.xml", Content: localfile},
- {Name: "beacon-rules.xml", Content: rules},
- {Name: "sample-event.jsonl", Content: sample},
- {Name: "apply-dashboard-default-columns.sh", Content: dashboardColumns},
- {Name: "README.md", Content: readme},
+ {Name: "beacon-rules.xml", Content: mustRead("pack/beacon-rules.xml")},
+ {Name: "sample-event.jsonl", Content: mustRead("pack/sample-event.jsonl")},
+ {Name: "apply-dashboard-default-columns.sh", Content: mustRead("pack/apply-dashboard-default-columns.sh")},
+ {Name: "README.md", Content: mustRead("pack/README.md")},
}
}
-func LocalfileSnippet(logPath string) string {
+// LocalfileSnippet returns the ossec.conf <localfile> block for logPath. Wazuh reads the path as
+// written and has no way to escape a '<' in it, so such a path is refused.
+func LocalfileSnippet(logPath string) (string, error) {
if logPath == "" {
logPath = DefaultLogPath
}
- return strings.ReplaceAll(mustRead("pack/ossec-localfile.xml"), "{{LOG_PATH}}", logPath)
+ if strings.Contains(logPath, "<") {
+ return "", fmt.Errorf("wazuh cannot read a log path containing '<': %s", logPath)
+ }
+ return siempack.RenderLogPath("ossec-localfile.xml", mustRead("pack/ossec-localfile.xml"), logPath), nil
}
func InstallPack(outputDir, logPath string) error {
+ localfile, err := LocalfileSnippet(logPath)
+ if err != nil {
+ return err
+ }
if err := os.MkdirAll(outputDir, 0755); err != nil {
return err
}
- for _, file := range Files() {
- content := file.Content
- if file.Name == "ossec-localfile.xml" {
- content = LocalfileSnippet(logPath)
- }
- if err := os.WriteFile(filepath.Join(outputDir, file.Name), []byte(content), 0644); err != nil {
+ for _, file := range files(localfile) {
+ if err := os.WriteFile(filepath.Join(outputDir, file.Name), []byte(file.Content), 0644); err != nil {
return err
}
}
diff --git a/cli/beacon/internal/endpoint/wazuh/wazuh_test.go b/cli/beacon/internal/endpoint/wazuh/wazuh_test.go
index b549494e..db51e3f6 100644
--- a/cli/beacon/internal/endpoint/wazuh/wazuh_test.go
+++ b/cli/beacon/internal/endpoint/wazuh/wazuh_test.go
@@ -8,7 +8,10 @@ import (
)
func TestLocalfileSnippetUsesConfiguredPath(t *testing.T) {
- got := LocalfileSnippet("/tmp/beacon/runtime.jsonl")
+ got, err := LocalfileSnippet("/tmp/beacon/runtime.jsonl")
+ if err != nil {
+ t.Fatal(err)
+ }
if !strings.Contains(got, "/tmp/beacon/runtime.jsonl") {
t.Fatalf("snippet did not include configured path: %s", got)
}
@@ -17,6 +20,21 @@ func TestLocalfileSnippetUsesConfiguredPath(t *testing.T) {
}
}
+// Wazuh's XML parser reads a '<' in the location as the start of an element and has no escape for it.
+func TestLogPathWithAngleBracketIsRefused(t *testing.T) {
+ const path = "/tmp/a<b/runtime.jsonl"
+ if got, err := LocalfileSnippet(path); err == nil {
+ t.Fatalf("LocalfileSnippet accepted %q: %s", path, got)
+ }
+ dir := filepath.Join(t.TempDir(), "pack")
+ if err := InstallPack(dir, path); err == nil {
+ t.Fatalf("InstallPack accepted %q", path)
+ }
+ if _, err := os.Stat(dir); !os.IsNotExist(err) {
+ t.Fatalf("InstallPack wrote %s before refusing the path: %v", dir, err)
+ }
+}
+
func TestInstallPackWritesExpectedFiles(t *testing.T) {
dir := t.TempDir()
if err := InstallPack(dir, "/tmp/beacon/runtime.jsonl"); err != nil {