The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu
Go agent-ai-cliagent-harnessesagent-securitybrowser-extensionciclaude-codecloudcodexcursordetection-engineeringjevknowledge-basememorymobile-device-managementobservabilitysecuritysecurity-information-and-event-managementsecurity-toolstelemetrytraces
fix(siem): escape log paths in the datadog and elastic yaml configs (#1078)
Rudy Mizrahi Celekli99a7e3ab8621
diff --git a/cli/beacon/cmd/endpoint_destinations.go b/cli/beacon/cmd/endpoint_destinations.goindex 3356a95e..28f4462c 100644--- a/cli/beacon/cmd/endpoint_destinations.go+++ b/cli/beacon/cmd/endpoint_destinations.go@@ -104,10 +104,8 @@ var siemDestinations = []siemDestination{name: "wazuh",short: "Manage Wazuh integration content",printConfig: &destPrintConfig{- short: "Print a Wazuh localfile snippet",- render: func(cfg endpointconfig.Config) (string, error) {- return wazuh.LocalfileSnippet(cfg.LogPath), nil- },+ short: "Print a Wazuh localfile snippet",+ render: func(cfg endpointconfig.Config) (string, error) { return wazuh.LocalfileSnippet(cfg.LogPath) },},installPack: &destInstallPack{short: "Write Wazuh rules and config snippets to a directory",@@ -121,8 +119,12 @@ var siemDestinations = []siemDestination{message: "Beacon endpoint Wazuh validation event",print: func(cfg endpointconfig.Config) {fmt.Println("Expected Wazuh fields: vendor=beacon product=endpoint-agent event.kind=agent_runtime")- fmt.Println("Wazuh localfile snippet:")- fmt.Print(wazuh.LocalfileSnippet(cfg.LogPath))+ if snippet, err := wazuh.LocalfileSnippet(cfg.LogPath); err != nil {+ fmt.Println(err)+ } else {+ fmt.Println("Wazuh localfile snippet:")+ fmt.Print(snippet)+ }fmt.Println("Expected base rule: 100500")},},diff --git a/cli/beacon/cmd/endpoint_destinations_characterization_test.go b/cli/beacon/cmd/endpoint_destinations_characterization_test.goindex 9f38581e..cf1d9732 100644--- a/cli/beacon/cmd/endpoint_destinations_characterization_test.go+++ b/cli/beacon/cmd/endpoint_destinations_characterization_test.go@@ -8,8 +8,6 @@ import ("testing""github.com/spf13/cobra"-- "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/siempack")// These tests drive every forwarding-destination command (print-config /@@ -52,8 +50,8 @@ func TestDestinationCommandsCharacterization(t *testing.T) {if err != nil {t.Fatalf("%s print-config returned error: %v", tc.group, err)}- // TOML configs write the path escaped; on Windows that doubles its backslashes.- if !strings.Contains(out, logPath) && !strings.Contains(out, siempack.TOMLEscape(logPath)) {+ // Quoted configs write the path escaped; on Windows that doubles its backslashes.+ if !strings.Contains(out, logPath) && !strings.Contains(out, strings.ReplaceAll(logPath, `\`, `\\`)) {t.Fatalf("%s print-config did not wire --log-path %q through to output:\n%s", tc.group, logPath, out)}})diff --git a/cli/beacon/internal/endpoint/elastic/elastic.go b/cli/beacon/internal/endpoint/elastic/elastic.goindex 9ead0000..90e0b3dc 100644--- a/cli/beacon/internal/endpoint/elastic/elastic.go+++ b/cli/beacon/internal/endpoint/elastic/elastic.go@@ -7,7 +7,6 @@ import ("io/fs""os""path/filepath"- "strings""github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/siempack")@@ -56,7 +55,7 @@ func inputSnippetFromFS(fsys fs.FS, logPath string) (string, error) {if err != nil {return "", fmt.Errorf("elastic pack asset %q: %w", "pack/filebeat.yml.tmpl", err)}- return strings.ReplaceAll(content, "{{LOG_PATH}}", logPath), nil+ return siempack.RenderLogPath("filebeat.yml", content, logPath), nil}// standaloneAgentConfigFromFS renders the elastic-agent standalone config using the supplied FS.@@ -68,7 +67,7 @@ func standaloneAgentConfigFromFS(fsys fs.FS, logPath string) (string, error) {if err != nil {return "", fmt.Errorf("elastic pack asset %q: %w", "pack/elastic-agent-standalone.yml.tmpl", err)}- return strings.ReplaceAll(content, "{{LOG_PATH}}", logPath), nil+ return siempack.RenderLogPath("elastic-agent-standalone.yml", content, logPath), nil}// filesFromFS builds the full file list from the supplied FS, propagating anydiff --git a/cli/beacon/internal/endpoint/elastic/elastic_test.go b/cli/beacon/internal/endpoint/elastic/elastic_test.goindex 32a232aa..04bc66c6 100644--- a/cli/beacon/internal/endpoint/elastic/elastic_test.go+++ b/cli/beacon/internal/endpoint/elastic/elastic_test.go@@ -160,7 +160,11 @@ func TestInputSnippet_DefaultLogPath(t *testing.T) {if err != nil {t.Fatalf("InputSnippet with empty path returned error: %v", err)}- if !strings.Contains(got, DefaultLogPath) {+ want, err := InputSnippet(DefaultLogPath)+ if err != nil {+ t.Fatal(err)+ }+ if got != want {t.Fatalf("InputSnippet with empty logPath should use DefaultLogPath %q, got: %s", DefaultLogPath, got)}}diff --git a/cli/beacon/internal/endpoint/sentinel/sentinel.go b/cli/beacon/internal/endpoint/sentinel/sentinel.goindex dee68e61..1432741f 100644--- a/cli/beacon/internal/endpoint/sentinel/sentinel.go+++ b/cli/beacon/internal/endpoint/sentinel/sentinel.go@@ -25,7 +25,7 @@ func Files() []siempack.File {{Name: "README.md", Content: mustRead("pack/README.md")},{Name: "dcr-transform.kql", Content: DCRTransform()},{Name: "table-schema.json", Content: mustRead("pack/table-schema.json")},- {Name: "dcr-template.json", Content: renderDCRTemplate("pack/dcr-template.json"), TemplateLogPath: true, JSONEscape: true},+ {Name: "dcr-template.json", Content: renderDCRTemplate("pack/dcr-template.json"), TemplateLogPath: true},// The Logs Ingestion API path, for hosts without Azure Monitor Agent: Vector sends each line// as RawData to a DCR that runs the same transform, so both paths fill the same table.{Name: "vector.toml", Content: mustRead("pack/vector.toml.tmpl"), TemplateLogPath: true},diff --git a/cli/beacon/internal/endpoint/siempack/log_paths_test.go b/cli/beacon/internal/endpoint/siempack/log_paths_test.gonew file mode 100644index 00000000..ccd0df40--- /dev/null+++ b/cli/beacon/internal/endpoint/siempack/log_paths_test.go@@ -0,0 +1,358 @@+package siempack_test++import (+ "bytes"+ "encoding/json"+ "os"+ "os/exec"+ "path/filepath"+ "reflect"+ "runtime"+ "slices"+ "strings"+ "testing"+ "testing/fstest"++ "github.com/pelletier/go-toml/v2"+ "gopkg.in/yaml.v3"+ "mvdan.cc/sh/v3/syntax"++ "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/asymptote"+ "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/cloudwatch"+ "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/datadog"+ "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/elastic"+ "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/falcon"+ "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/gcs"+ "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/rapid7"+ "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/s3"+ "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/sentinel"+ "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/siempack"+ "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/sumo"+)++// logPaths are the path shapes every pack must carry through unchanged: the system default on each+// platform, and paths full of the syntax each format quotes.+var logPaths = []struct{ name, path string }{+ {"posix", "/var/log/beacon-agent/runtime.jsonl"},+ {"windows", `C:\Users\x\runtime.jsonl`},+ {"escape_sequences", `C:\new\temp\runtime.jsonl`},+ {"syntax", "/tmp/a: b/\"q\"/$(touch pwned)/`x`/}{/runtime.jsonl"},+ {"spaces_unicode", "/tmp/telemetry logs/日本語/runtime.jsonl"},+ {"controls", "/tmp/beacon\n\r\t\b\f\x01\x1f\x7f\u0085\u009f\ufeff\uffff/runtime.jsonl"},+}++func TestTOMLLogPathsRoundTrip(t *testing.T) {+ packs := []struct {+ name string+ install func(string, string) error+ render func(string) (string, error)+ inventory bool+ backfill bool+ }{+ {name: "asymptote", install: asymptote.InstallPack, render: asymptote.VectorConfig, inventory: true, backfill: true},+ {name: "cloudwatch", install: cloudwatch.InstallPack, render: cloudwatch.ConfigSnippet},+ {name: "falcon", install: falcon.InstallPack, render: falcon.ConfigSnippet},+ {name: "gcs", install: gcs.InstallPack, inventory: true},+ {name: "rapid7", install: rapid7.InstallPack},+ {name: "s3", install: s3.InstallPack, inventory: true},+ {name: "sentinel", install: sentinel.InstallPack},+ {name: "sumo", install: sumo.InstallPack},+ }+ for _, pack := range packs {+ for _, path := range logPaths {+ t.Run(pack.name+"/"+path.name, func(t *testing.T) {+ want := map[string][]string{"beacon_runtime": {path.path}}+ if pack.inventory {+ want["beacon_inventory"] = []string{siempack.InventoryLogPath(path.path)}+ }+ if pack.backfill {+ want["beacon_backfill"] = []string{siempack.BackfillLogPath(path.path)}+ }+ installed := installedFile(t, pack.install, path.path, "vector.toml")+ assertSourcePaths(t, installed, want)+ if pack.render != nil {+ assertRendered(t, pack.render, path.path, installed)+ }+ })+ }+ }+}++func TestYAMLLogPathsRoundTrip(t *testing.T) {+ files := []struct {+ pack string+ install func(string, string) error+ name string+ render func(string) (string, error)+ }{+ {"datadog", datadog.InstallPack, "conf.yaml", datadog.ConfigSnippet},+ {"elastic", elastic.InstallPack, "filebeat.yml", elastic.InputSnippet},+ {"elastic", elastic.InstallPack, "elastic-agent-standalone.yml", nil},+ }+ for _, file := range files {+ for _, path := range logPaths {+ t.Run(file.name+"/"+path.name, func(t *testing.T) {+ installed := installedFile(t, file.install, path.path, file.name)+ var doc any+ if err := yaml.Unmarshal([]byte(installed), &doc); err != nil {+ t.Fatalf("invalid rendered YAML: %v\n%s", err, installed)+ }+ if got := stringsUnder(doc, "path", "paths"); !reflect.DeepEqual(got, []string{path.path}) {+ t.Fatalf("log paths = %q, want %q", got, path.path)+ }+ if file.render != nil {+ assertRendered(t, file.render, path.path, installed)+ }+ })+ }+ }+}++func TestJSONLogPathsRoundTrip(t *testing.T) {+ for _, path := range logPaths {+ t.Run(path.name, func(t *testing.T) {+ installed := installedFile(t, sentinel.InstallPack, path.path, "dcr-template.json")+ var doc any+ if err := json.Unmarshal([]byte(installed), &doc); err != nil {+ t.Fatalf("invalid rendered JSON: %v", err)+ }+ if got := stringsUnder(doc, "filePatterns"); !reflect.DeepEqual(got, []string{path.path}) {+ t.Fatalf("filePatterns = %q, want %q", got, path.path)+ }+ })+ }+}++// The smoke-test scripts take the log path as the default of "${BEACON_LOG:-...}". Each script must+// parse, and its defaults must evaluate to the paths without running anything inside them.+func TestShellLogPathsRoundTrip(t *testing.T) {+ // The smoke-test scripts run on macOS and Linux hosts. Git for Windows' sh drops carriage+ // returns from its input, so it cannot read these values back exactly.+ if runtime.GOOS == "windows" {+ t.Skip("the smoke-test scripts are POSIX shell scripts for macOS and Linux")+ }+ sh, err := exec.LookPath("sh")+ if err != nil {+ t.Skip("no POSIX shell to run the smoke-test scripts")+ }+ scripts := []struct {+ pack string+ install func(string, string) error+ name string+ render func(string) (string, error)+ inventory bool+ }{+ {"asymptote", asymptote.InstallPack, "asymptote-ingest-smoke-test.sh", asymptote.IngestSmokeTest, true},+ {"falcon", falcon.InstallPack, "falcon-hec-smoke-test.sh", falcon.SmokeTest, false},+ {"gcs", gcs.InstallPack, "gcs-upload-smoke-test.sh", gcs.UploadSmokeTest, true},+ {"rapid7", rapid7.InstallPack, "rapid7-upload-smoke-test.sh", rapid7.UploadSmokeTest, false},+ {"s3", s3.InstallPack, "s3-upload-smoke-test.sh", s3.UploadSmokeTest, true},+ {"sumo", sumo.InstallPack, "sumo-upload-smoke-test.sh", sumo.UploadSmokeTest, false},+ }+ for _, script := range scripts {+ for _, path := range logPaths {+ t.Run(script.pack+"/"+path.name, func(t *testing.T) {+ dir := t.TempDir()+ if err := script.install(dir, path.path); err != nil {+ t.Fatal(err)+ }+ scriptPath := filepath.Join(dir, script.name)+ if out, err := exec.Command(sh, "-n", scriptPath).CombinedOutput(); err != nil {+ t.Fatalf("sh -n: %v\n%s", err, out)+ }+ content, err := os.ReadFile(scriptPath)+ if err != nil {+ t.Fatal(err)+ }+ assertRendered(t, script.render, path.path, string(content))++ want := map[string]string{"BEACON_LOG": path.path}+ if script.inventory {+ want["BEACON_INVENTORY_LOG"] = siempack.InventoryLogPath(path.path)+ }+ // Run only the default assignments: the rest of the script needs credentials.+ defaults, names := defaultAssignments(t, string(content), want)+ for _, name := range names {+ defaults += "\nprintf '%s\\0' \"$" + name + "\""+ }+ cwd := t.TempDir()+ cmd := exec.Command(sh, "-c", defaults)+ cmd.Dir = cwd+ cmd.Env = []string{"PATH=" + os.Getenv("PATH")}+ var stdout, stderr bytes.Buffer+ cmd.Stdout, cmd.Stderr = &stdout, &stderr+ if err := cmd.Run(); err != nil || stderr.Len() > 0 {+ t.Fatalf("defaults failed: %v\n%s\n%s", err, stderr.String(), defaults)+ }+ got := strings.Split(strings.TrimSuffix(stdout.String(), "\x00"), "\x00")+ for i, name := range names {+ if got[i] != want[name] {+ t.Errorf("%s = %q, want %q", name, got[i], want[name])+ }+ }+ if entries, _ := os.ReadDir(cwd); len(entries) > 0 {+ t.Fatalf("evaluating the defaults ran a command that created %s", entries[0].Name())+ }+ })+ }+ }+}++// defaultAssignments returns the script's top-level assignments to the variables in want, in order.+func defaultAssignments(t *testing.T, content string, want map[string]string) (string, []string) {+ t.Helper()+ file, err := syntax.NewParser().Parse(strings.NewReader(content), "")+ if err != nil {+ t.Fatal(err)+ }+ var lines, names []string+ for _, stmt := range file.Stmts {+ call, ok := stmt.Cmd.(*syntax.CallExpr)+ if !ok || len(call.Args) != 0 || len(call.Assigns) != 1 {+ continue+ }+ if name := call.Assigns[0].Name.Value; want[name] != "" {+ lines = append(lines, content[stmt.Pos().Offset():stmt.End().Offset()])+ names = append(names, name)+ }+ }+ if len(names) != len(want) {+ t.Fatalf("found assignments to %q, want %d", names, len(want))+ }+ return strings.Join(lines, "\n"), names+}++func installedFile(t *testing.T, install func(string, string) error, logPath, name string) string {+ t.Helper()+ dir := t.TempDir()+ if err := install(dir, logPath); err != nil {+ t.Fatal(err)+ }+ content, err := os.ReadFile(filepath.Join(dir, name))+ if err != nil {+ t.Fatal(err)+ }+ return string(content)+}++// assertRendered checks that the printed config is the installed file.+func assertRendered(t *testing.T, render func(string) (string, error), logPath, installed string) {+ t.Helper()+ rendered, err := render(logPath)+ if err != nil {+ t.Fatal(err)+ }+ if rendered != installed {+ t.Fatalf("rendered config and installed file differ:\n%s\n---\n%s", rendered, installed)+ }+}++// stringsUnder collects the strings stored under any of keys in a decoded YAML or JSON document.+func stringsUnder(node any, keys ...string) []string {+ var out []string+ switch n := node.(type) {+ case map[string]any:+ for key, value := range n {+ if !slices.Contains(keys, key) {+ out = append(out, stringsUnder(value, keys...)...)+ continue+ }+ switch v := value.(type) {+ case string:+ out = append(out, v)+ case []any:+ for _, item := range v {+ if s, ok := item.(string); ok {+ out = append(out, s)+ }+ }+ }+ }+ case []any:+ for _, item := range n {+ out = append(out, stringsUnder(item, keys...)...)+ }+ }+ return out+}++func assertSourcePaths(t *testing.T, content string, want map[string][]string) {+ t.Helper()+ var config struct {+ Sources map[string]struct {+ Include []string `toml:"include"`+ } `toml:"sources"`+ }+ if err := toml.Unmarshal([]byte(content), &config); err != nil {+ t.Fatalf("invalid rendered TOML: %v", err)+ }+ for source, paths := range want {+ if got := config.Sources[source].Include; !reflect.DeepEqual(got, paths) {+ t.Errorf("%s.include = %q, want %q", source, got, paths)+ }+ }+}++func TestPackTOMLRenderingPreservesSiblingPathsAndArchiveGlob(t *testing.T) {+ const content = "[sources.beacon_runtime]\ninclude = [\"{{LOG_PATH}}\", \"{{LOG_PATH}}.*\"]\n" ++ "[sources.beacon_inventory]\ninclude = [\"{{INVENTORY_LOG_PATH}}\"]\n" ++ "[sources.beacon_backfill]\ninclude = [\"{{BACKFILL_LOG_PATH}}\"]\n"+ path := `/tmp/beacon"quoted\temp/runtime.jsonl`+ pack := siempack.Pack{+ DefaultLogPath: path,+ FS: fstest.MapFS{"pack/vector.toml.tmpl": {Data: []byte(content)}},+ Assets: []siempack.Asset{{Source: "pack/vector.toml.tmpl", Name: "vector.toml", TemplateLogPath: true}},+ }+ want := map[string][]string{+ "beacon_runtime": {path, path + ".*"},+ "beacon_inventory": {filepath.Join(filepath.Dir(path), "inventory_state.jsonl")},+ "beacon_backfill": {filepath.Join(filepath.Dir(path), "cloud-backfill.jsonl")},+ }+ rendered, err := pack.Render("pack/vector.toml.tmpl", "")+ if err != nil {+ t.Fatal(err)+ }+ assertSourcePaths(t, rendered, want)+ dir := t.TempDir()+ if err := pack.Install(dir, ""); err != nil {+ t.Fatal(err)+ }+ installed, err := os.ReadFile(filepath.Join(dir, "vector.toml"))+ if err != nil {+ t.Fatal(err)+ }+ if string(installed) != rendered {+ t.Fatal("default path differs between render and install")+ }+}++func TestOtherFormatsGetTheLogPathVerbatim(t *testing.T) {+ path := `/tmp/beacon"quoted\temp/runtime.jsonl`+ const content = "path={{LOG_PATH}}; inventory={{INVENTORY_LOG_PATH}}; backfill={{BACKFILL_LOG_PATH}}"+ want := "path=" + path + "; inventory=" + siempack.InventoryLogPath(path) + "; backfill=" + siempack.BackfillLogPath(path)+ for _, name := range []string{"README.md", "log.xml"} {+ t.Run(name, func(t *testing.T) {+ pack := siempack.Pack{FS: fstest.MapFS{"pack/" + name + ".tmpl": {Data: []byte(content)}}}+ rendered, err := pack.Render("pack/"+name+".tmpl", path)+ if err != nil || rendered != want {+ t.Fatalf("Render = %q, %v; want %q", rendered, err, want)+ }+ dir := t.TempDir()+ if err := siempack.Install(dir, []siempack.File{{Name: name, Content: content, TemplateLogPath: true}}, path); err != nil {+ t.Fatal(err)+ }+ installed, err := os.ReadFile(filepath.Join(dir, name))+ if err != nil || string(installed) != want {+ t.Fatalf("Install = %q, %v; want %q", installed, err, want)+ }+ })+ }+ t.Run("ordinary_posix_toml", func(t *testing.T) {+ const content = "include = [\"{{LOG_PATH}}\"]\n"+ pack := siempack.Pack{FS: fstest.MapFS{"pack/vector.toml.tmpl": {Data: []byte(content)}}}+ rendered, err := pack.Render("pack/vector.toml.tmpl", "/var/log/beacon-agent/runtime.jsonl")+ if err != nil || rendered != strings.ReplaceAll(content, "{{LOG_PATH}}", "/var/log/beacon-agent/runtime.jsonl") {+ t.Fatalf("ordinary POSIX render changed: %q, %v", rendered, err)+ }+ })+}diff --git a/cli/beacon/internal/endpoint/siempack/pack_test.go b/cli/beacon/internal/endpoint/siempack/pack_test.goindex 5d305ce0..09bb4c4b 100644--- a/cli/beacon/internal/endpoint/siempack/pack_test.go+++ b/cli/beacon/internal/endpoint/siempack/pack_test.go@@ -24,7 +24,7 @@ func testPack() Pack {{Source: "pack/README.md", Name: "README.md"},{Source: "pack/run.sh.tmpl", Name: "run.sh", TemplateLogPath: true},{Source: "pack/sample.jsonl", Name: "sample.jsonl"},- {Source: "pack/conf.json.tmpl", Name: "conf.json", TemplateLogPath: true, JSONEscape: true},+ {Source: "pack/conf.json.tmpl", Name: "conf.json", TemplateLogPath: true},},}}diff --git a/cli/beacon/internal/endpoint/siempack/siempack.go b/cli/beacon/internal/endpoint/siempack/siempack.goindex 9c6736d1..427bf9a8 100644--- a/cli/beacon/internal/endpoint/siempack/siempack.go+++ b/cli/beacon/internal/endpoint/siempack/siempack.go@@ -22,7 +22,6 @@ type File struct {Name stringContent stringTemplateLogPath bool- JSONEscape bool}func ReadFile(fsys fs.FS, path string) (string, error) {@@ -33,27 +32,72 @@ func ReadFile(fsys fs.FS, path string) (string, error) {return string(data), nil}-func RenderLogPath(content, logPath string) string {- content = strings.ReplaceAll(content, logPathToken, logPath)- content = strings.ReplaceAll(content, inventoryLogPathToken, InventoryLogPath(logPath))- return strings.ReplaceAll(content, backfillLogPathToken, BackfillLogPath(logPath))-}--func renderTOMLLogPath(content, logPath string) string {+// RenderLogPath substitutes the log path tokens in content, escaped for the format of the file+// name (a trailing .tmpl is ignored). Sibling paths are derived before escaping, so escapes never+// reach filepath.Dir.+func RenderLogPath(name, content, logPath string) string {+ escape := logPathEscaper(name)return strings.NewReplacer(- logPathToken, TOMLEscape(logPath),- inventoryLogPathToken, TOMLEscape(InventoryLogPath(logPath)),- backfillLogPathToken, TOMLEscape(BackfillLogPath(logPath)),+ logPathToken, escape(logPath),+ inventoryLogPathToken, escape(InventoryLogPath(logPath)),+ backfillLogPathToken, escape(BackfillLogPath(logPath)),).Replace(content)}-// TOMLEscape is value as it is written inside a TOML basic string. JSON string escapes are also-// valid there, but TOML additionally forbids a literal DEL. Sibling paths are derived before-// encoding, so escapes never reach filepath.Dir.-func TOMLEscape(value string) string {+// logPathEscaper returns how a path is written where the templates of that format place their+// tokens: TOML basic strings, YAML double-quoted scalars, JSON strings, and the default of a+// double-quoted "${VAR:-...}" shell expansion. Any other file gets the path verbatim.+func logPathEscaper(name string) func(string) string {+ switch filepath.Ext(strings.TrimSuffix(name, ".tmpl")) {+ case ".toml":+ return tomlEscape+ case ".yaml", ".yml":+ return yamlEscape+ case ".json":+ return JSONEscapeForString+ case ".sh":+ return shellDefaultEscaper.Replace+ case ".xml":+ // Wazuh's os_xml keeps element text verbatim and decodes no entities, so escaping would change the path.+ return verbatim+ }+ return verbatim+}++func verbatim(value string) string { return value }++// tomlEscape takes JSON string escapes, which TOML basic strings share, plus DEL, which TOML forbids.+func tomlEscape(value string) string {return strings.ReplaceAll(JSONEscapeForString(value), "\x7f", `\u007f`)}+// yamlEscape takes JSON string escapes, which YAML double-quoted scalars share, plus the characters+// YAML forbids or folds: DEL, the C1 controls (NEL reads as a line break), the BOM, U+FFFE and U+FFFF.+func yamlEscape(value string) string {+ escaped := JSONEscapeForString(value)+ if strings.IndexFunc(escaped, yamlNeedsEscape) < 0 {+ return escaped+ }+ var b strings.Builder+ for _, r := range escaped {+ if yamlNeedsEscape(r) {+ fmt.Fprintf(&b, `\u%04x`, r)+ } else {+ b.WriteRune(r)+ }+ }+ return b.String()+}++func yamlNeedsEscape(r rune) bool {+ return (r >= 0x7f && r <= 0x9f) || r == 0xfeff || r == 0xfffe || r == 0xffff+}++// shellDefaultEscaper writes a path inside "${VAR:-...}": a backslash before \ $ ` and ", which keeps+// them literal, and } quoted rather than backslashed, because bash 3.2 (macOS /bin/sh) keeps the+// backslash of \} there.+var shellDefaultEscaper = strings.NewReplacer(`\`, `\\`, `$`, `\$`, "`", "\\`", `"`, `\"`, `}`, `"}"`)+// BackfillLogPath is BackfillLogName beside logPath.func BackfillLogPath(logPath string) string {if logPath == "" {@@ -83,14 +127,7 @@ func Install(outputDir string, files []File, logPath string) error {for _, file := range files {content := file.Contentif file.TemplateLogPath {- switch {- case file.JSONEscape:- content = RenderLogPath(content, JSONEscapeForString(logPath))- case isTOMLAsset(file.Name):- content = renderTOMLLogPath(content, logPath)- default:- content = RenderLogPath(content, logPath)- }+ content = RenderLogPath(file.Name, content, logPath)}if err := os.WriteFile(filepath.Join(outputDir, file.Name), []byte(content), Mode(file.Name)); err != nil {return err@@ -114,8 +151,6 @@ type Asset struct {Name string// TemplateLogPath substitutes the runtime log path into the content on install.TemplateLogPath bool- // JSONEscape escapes the substituted log path for embedding in a JSON string.- JSONEscape bool}// Pack is an embedded content pack for a single forwarding destination. It@@ -162,7 +197,6 @@ func (p Pack) Files() ([]File, error) {Name: a.Name,Content: content,TemplateLogPath: a.TemplateLogPath,- JSONEscape: a.JSONEscape,})}return files, nil@@ -178,16 +212,7 @@ func (p Pack) Render(assetPath, logPath string) (string, error) {if err != nil {return "", err}- if isTOMLAsset(assetPath) {- return renderTOMLLogPath(content, logPath), nil- }- return RenderLogPath(content, logPath), nil-}--// isTOMLAsset reports whether name, with or without its .tmpl suffix, is a TOML file, whose log-// paths must be escaped for a basic string.-func isTOMLAsset(name string) bool {- return filepath.Ext(strings.TrimSuffix(name, ".tmpl")) == ".toml"+ return RenderLogPath(assetPath, content, logPath), nil}// Install writes the pack files to outputDir (defaulting to DefaultOutputDir),diff --git a/cli/beacon/internal/endpoint/siempack/smoke_scripts_test.go b/cli/beacon/internal/endpoint/siempack/smoke_scripts_test.goindex 350e0c85..ac939ba4 100644--- a/cli/beacon/internal/endpoint/siempack/smoke_scripts_test.go+++ b/cli/beacon/internal/endpoint/siempack/smoke_scripts_test.go@@ -11,6 +11,7 @@ import ("github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/gcs""github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/rapid7""github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/s3"+ "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/siempack""github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/sumo")@@ -51,8 +52,9 @@ func TestInstalledSmokeScriptIsThePacksSmokeTest(t *testing.T) {if string(installed) != want {t.Fatalf("installed %s is not the pack's smoke test for %q:\n%s", pack.script, logPath, installed)}- if !strings.HasPrefix(want, "#!") || !strings.Contains(want, logPath) {- t.Fatalf("%s smoke test is not a script tailing %q:\n%s", pack.name, logPath, want)+ // The script holds the path escaped for its double-quoted default (Windows backslashes double).+ if escaped := siempack.RenderLogPath(pack.script, "{{LOG_PATH}}", logPath); !strings.HasPrefix(want, "#!") || !strings.Contains(want, escaped) {+ t.Fatalf("%s smoke test is not a script tailing %q:\n%s", pack.name, escaped, want)}})}diff --git a/cli/beacon/internal/endpoint/siempack/vector_paths_test.go b/cli/beacon/internal/endpoint/siempack/vector_paths_test.godeleted file mode 100644index f0924034..00000000--- a/cli/beacon/internal/endpoint/siempack/vector_paths_test.go+++ /dev/null@@ -1,188 +0,0 @@-package siempack_test--import (- "encoding/json"- "os"- "path/filepath"- "reflect"- "strings"- "testing"- "testing/fstest"-- "github.com/pelletier/go-toml/v2"-- "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/asymptote"- "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/cloudwatch"- "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/falcon"- "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/gcs"- "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/rapid7"- "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/s3"- "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/sentinel"- "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/siempack"- "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/sumo"-)--func TestVectorPackLogPathsRoundTrip(t *testing.T) {- paths := []struct{ name, path string }{- {"posix", "/var/log/beacon-agent/runtime.jsonl"},- {"windows", `C:\ProgramData\Beacon\Endpoint\logs\runtime.jsonl`},- {"escape_sequences", `C:\new\temp\runtime.jsonl`},- {"quotes", `/tmp/beacon"quoted/runtime.jsonl`},- {"spaces_unicode", "/tmp/telemetry logs/日本語/runtime.jsonl"},- {"backslash", `/tmp/beacon\temp/runtime.jsonl`},- {"controls", "/tmp/beacon\n\r\t\b\f\x01\x1f\x7f/runtime.jsonl"},- }- packs := []struct {- name string- install func(string, string) error- render func(string) (string, error)- inventory bool- backfill bool- }{- {name: "asymptote", install: asymptote.InstallPack, render: asymptote.VectorConfig, inventory: true, backfill: true},- {name: "cloudwatch", install: cloudwatch.InstallPack, render: cloudwatch.ConfigSnippet},- {name: "falcon", install: falcon.InstallPack, render: falcon.ConfigSnippet},- {name: "gcs", install: gcs.InstallPack, inventory: true},- {name: "rapid7", install: rapid7.InstallPack},- {name: "s3", install: s3.InstallPack, inventory: true},- {name: "sentinel", install: sentinel.InstallPack},- {name: "sumo", install: sumo.InstallPack},- }- // Escaping is shared siempack code, so every path shape runs through the asymptote pack (which- // renders all three sources). The other packs only need the hardest shapes to prove their- // templates place paths in TOML basic strings.- hardest := []struct{ name, path string }{paths[1], paths[6]}- for _, pack := range packs {- t.Run(pack.name, func(t *testing.T) {- cases := hardest- if pack.name == "asymptote" {- cases = paths- }- for _, path := range cases {- t.Run(path.name, func(t *testing.T) {- want := map[string][]string{"beacon_runtime": {path.path}}- if pack.inventory {- want["beacon_inventory"] = []string{filepath.Join(filepath.Dir(path.path), "inventory_state.jsonl")}- }- if pack.backfill {- want["beacon_backfill"] = []string{filepath.Join(filepath.Dir(path.path), "cloud-backfill.jsonl")}- }- dir := t.TempDir()- if err := pack.install(dir, path.path); err != nil {- t.Fatal(err)- }- content, err := os.ReadFile(filepath.Join(dir, "vector.toml"))- if err != nil {- t.Fatal(err)- }- assertSourcePaths(t, string(content), want)- if pack.render != nil {- rendered, err := pack.render(path.path)- if err != nil {- t.Fatal(err)- }- assertSourcePaths(t, rendered, want)- if rendered != string(content) {- t.Fatal("rendered snippet and installed config differ")- }- }- })- }- })- }-}--func assertSourcePaths(t *testing.T, content string, want map[string][]string) {- t.Helper()- var config struct {- Sources map[string]struct {- Include []string `toml:"include"`- } `toml:"sources"`- }- if err := toml.Unmarshal([]byte(content), &config); err != nil {- t.Fatalf("invalid rendered TOML: %v", err)- }- for source, paths := range want {- if got := config.Sources[source].Include; !reflect.DeepEqual(got, paths) {- t.Errorf("%s.include = %q, want %q", source, got, paths)- }- }-}--func TestPackTOMLRenderingPreservesSiblingPathsAndArchiveGlob(t *testing.T) {- const content = "[sources.beacon_runtime]\ninclude = [\"{{LOG_PATH}}\", \"{{LOG_PATH}}.*\"]\n" +- "[sources.beacon_inventory]\ninclude = [\"{{INVENTORY_LOG_PATH}}\"]\n" +- "[sources.beacon_backfill]\ninclude = [\"{{BACKFILL_LOG_PATH}}\"]\n"- path := `/tmp/beacon"quoted\temp/runtime.jsonl`- pack := siempack.Pack{- DefaultLogPath: path,- FS: fstest.MapFS{"pack/vector.toml.tmpl": {Data: []byte(content)}},- Assets: []siempack.Asset{{Source: "pack/vector.toml.tmpl", Name: "vector.toml", TemplateLogPath: true}},- }- want := map[string][]string{- "beacon_runtime": {path, path + ".*"},- "beacon_inventory": {filepath.Join(filepath.Dir(path), "inventory_state.jsonl")},- "beacon_backfill": {filepath.Join(filepath.Dir(path), "cloud-backfill.jsonl")},- }- rendered, err := pack.Render("pack/vector.toml.tmpl", "")- if err != nil {- t.Fatal(err)- }- assertSourcePaths(t, rendered, want)- dir := t.TempDir()- if err := pack.Install(dir, ""); err != nil {- t.Fatal(err)- }- installed, err := os.ReadFile(filepath.Join(dir, "vector.toml"))- if err != nil {- t.Fatal(err)- }- if string(installed) != rendered {- t.Fatal("default path differs between render and install")- }-}--func TestPathRenderingPreservesOtherFormats(t *testing.T) {- path := `/tmp/beacon"quoted\temp/runtime.jsonl`- for _, name := range []string{"run.sh", "conf.yaml", "README.md", "log.xml"} {- t.Run(name, func(t *testing.T) {- const content = "path={{LOG_PATH}}; inventory={{INVENTORY_LOG_PATH}}; backfill={{BACKFILL_LOG_PATH}}"- want := siempack.RenderLogPath(content, path)- pack := siempack.Pack{FS: fstest.MapFS{"pack/" + name + ".tmpl": {Data: []byte(content)}}}- rendered, err := pack.Render("pack/"+name+".tmpl", path)- if err != nil || rendered != want {- t.Fatalf("Render = %q, %v; want %q", rendered, err, want)- }- dir := t.TempDir()- if err := siempack.Install(dir, []siempack.File{{Name: name, Content: content, TemplateLogPath: true}}, path); err != nil {- t.Fatal(err)- }- installed, err := os.ReadFile(filepath.Join(dir, name))- if err != nil || string(installed) != want {- t.Fatalf("Install = %q, %v; want %q", installed, err, want)- }- })- }- t.Run("json", func(t *testing.T) {- dir := t.TempDir()- if err := siempack.Install(dir, []siempack.File{{Name: "conf.json", Content: `{"path":"{{LOG_PATH}}"}`, TemplateLogPath: true, JSONEscape: true}}, path); err != nil {- t.Fatal(err)- }- content, err := os.ReadFile(filepath.Join(dir, "conf.json"))- if err != nil {- t.Fatal(err)- }- var got struct{ Path string }- if err := json.Unmarshal(content, &got); err != nil || got.Path != path {- t.Fatalf("JSON path = %q, %v; want %q", got.Path, err, path)- }- })- t.Run("ordinary_posix", func(t *testing.T) {- const content = "include = [\"{{LOG_PATH}}\"]\n"- pack := siempack.Pack{FS: fstest.MapFS{"pack/vector.toml.tmpl": {Data: []byte(content)}}}- rendered, err := pack.Render("pack/vector.toml.tmpl", "/var/log/beacon-agent/runtime.jsonl")- if err != nil || rendered != strings.ReplaceAll(content, "{{LOG_PATH}}", "/var/log/beacon-agent/runtime.jsonl") {- t.Fatalf("ordinary POSIX render changed: %q, %v", rendered, err)- }- })-}diff --git a/cli/beacon/internal/endpoint/wazuh/wazuh.go b/cli/beacon/internal/endpoint/wazuh/wazuh.goindex c72593f9..4ad721ef 100644--- a/cli/beacon/internal/endpoint/wazuh/wazuh.go+++ b/cli/beacon/internal/endpoint/wazuh/wazuh.go@@ -2,10 +2,13 @@ package wazuhimport ("embed"- endpointconfig "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/config"+ "fmt""os""path/filepath""strings"++ endpointconfig "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/config"+ "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/siempack")//go:embed pack/*@@ -20,38 +23,38 @@ type File struct {Content string}-func Files() []File {- localfile := LocalfileSnippet(DefaultLogPath)- rules := mustRead("pack/beacon-rules.xml")- sample := mustRead("pack/sample-event.jsonl")- readme := mustRead("pack/README.md")- dashboardColumns := mustRead("pack/apply-dashboard-default-columns.sh")+func files(localfile string) []File {return []File{{Name: "ossec-localfile.xml", Content: localfile},- {Name: "beacon-rules.xml", Content: rules},- {Name: "sample-event.jsonl", Content: sample},- {Name: "apply-dashboard-default-columns.sh", Content: dashboardColumns},- {Name: "README.md", Content: readme},+ {Name: "beacon-rules.xml", Content: mustRead("pack/beacon-rules.xml")},+ {Name: "sample-event.jsonl", Content: mustRead("pack/sample-event.jsonl")},+ {Name: "apply-dashboard-default-columns.sh", Content: mustRead("pack/apply-dashboard-default-columns.sh")},+ {Name: "README.md", Content: mustRead("pack/README.md")},}}-func LocalfileSnippet(logPath string) string {+// LocalfileSnippet returns the ossec.conf <localfile> block for logPath. Wazuh reads the path as+// written and has no way to escape a '<' in it, so such a path is refused.+func LocalfileSnippet(logPath string) (string, error) {if logPath == "" {logPath = DefaultLogPath}- return strings.ReplaceAll(mustRead("pack/ossec-localfile.xml"), "{{LOG_PATH}}", logPath)+ if strings.Contains(logPath, "<") {+ return "", fmt.Errorf("wazuh cannot read a log path containing '<': %s", logPath)+ }+ return siempack.RenderLogPath("ossec-localfile.xml", mustRead("pack/ossec-localfile.xml"), logPath), nil}func InstallPack(outputDir, logPath string) error {+ localfile, err := LocalfileSnippet(logPath)+ if err != nil {+ return err+ }if err := os.MkdirAll(outputDir, 0755); err != nil {return err}- for _, file := range Files() {- content := file.Content- if file.Name == "ossec-localfile.xml" {- content = LocalfileSnippet(logPath)- }- if err := os.WriteFile(filepath.Join(outputDir, file.Name), []byte(content), 0644); err != nil {+ for _, file := range files(localfile) {+ if err := os.WriteFile(filepath.Join(outputDir, file.Name), []byte(file.Content), 0644); err != nil {return err}}diff --git a/cli/beacon/internal/endpoint/wazuh/wazuh_test.go b/cli/beacon/internal/endpoint/wazuh/wazuh_test.goindex b549494e..db51e3f6 100644--- a/cli/beacon/internal/endpoint/wazuh/wazuh_test.go+++ b/cli/beacon/internal/endpoint/wazuh/wazuh_test.go@@ -8,7 +8,10 @@ import ()func TestLocalfileSnippetUsesConfiguredPath(t *testing.T) {- got := LocalfileSnippet("/tmp/beacon/runtime.jsonl")+ got, err := LocalfileSnippet("/tmp/beacon/runtime.jsonl")+ if err != nil {+ t.Fatal(err)+ }if !strings.Contains(got, "/tmp/beacon/runtime.jsonl") {t.Fatalf("snippet did not include configured path: %s", got)}@@ -17,6 +20,21 @@ func TestLocalfileSnippetUsesConfiguredPath(t *testing.T) {}}+// Wazuh's XML parser reads a '<' in the location as the start of an element and has no escape for it.+func TestLogPathWithAngleBracketIsRefused(t *testing.T) {+ const path = "/tmp/a<b/runtime.jsonl"+ if got, err := LocalfileSnippet(path); err == nil {+ t.Fatalf("LocalfileSnippet accepted %q: %s", path, got)+ }+ dir := filepath.Join(t.TempDir(), "pack")+ if err := InstallPack(dir, path); err == nil {+ t.Fatalf("InstallPack accepted %q", path)+ }+ if _, err := os.Stat(dir); !os.IsNotExist(err) {+ t.Fatalf("InstallPack wrote %s before refusing the path: %v", dir, err)+ }+}+func TestInstallPackWritesExpectedFiles(t *testing.T) {dir := t.TempDir()if err := InstallPack(dir, "/tmp/beacon/runtime.jsonl"); err != nil {