The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu
Go agent-ai-cliagent-harnessesagent-securitybrowser-extensionciclaude-codecloudcodexcursordetection-engineeringjevknowledge-basememorymobile-device-managementobservabilitysecuritysecurity-information-and-event-managementsecurity-toolstelemetrytraces
add vector
Justin D'Souzadd3e294edc3e
diff --git a/cli/beacon/internal/endpoint/rapid7/pack/README.md b/cli/beacon/internal/endpoint/rapid7/pack/README.mdindex 232c1936..0fc8dc0b 100644--- a/cli/beacon/internal/endpoint/rapid7/pack/README.md+++ b/cli/beacon/internal/endpoint/rapid7/pack/README.md@@ -44,6 +44,8 @@ For MDM or managed endpoint deployment, prefer Beacon system mode so yourcustomer-managed log shipper can tail `/var/log/beacon-agent/runtime.jsonl`without per-user home directory ACLs.+The generated `vector.toml` uses the same selected Beacon log path.+## One-Shot Smoke TestUse the generated script to upload the current file once. This is only for@@ -67,9 +69,33 @@ individual event.## Production Forwarding-For production, use your fleet's existing log shipper or a customer-managed-collector to tail `runtime.jsonl` and POST batches to the Rapid7 Custom Logs-webhook. The forwarder should:+For production, use the generated Vector config as a customer-managed host-agent+forwarding template. Beacon remains the local JSONL producer; Vector tails+`runtime.jsonl`, checkpoints file offsets in its `data_dir`, batches Beacon+events, and posts NDJSON payloads to the Rapid7 Custom Logs webhook.++Install Vector using your normal endpoint management tooling, then copy the+generated config into Vector's config directory. On a macOS system-mode Beacon+deployment, the generated config tails `/var/log/beacon-agent/runtime.jsonl`:++```bash+sudo mkdir -p /etc/vector+sudo cp ./beacon-rapid7-pack/vector.toml /etc/vector/beacon-rapid7.toml+export RAPID7_WEBHOOK_URL="https://..."+vector validate /etc/vector/beacon-rapid7.toml+vector --config /etc/vector/beacon-rapid7.toml+```++In managed deployments, provide `RAPID7_WEBHOOK_URL` through the Vector service+environment or your MDM/secret tooling. Do not store Rapid7 webhook URLs in+Beacon endpoint configuration.++The Vector template is intentionally simple and expects a Vector version with+the `file` source, `remap` transform, and `http` sink. It parses each Beacon+JSONL line and re-encodes the original Beacon event as NDJSON so Rapid7 receives+one Beacon event per line, without a Vector wrapper.++If you adapt the config or use another forwarder, it should:- checkpoint file offsets,- batch newline-delimited JSON records,diff --git a/cli/beacon/internal/endpoint/rapid7/pack/vector.toml.tmpl b/cli/beacon/internal/endpoint/rapid7/pack/vector.toml.tmplnew file mode 100644index 00000000..db7a225a--- /dev/null+++ b/cli/beacon/internal/endpoint/rapid7/pack/vector.toml.tmpl@@ -0,0 +1,39 @@+# Beacon Endpoint Agent -> Rapid7 InsightIDR Custom Logs+#+# Run Vector as a customer-managed host agent. Keep RAPID7_WEBHOOK_URL in your+# deployment tooling or Vector service environment, not in Beacon config.++data_dir = "/var/lib/vector/beacon-rapid7"++[sources.beacon_runtime]+type = "file"+include = ["{{LOG_PATH}}"]+read_from = "end"++[transforms.beacon_json]+type = "remap"+inputs = ["beacon_runtime"]+source = '''+. = parse_json!(.message)+'''++[sinks.rapid7_custom_logs]+type = "http"+inputs = ["beacon_json"]+uri = "${RAPID7_WEBHOOK_URL}"+method = "post"++[sinks.rapid7_custom_logs.encoding]+codec = "ndjson"++[sinks.rapid7_custom_logs.batch]+max_events = 500+timeout_secs = 5++[sinks.rapid7_custom_logs.request]+retry_attempts = 10+retry_initial_backoff_secs = 1+retry_max_duration_secs = 300++[sinks.rapid7_custom_logs.headers]+Content-Type = "application/x-ndjson"diff --git a/cli/beacon/internal/endpoint/rapid7/rapid7.go b/cli/beacon/internal/endpoint/rapid7/rapid7.goindex 9863b5a1..8b305dce 100644--- a/cli/beacon/internal/endpoint/rapid7/rapid7.go+++ b/cli/beacon/internal/endpoint/rapid7/rapid7.go@@ -3,9 +3,8 @@ package rapid7import ("embed""fmt"- "os"- "path/filepath"- "strings"++ "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/siempack")//go:embed pack/*@@ -17,15 +16,17 @@ const ()type File struct {- Name string- Content string+ Name string+ Content string+ TemplateLogPath bool}func Files() []File {return []File{{Name: "README.md", Content: mustRead("pack/README.md")},- {Name: "rapid7-upload-smoke-test.sh", Content: UploadSmokeTest(DefaultLogPath)},+ {Name: "rapid7-upload-smoke-test.sh", Content: mustRead("pack/rapid7-upload-smoke-test.sh.tmpl"), TemplateLogPath: true},{Name: "sample-event.jsonl", Content: mustRead("pack/sample-event.jsonl")},+ {Name: "vector.toml", Content: mustRead("pack/vector.toml.tmpl"), TemplateLogPath: true},}}@@ -33,39 +34,27 @@ func UploadSmokeTest(logPath string) string {if logPath == "" {logPath = DefaultLogPath}- return strings.ReplaceAll(mustRead("pack/rapid7-upload-smoke-test.sh.tmpl"), "{{LOG_PATH}}", logPath)+ return siempack.RenderLogPath(mustRead("pack/rapid7-upload-smoke-test.sh.tmpl"), logPath)}func InstallPack(outputDir, logPath string) error {if outputDir == "" {outputDir = DefaultOutputDir}- if err := os.MkdirAll(outputDir, 0755); err != nil {- return err+ if logPath == "" {+ logPath = DefaultLogPath}+ files := make([]siempack.File, 0, len(Files()))for _, file := range Files() {- content := file.Content- if file.Name == "rapid7-upload-smoke-test.sh" {- content = UploadSmokeTest(logPath)- }- if err := os.WriteFile(filepath.Join(outputDir, file.Name), []byte(content), fileMode(file.Name)); err != nil {- return err- }- }- return nil-}--func fileMode(name string) os.FileMode {- if strings.HasSuffix(name, ".sh") {- return 0755+ files = append(files, siempack.File(file))}- return 0644+ return siempack.Install(outputDir, files, logPath)}func mustRead(path string) string {- data, err := packFS.ReadFile(path)+ data, err := siempack.ReadFile(packFS, path)if err != nil {panic(fmt.Sprintf("rapid7 pack asset %s: %v", path, err))}- return string(data)+ return data}diff --git a/cli/beacon/internal/endpoint/rapid7/rapid7_test.go b/cli/beacon/internal/endpoint/rapid7/rapid7_test.goindex 294c4823..fdd671f3 100644--- a/cli/beacon/internal/endpoint/rapid7/rapid7_test.go+++ b/cli/beacon/internal/endpoint/rapid7/rapid7_test.go@@ -35,7 +35,7 @@ func TestInstallPackWritesExpectedFiles(t *testing.T) {if err := InstallPack(dir, "/tmp/beacon/runtime.jsonl"); err != nil {t.Fatalf("InstallPack returned error: %v", err)}- for _, name := range []string{"README.md", "rapid7-upload-smoke-test.sh", "sample-event.jsonl"} {+ for _, name := range []string{"README.md", "rapid7-upload-smoke-test.sh", "sample-event.jsonl", "vector.toml"} {if _, err := os.Stat(filepath.Join(dir, name)); err != nil {t.Fatalf("expected %s: %v", name, err)}@@ -55,6 +55,38 @@ func TestInstallPackWritesExpectedFiles(t *testing.T) {if info.Mode().Perm()&0111 == 0 {t.Fatalf("generated script should be executable, mode=%s", info.Mode())}+ vectorPath := filepath.Join(dir, "vector.toml")+ vectorConfig, err := os.ReadFile(vectorPath)+ if err != nil {+ t.Fatal(err)+ }+ if !strings.Contains(string(vectorConfig), "/tmp/beacon/runtime.jsonl") {+ t.Fatalf("generated vector config missing configured log path: %s", vectorConfig)+ }+ info, err = os.Stat(vectorPath)+ if err != nil {+ t.Fatal(err)+ }+ if info.Mode().Perm() != 0644 {+ t.Fatalf("generated vector config should be 0644, mode=%s", info.Mode().Perm())+ }+}++func TestVectorConfigUsesRapid7WebhookAndPreservesJSONShape(t *testing.T) {+ got := mustRead("pack/vector.toml.tmpl")+ for _, want := range []string{+ `include = ["{{LOG_PATH}}"]`,+ `read_from = "end"`,+ `. = parse_json!(.message)`,+ `uri = "${RAPID7_WEBHOOK_URL}"`,+ `codec = "ndjson"`,+ `retry_attempts = 10`,+ `Content-Type = "application/x-ndjson"`,+ } {+ if !strings.Contains(got, want) {+ t.Fatalf("vector config missing %q: %s", want, got)+ }+ }}func TestSampleEventsCoverValidationHookAndOTelShapes(t *testing.T) {@@ -94,6 +126,9 @@ func TestPackREADMEMentionsRapid7SetupAndProductionForwarding(t *testing.T) {"application/x-ndjson","Beacon endpoint Rapid7 validation event","vendor=beacon product=endpoint-agent destination.type=rapid7",+ "vector.toml",+ "customer-managed host-agent",+ "without a Vector wrapper","content retention","/var/log/beacon-agent/runtime.jsonl",} {diff --git a/cli/beacon/internal/endpoint/siempack/siempack.go b/cli/beacon/internal/endpoint/siempack/siempack.gonew file mode 100644index 00000000..e58d5f90--- /dev/null+++ b/cli/beacon/internal/endpoint/siempack/siempack.go@@ -0,0 +1,51 @@+package siempack++import (+ "io/fs"+ "os"+ "path/filepath"+ "strings"+)++const logPathToken = "{{LOG_PATH}}"++type File struct {+ Name string+ Content string+ TemplateLogPath bool+}++func ReadFile(fsys fs.FS, path string) (string, error) {+ data, err := fs.ReadFile(fsys, path)+ if err != nil {+ return "", err+ }+ return string(data), nil+}++func RenderLogPath(content, logPath string) string {+ return strings.ReplaceAll(content, logPathToken, logPath)+}++func Install(outputDir string, files []File, logPath string) error {+ if err := os.MkdirAll(outputDir, 0755); err != nil {+ return err+ }+ for _, file := range files {+ content := file.Content+ if file.TemplateLogPath {+ content = RenderLogPath(content, logPath)+ }+ if err := os.WriteFile(filepath.Join(outputDir, file.Name), []byte(content), Mode(file.Name)); err != nil {+ return err+ }+ }+ return nil+}++func Mode(name string) os.FileMode {+ if strings.HasSuffix(name, ".sh") {+ return 0755+ }+ return 0644+}diff --git a/cli/beacon/internal/endpoint/siempack/siempack_test.go b/cli/beacon/internal/endpoint/siempack/siempack_test.gonew file mode 100644index 00000000..1c559c9a--- /dev/null+++ b/cli/beacon/internal/endpoint/siempack/siempack_test.go@@ -0,0 +1,64 @@+package siempack++import (+ "os"+ "path/filepath"+ "strings"+ "testing"+ "testing/fstest"+)++func TestRenderLogPath(t *testing.T) {+ got := RenderLogPath("path={{LOG_PATH}}", "/var/log/beacon-agent/runtime.jsonl")+ if got != "path=/var/log/beacon-agent/runtime.jsonl" {+ t.Fatalf("RenderLogPath() = %q", got)+ }+}++func TestReadFile(t *testing.T) {+ got, err := ReadFile(fstest.MapFS{+ "pack/example.txt": {Data: []byte("hello")},+ }, "pack/example.txt")+ if err != nil {+ t.Fatal(err)+ }+ if got != "hello" {+ t.Fatalf("ReadFile() = %q", got)+ }+}++func TestInstallWritesFilesWithModesAndLogPath(t *testing.T) {+ dir := t.TempDir()+ err := Install(dir, []File{+ {Name: "README.md", Content: "docs"},+ {Name: "upload.sh", Content: "log={{LOG_PATH}}", TemplateLogPath: true},+ {Name: "vector.toml", Content: "path={{LOG_PATH}}", TemplateLogPath: true},+ }, "/tmp/beacon/runtime.jsonl")+ if err != nil {+ t.Fatal(err)+ }++ scriptPath := filepath.Join(dir, "upload.sh")+ script, err := os.ReadFile(scriptPath)+ if err != nil {+ t.Fatal(err)+ }+ if !strings.Contains(string(script), "/tmp/beacon/runtime.jsonl") {+ t.Fatalf("script missing rendered log path: %s", script)+ }+ info, err := os.Stat(scriptPath)+ if err != nil {+ t.Fatal(err)+ }+ if info.Mode().Perm() != 0755 {+ t.Fatalf("script mode = %s, want 0755", info.Mode().Perm())+ }++ configInfo, err := os.Stat(filepath.Join(dir, "vector.toml"))+ if err != nil {+ t.Fatal(err)+ }+ if configInfo.Mode().Perm() != 0644 {+ t.Fatalf("config mode = %s, want 0644", configInfo.Mode().Perm())+ }+}diff --git a/cli/beacon/internal/endpoint/sumo/pack/README.md b/cli/beacon/internal/endpoint/sumo/pack/README.mdindex 9af1d957..e7e578fe 100644--- a/cli/beacon/internal/endpoint/sumo/pack/README.md+++ b/cli/beacon/internal/endpoint/sumo/pack/README.md@@ -42,6 +42,8 @@ For MDM or managed endpoint deployment, prefer Beacon system mode so yourcustomer-managed log shipper can tail `/var/log/beacon-agent/runtime.jsonl`without per-user home directory ACLs.+The generated `vector.toml` uses the same selected Beacon log path.+## One-Shot Smoke TestUse the generated script to upload the current file once. This is only for@@ -76,9 +78,35 @@ boundary detection.## Production Forwarding-For production, use your fleet's existing log shipper or Sumo's OpenTelemetry-Collector distribution to tail `runtime.jsonl` and POST batches to the Sumo HTTP-Source. The forwarder should:+For production, use the generated Vector config as a customer-managed host-agent+forwarding template. Beacon remains the local JSONL producer; Vector tails+`runtime.jsonl`, checkpoints file offsets in its `data_dir`, batches Beacon+events, and posts JSONL payloads to the Sumo HTTP Logs & Metrics Source.++Install Vector using your normal endpoint management tooling, then copy the+generated config into Vector's config directory. On a macOS system-mode Beacon+deployment, the generated config tails `/var/log/beacon-agent/runtime.jsonl`:++```bash+sudo mkdir -p /etc/vector+sudo cp ./beacon-sumo-pack/vector.toml /etc/vector/beacon-sumo.toml+export SUMO_URL="https://collectors.sumologic.com/receiver/v1/http/..."+export SUMO_TOKEN="..."+vector validate /etc/vector/beacon-sumo.toml+vector --config /etc/vector/beacon-sumo.toml+```++`SUMO_TOKEN` is optional when `SUMO_URL` is a presigned Source URL. In managed+deployments, provide `SUMO_URL`, optional `SUMO_TOKEN`, `SUMO_SOURCE_CATEGORY`,+and `SUMO_FIELDS` through the Vector service environment or your MDM/secret+tooling. Do not store Sumo destination secrets in Beacon endpoint configuration.++The Vector template is intentionally simple and expects a Vector version with+the `file` source, `remap` transform, and `http` sink. It parses each Beacon+JSONL line and re-encodes the original Beacon event as NDJSON so Sumo receives+one Beacon event per line, without a Vector wrapper.++If you adapt the config or use another forwarder, it should:- checkpoint file offsets,- batch newline-delimited JSON records,diff --git a/cli/beacon/internal/endpoint/sumo/pack/vector.toml.tmpl b/cli/beacon/internal/endpoint/sumo/pack/vector.toml.tmplnew file mode 100644index 00000000..7f6d64e7--- /dev/null+++ b/cli/beacon/internal/endpoint/sumo/pack/vector.toml.tmpl@@ -0,0 +1,42 @@+# Beacon Endpoint Agent -> Sumo Logic HTTP Logs & Metrics Source+#+# Run Vector as a customer-managed host agent. Keep SUMO_URL and optional+# SUMO_TOKEN in your deployment tooling or Vector service environment, not in+# Beacon config.++data_dir = "/var/lib/vector/beacon-sumo"++[sources.beacon_runtime]+type = "file"+include = ["{{LOG_PATH}}"]+read_from = "end"++[transforms.beacon_json]+type = "remap"+inputs = ["beacon_runtime"]+source = '''+. = parse_json!(.message)+'''++[sinks.sumo_http_logs]+type = "http"+inputs = ["beacon_json"]+uri = "${SUMO_URL}"+method = "post"++[sinks.sumo_http_logs.encoding]+codec = "ndjson"++[sinks.sumo_http_logs.batch]+max_events = 500+timeout_secs = 5++[sinks.sumo_http_logs.request]+retry_attempts = 10+retry_initial_backoff_secs = 1+retry_max_duration_secs = 300++[sinks.sumo_http_logs.headers]+X-Sumo-Category = "${SUMO_SOURCE_CATEGORY:-security/agentbeacon}"+X-Sumo-Fields = "${SUMO_FIELDS:-product=agentbeacon,telemetry=ai_agent,env=prod}"+x-sumo-token = "${SUMO_TOKEN:-}"diff --git a/cli/beacon/internal/endpoint/sumo/sumo.go b/cli/beacon/internal/endpoint/sumo/sumo.goindex 51b39b62..199823b3 100644--- a/cli/beacon/internal/endpoint/sumo/sumo.go+++ b/cli/beacon/internal/endpoint/sumo/sumo.go@@ -3,9 +3,8 @@ package sumoimport ("embed""fmt"- "os"- "path/filepath"- "strings"++ "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/siempack")//go:embed pack/*@@ -17,15 +16,17 @@ const ()type File struct {- Name string- Content string+ Name string+ Content string+ TemplateLogPath bool}func Files() []File {return []File{{Name: "README.md", Content: mustRead("pack/README.md")},- {Name: "sumo-upload-smoke-test.sh", Content: UploadSmokeTest(DefaultLogPath)},+ {Name: "sumo-upload-smoke-test.sh", Content: mustRead("pack/sumo-upload-smoke-test.sh.tmpl"), TemplateLogPath: true},{Name: "sample-event.jsonl", Content: mustRead("pack/sample-event.jsonl")},+ {Name: "vector.toml", Content: mustRead("pack/vector.toml.tmpl"), TemplateLogPath: true},}}@@ -33,39 +34,27 @@ func UploadSmokeTest(logPath string) string {if logPath == "" {logPath = DefaultLogPath}- return strings.ReplaceAll(mustRead("pack/sumo-upload-smoke-test.sh.tmpl"), "{{LOG_PATH}}", logPath)+ return siempack.RenderLogPath(mustRead("pack/sumo-upload-smoke-test.sh.tmpl"), logPath)}func InstallPack(outputDir, logPath string) error {if outputDir == "" {outputDir = DefaultOutputDir}- if err := os.MkdirAll(outputDir, 0755); err != nil {- return err+ if logPath == "" {+ logPath = DefaultLogPath}+ files := make([]siempack.File, 0, len(Files()))for _, file := range Files() {- content := file.Content- if file.Name == "sumo-upload-smoke-test.sh" {- content = UploadSmokeTest(logPath)- }- if err := os.WriteFile(filepath.Join(outputDir, file.Name), []byte(content), fileMode(file.Name)); err != nil {- return err- }- }- return nil-}--func fileMode(name string) os.FileMode {- if strings.HasSuffix(name, ".sh") {- return 0755+ files = append(files, siempack.File(file))}- return 0644+ return siempack.Install(outputDir, files, logPath)}func mustRead(path string) string {- data, err := packFS.ReadFile(path)+ data, err := siempack.ReadFile(packFS, path)if err != nil {panic(fmt.Sprintf("sumo pack asset %s: %v", path, err))}- return string(data)+ return data}diff --git a/cli/beacon/internal/endpoint/sumo/sumo_test.go b/cli/beacon/internal/endpoint/sumo/sumo_test.goindex 0dd73b51..a4b5cd62 100644--- a/cli/beacon/internal/endpoint/sumo/sumo_test.go+++ b/cli/beacon/internal/endpoint/sumo/sumo_test.go@@ -39,7 +39,7 @@ func TestInstallPackWritesExpectedFiles(t *testing.T) {if err := InstallPack(dir, "/tmp/beacon/runtime.jsonl"); err != nil {t.Fatalf("InstallPack returned error: %v", err)}- for _, name := range []string{"README.md", "sumo-upload-smoke-test.sh", "sample-event.jsonl"} {+ for _, name := range []string{"README.md", "sumo-upload-smoke-test.sh", "sample-event.jsonl", "vector.toml"} {if _, err := os.Stat(filepath.Join(dir, name)); err != nil {t.Fatalf("expected %s: %v", name, err)}@@ -59,6 +59,40 @@ func TestInstallPackWritesExpectedFiles(t *testing.T) {if info.Mode().Perm()&0111 == 0 {t.Fatalf("generated script should be executable, mode=%s", info.Mode())}+ vectorPath := filepath.Join(dir, "vector.toml")+ vectorConfig, err := os.ReadFile(vectorPath)+ if err != nil {+ t.Fatal(err)+ }+ if !strings.Contains(string(vectorConfig), "/tmp/beacon/runtime.jsonl") {+ t.Fatalf("generated vector config missing configured log path: %s", vectorConfig)+ }+ info, err = os.Stat(vectorPath)+ if err != nil {+ t.Fatal(err)+ }+ if info.Mode().Perm() != 0644 {+ t.Fatalf("generated vector config should be 0644, mode=%s", info.Mode().Perm())+ }+}++func TestVectorConfigUsesSumoDestinationAndPreservesJSONShape(t *testing.T) {+ got := mustRead("pack/vector.toml.tmpl")+ for _, want := range []string{+ `include = ["{{LOG_PATH}}"]`,+ `read_from = "end"`,+ `. = parse_json!(.message)`,+ `uri = "${SUMO_URL}"`,+ `codec = "ndjson"`,+ `retry_attempts = 10`,+ `X-Sumo-Category = "${SUMO_SOURCE_CATEGORY:-security/agentbeacon}"`,+ `X-Sumo-Fields = "${SUMO_FIELDS:-product=agentbeacon,telemetry=ai_agent,env=prod}"`,+ `x-sumo-token = "${SUMO_TOKEN:-}"`,+ } {+ if !strings.Contains(got, want) {+ t.Fatalf("vector config missing %q: %s", want, got)+ }+ }}func TestSampleEventsCoverValidationHookAndOTelShapes(t *testing.T) {@@ -97,6 +131,10 @@ func TestPackREADMEMentionsSumoSetupAndProductionForwarding(t *testing.T) {"100 KB to 1 MB","Content-Encoding: gzip","Live Tail",+ "vector.toml",+ "customer-managed host-agent",+ "SUMO_URL",+ "without a Vector wrapper","content retention","/var/log/beacon-agent/runtime.jsonl",} {