The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu
Go agent-ai-cliagent-harnessesagent-securitybrowser-extensionciclaude-codecloudcodexcursordetection-engineeringjevknowledge-basememorymobile-device-managementobservabilitysecuritysecurity-information-and-event-managementsecurity-toolstelemetrytraces
add datadog endpoint integration
shukieshah0863a4c778d4
diff --git a/README.md b/README.mdindex 412ed1e1..0b481e41 100644--- a/README.md+++ b/README.md@@ -40,8 +40,9 @@ events your team can inspect and retain locally.Beacon is built to be easy to deploy for Security and IT teams through[MDM deployment](https://docs.asymptotelabs.ai/cli/security-it-teams) and to-connect to Wazuh, Elastic, Splunk HEC, or customer-managed SIEM pipelines, while-remaining visibility-first and local-first during normal endpoint collection.+connect to Wazuh, Elastic, Datadog, Splunk HEC, or customer-managed SIEM+pipelines, while remaining visibility-first and local-first during normal+endpoint collection.## Supported Surfaces@@ -70,6 +71,7 @@ security pipelines.| Local JSONL | Default endpoint log and local dashboard source || Wazuh | Localfile configuration and Beacon Wazuh content pack || Elastic | Filebeat or Elastic Agent content pack over local JSONL |+| Datadog | Datadog Agent custom log collection over local JSONL || Splunk HEC | Optional endpoint forwarding during install or repair || Customer-managed SIEM pipelines | Forwarding from local Beacon JSONL under customer control |@@ -94,8 +96,8 @@ leaving forwarding under customer control.- **Beacon endpoint layer:** Local processing normalizes events, appliesretention and redaction settings, and writes durable endpoint telemetry.- **Output layer:** Teams inspect events in the local dashboard, retain JSONL,- or forward records into Wazuh, Elastic, Splunk HEC, and customer-managed SIEM- pipelines.+ or forward records into Wazuh, Elastic, Datadog, Splunk HEC, and+ customer-managed SIEM pipelines.Beacon filters generic process and runtime metrics, such as Node.js event loop,V8 heap, process CPU, and process memory telemetry, out of the local endpointdiff --git a/cli/beacon/cmd/endpoint.go b/cli/beacon/cmd/endpoint.goindex c91992f5..989fc136 100644--- a/cli/beacon/cmd/endpoint.go+++ b/cli/beacon/cmd/endpoint.go@@ -15,6 +15,7 @@ import (endpointconfig "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/config""github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/dashboard"+ "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/datadog""github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/elastic""github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/harness"endpointhooks "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/hooks"@@ -124,6 +125,11 @@ var endpointElasticCmd = &cobra.Command{Short: "Manage Elasticsearch integration content",}+var endpointDatadogCmd = &cobra.Command{+ Use: "datadog",+ Short: "Manage Datadog integration content",+}+var endpointIntegrationsCmd = &cobra.Command{Use: "integrations",Short: "Manage admin-configured endpoint integrations",@@ -328,6 +334,29 @@ var endpointElasticDownCmd = &cobra.Command{},}+var endpointDatadogPrintConfigCmd = &cobra.Command{+ Use: "print-config",+ Short: "Print a Datadog Agent custom log config for Beacon endpoint events",+ Run: func(cmd *cobra.Command, args []string) {+ cfg := loadOrDefaultConfig()+ fmt.Print(datadog.ConfigSnippet(cfg.LogPath))+ },+}++var endpointDatadogInstallPackCmd = &cobra.Command{+ Use: "install-pack",+ Short: "Write Datadog Agent custom log integration content to a directory",+ SilenceUsage: true,+ RunE: runEndpointDatadogInstallPack,+}++var endpointDatadogValidateCmd = &cobra.Command{+ Use: "validate",+ Short: "Write and describe a Datadog validation event",+ SilenceUsage: true,+ RunE: runEndpointDatadogValidate,+}+func init() {rootCmd.AddCommand(endpointCmd)@@ -339,6 +368,7 @@ func init() {endpointCmd.AddCommand(endpointDashboardCmd)endpointCmd.AddCommand(endpointWazuhCmd)endpointCmd.AddCommand(endpointElasticCmd)+ endpointCmd.AddCommand(endpointDatadogCmd)endpointCmd.AddCommand(endpointIntegrationsCmd)endpointCmd.AddCommand(endpointHooksCmd)endpointWazuhCmd.AddCommand(endpointWazuhPrintConfigCmd)@@ -348,6 +378,9 @@ func init() {endpointElasticCmd.AddCommand(endpointElasticInstallPackCmd)endpointElasticCmd.AddCommand(endpointElasticUpCmd)endpointElasticCmd.AddCommand(endpointElasticDownCmd)+ endpointDatadogCmd.AddCommand(endpointDatadogPrintConfigCmd)+ endpointDatadogCmd.AddCommand(endpointDatadogInstallPackCmd)+ endpointDatadogCmd.AddCommand(endpointDatadogValidateCmd)endpointIntegrationsCmd.AddCommand(endpointCoworkCmd)endpointIntegrationsCmd.AddCommand(endpointOpenClawCmd)endpointHooksCmd.AddCommand(endpointHooksInstallCmd)@@ -411,6 +444,12 @@ func init() {endpointElasticInstallPackCmd.Flags().StringVar(&endpointOpts.outputDir, "output", "", "Output directory for Elasticsearch content pack")endpointElasticUpCmd.Flags().StringVar(&endpointOpts.elasticPackDir, "pack-dir", elastic.DefaultOutputDir, "Elasticsearch pack directory")endpointElasticDownCmd.Flags().StringVar(&endpointOpts.elasticPackDir, "pack-dir", elastic.DefaultOutputDir, "Elasticsearch pack directory")+ for _, c := range []*cobra.Command{endpointDatadogPrintConfigCmd, endpointDatadogInstallPackCmd, endpointDatadogValidateCmd} {+ c.Flags().BoolVar(&endpointOpts.userMode, "user", true, "Use per-user endpoint paths")+ c.Flags().BoolVar(&endpointOpts.systemMode, "system", false, "Use system endpoint paths and launch daemon")+ c.Flags().StringVar(&endpointOpts.logPath, "log-path", "", "Runtime JSONL log path")+ }+ endpointDatadogInstallPackCmd.Flags().StringVar(&endpointOpts.outputDir, "output", "", "Output directory for Datadog content pack")for _, c := range []*cobra.Command{endpointCoworkPrintConfigCmd, endpointCoworkSetupCmd, endpointCoworkStatusCmd, endpointCoworkValidateCmd} {c.Flags().BoolVar(&endpointOpts.userMode, "user", true, "Use per-user endpoint paths")c.Flags().BoolVar(&endpointOpts.systemMode, "system", false, "Use system endpoint paths and launch daemon")@@ -628,6 +667,31 @@ func runEndpointWazuhValidate(cmd *cobra.Command, args []string) error {return nil}+func runEndpointDatadogInstallPack(cmd *cobra.Command, args []string) error {+ cfg := loadOrDefaultConfig()+ outputDir := endpointOpts.outputDir+ if outputDir == "" {+ outputDir = datadog.DefaultOutputDir+ }+ if err := datadog.InstallPack(outputDir, cfg.LogPath); err != nil {+ return err+ }+ fmt.Printf("Datadog content pack written to %s\n", outputDir)+ return nil+}++func runEndpointDatadogValidate(cmd *cobra.Command, args []string) error {+ cfg := loadOrDefaultConfig()+ path, err := writeValidationEvent(cfg, "datadog")+ if err != nil {+ return err+ }+ fmt.Printf("Validation event written to %s\n", path)+ fmt.Println("Expected Datadog fields: service=beacon-endpoint-agent vendor=beacon product=endpoint-agent")+ fmt.Println(`Expected validation query: service:beacon-endpoint-agent "Beacon endpoint datadog validation event"`)+ return nil+}+func runEndpointOpenClawValidate() error {cfg := loadOrDefaultConfig()setup := func() {@@ -919,15 +983,21 @@ func runEndpointDiscover(cmd *cobra.Command, args []string) error {}func writeValidationEvent(cfg endpointconfig.Config, destination string) (string, error) {+ message := "Beacon endpoint Wazuh validation event"+ mode := "localfile"+ if destination == "datadog" {+ message = "Beacon endpoint datadog validation event"+ mode = "agent_file"+ }event := schema.NewEvent(schema.NewEventOptions{Action: "agent.detected",Category: "validation",Severity: schema.SeverityInfo,AgentVersion: version.GetVersion(),Harness: schema.HarnessInfo{Name: "test_harness", Version: "test"},- Message: "Beacon endpoint Wazuh validation event",+ Message: message,})- event.Destination = &schema.DestinationInfo{Type: destination, Mode: "localfile", Status: "configured"}+ event.Destination = &schema.DestinationInfo{Type: destination, Mode: mode, Status: "configured"}return writer.AppendEvent(event, writer.Options{Path: cfg.LogPath, UserMode: cfg.UserMode})}diff --git a/cli/beacon/cmd/endpoint_test.go b/cli/beacon/cmd/endpoint_test.goindex f9a010ed..e3a0564b 100644--- a/cli/beacon/cmd/endpoint_test.go+++ b/cli/beacon/cmd/endpoint_test.go@@ -109,6 +109,30 @@ func TestEndpointElasticCommandsRegistered(t *testing.T) {}}+func TestEndpointDatadogCommandsRegistered(t *testing.T) {+ for _, path := range [][]string{+ {"datadog", "print-config"},+ {"datadog", "install-pack"},+ {"datadog", "validate"},+ } {+ cmd, _, err := endpointCmd.Find(path)+ if err != nil {+ t.Fatalf("Find %v returned error: %v", path, err)+ }+ if cmd == nil || cmd.Use != path[len(path)-1] {+ t.Fatalf("datadog command %v not registered: %#v", path, cmd)+ }+ for _, name := range []string{"user", "system", "log-path"} {+ if cmd.Flags().Lookup(name) == nil {+ t.Fatalf("datadog command %v missing --%s flag", path, name)+ }+ }+ }+ if endpointDatadogInstallPackCmd.Flags().Lookup("output") == nil {+ t.Fatal("datadog install-pack command missing --output flag")+ }+}+func TestEnsureElasticPackDoesNotOverwriteExistingPack(t *testing.T) {dir := t.TempDir()composePath := filepath.Join(dir, "docker-compose.yml")diff --git a/cli/beacon/internal/endpoint/datadog/datadog.go b/cli/beacon/internal/endpoint/datadog/datadog.gonew file mode 100644index 00000000..f230e0d2--- /dev/null+++ b/cli/beacon/internal/endpoint/datadog/datadog.go@@ -0,0 +1,64 @@+package datadog++import (+ "embed"+ "fmt"+ "os"+ "path/filepath"+ "strings"+)++//go:embed pack/*+var packFS embed.FS++const (+ DefaultLogPath = "/var/log/beacon-agent/runtime.jsonl"+ DefaultOutputDir = "beacon-datadog-pack"+)++type File struct {+ Name string+ Content string+}++func Files() []File {+ return []File{+ {Name: "README.md", Content: mustRead("pack/README.md")},+ {Name: "conf.yaml", Content: ConfigSnippet(DefaultLogPath)},+ {Name: "sample-event.jsonl", Content: mustRead("pack/sample-event.jsonl")},+ }+}++func ConfigSnippet(logPath string) string {+ if logPath == "" {+ logPath = DefaultLogPath+ }+ return strings.ReplaceAll(mustRead("pack/conf.yaml.tmpl"), "{{LOG_PATH}}", logPath)+}++func InstallPack(outputDir, logPath string) error {+ if outputDir == "" {+ outputDir = DefaultOutputDir+ }+ if err := os.MkdirAll(outputDir, 0755); err != nil {+ return err+ }+ for _, file := range Files() {+ content := file.Content+ if file.Name == "conf.yaml" {+ content = ConfigSnippet(logPath)+ }+ if err := os.WriteFile(filepath.Join(outputDir, file.Name), []byte(content), 0644); err != nil {+ return err+ }+ }+ return nil+}++func mustRead(path string) string {+ data, err := packFS.ReadFile(path)+ if err != nil {+ panic(fmt.Sprintf("datadog pack asset %s: %v", path, err))+ }+ return string(data)+}diff --git a/cli/beacon/internal/endpoint/datadog/datadog_test.go b/cli/beacon/internal/endpoint/datadog/datadog_test.gonew file mode 100644index 00000000..1f560165--- /dev/null+++ b/cli/beacon/internal/endpoint/datadog/datadog_test.go@@ -0,0 +1,93 @@+package datadog++import (+ "bufio"+ "encoding/json"+ "os"+ "path/filepath"+ "strings"+ "testing"+)++func TestConfigSnippetUsesConfiguredPath(t *testing.T) {+ got := ConfigSnippet("/tmp/beacon/runtime.jsonl")+ if !strings.Contains(got, "/tmp/beacon/runtime.jsonl") {+ t.Fatalf("snippet did not include configured path: %s", got)+ }+ if strings.Contains(got, "{{LOG_PATH}}") {+ t.Fatalf("snippet still contains template token: %s", got)+ }+ for _, want := range []string{+ "logs:",+ "type: file",+ "service: beacon-endpoint-agent",+ "source: beacon",+ "vendor:beacon",+ "product:endpoint-agent",+ } {+ if !strings.Contains(got, want) {+ t.Fatalf("snippet missing %q: %s", want, got)+ }+ }+}++func TestInstallPackWritesExpectedFiles(t *testing.T) {+ dir := t.TempDir()+ if err := InstallPack(dir, "/tmp/beacon/runtime.jsonl"); err != nil {+ t.Fatalf("InstallPack returned error: %v", err)+ }+ for _, name := range []string{"README.md", "conf.yaml", "sample-event.jsonl"} {+ if _, err := os.Stat(filepath.Join(dir, name)); err != nil {+ t.Fatalf("expected %s: %v", name, err)+ }+ }+ config, err := os.ReadFile(filepath.Join(dir, "conf.yaml"))+ if err != nil {+ t.Fatal(err)+ }+ if !strings.Contains(string(config), "/tmp/beacon/runtime.jsonl") {+ t.Fatalf("generated conf.yaml missing configured log path: %s", config)+ }+}++func TestSampleEventsCoverValidationHookAndOTelShapes(t *testing.T) {+ scanner := bufio.NewScanner(strings.NewReader(mustRead("pack/sample-event.jsonl")))+ var sawValidation, sawHook, sawOTel bool+ for scanner.Scan() {+ var doc map[string]interface{}+ if err := json.Unmarshal(scanner.Bytes(), &doc); err != nil {+ t.Fatalf("sample-event.jsonl is not valid JSONL: %v", err)+ }+ if destination, ok := doc["destination"].(map[string]interface{}); ok && destination["type"] == "datadog" {+ sawValidation = true+ }+ if _, ok := doc["tool"].(map[string]interface{}); ok {+ sawHook = true+ }+ if raw, ok := doc["raw"].(map[string]interface{}); ok && raw["otel_signal"] != nil {+ sawOTel = true+ }+ }+ if err := scanner.Err(); err != nil {+ t.Fatal(err)+ }+ if !sawValidation || !sawHook || !sawOTel {+ t.Fatalf("sample events should include validation, hook-rich, and OTel shapes; validation=%t hook=%t otel=%t", sawValidation, sawHook, sawOTel)+ }+}++func TestPackREADMEMentionsValidationRetentionAndMacOSPermissions(t *testing.T) {+ readme := mustRead("pack/README.md")+ for _, want := range []string{+ "beacon endpoint datadog validate",+ "service:beacon-endpoint-agent",+ "Beacon endpoint datadog validation event",+ "content retention",+ "_dd-agent",+ "/var/log/beacon-agent/runtime.jsonl",+ } {+ if !strings.Contains(readme, want) {+ t.Fatalf("pack README missing %q", want)+ }+ }+}diff --git a/cli/beacon/internal/endpoint/datadog/pack/README.md b/cli/beacon/internal/endpoint/datadog/pack/README.mdnew file mode 100644index 00000000..df45379d--- /dev/null+++ b/cli/beacon/internal/endpoint/datadog/pack/README.md@@ -0,0 +1,89 @@+# Beacon Endpoint Agent Datadog Pack++This pack forwards Beacon endpoint JSONL events into Datadog with the Datadog+Agent's native custom log collection. Beacon still writes one local source of+truth: `runtime.jsonl`. Datadog API keys and site configuration stay in the+Datadog Agent, not in Beacon endpoint configuration.++## Prerequisites++- Datadog Agent installed on the endpoint.+- Log collection enabled in `/opt/datadog-agent/etc/datadog.yaml`:++```yaml+logs_enabled: true+```++## Install++Generate this pack:++```bash+beacon endpoint datadog install-pack --output ./beacon-datadog-pack+```++Install the generated config on macOS:++```bash+sudo mkdir -p /opt/datadog-agent/etc/conf.d/beacon.d+sudo cp ./beacon-datadog-pack/conf.yaml /opt/datadog-agent/etc/conf.d/beacon.d/conf.yaml+sudo chmod 0644 /opt/datadog-agent/etc/conf.d/beacon.d/conf.yaml+sudo launchctl kickstart -k system/com.datadoghq.agent+```++The generated `conf.yaml` points at the Beacon log path selected by the CLI:++- User mode: `~/.beacon/endpoint/logs/runtime.jsonl`+- System mode: `/var/log/beacon-agent/runtime.jsonl`+- Custom mode: the value passed with `--log-path`++For MDM or managed endpoint deployment, prefer Beacon system mode so Datadog can+tail `/var/log/beacon-agent/runtime.jsonl` without per-user home directory ACLs.++## User-Mode macOS Permissions++The Datadog Agent usually runs as `_dd-agent`. When tailing a user-mode Beacon+log inside a home directory, `_dd-agent` must be able to traverse the parent+directories and read the log file. If `datadog-agent status` reports+`permission denied`, either use Beacon system mode or grant a narrow ACL for the+configured path.++## Validate++Write a fresh Beacon validation event:++```bash+beacon endpoint datadog validate+```++Check the Datadog Agent status:++```bash+sudo datadog-agent status+```++The Logs Agent section should show logs processed and sent. In Datadog Log+Explorer, search for:++```text+service:beacon-endpoint-agent vendor:beacon product:endpoint-agent+```++You can also search for the validation event without requiring a custom Datadog+facet or parsing pipeline:++```text+service:beacon-endpoint-agent "Beacon endpoint datadog validation event"+```++## Content Retention++Beacon content retention defaults to `full`, so prompt text, tool input, command+output, and other retained content may be forwarded to Datadog. Use Beacon's+`metadata` or `redacted` content retention modes for stricter deployments.++## OpenTelemetry Note++Datadog's DDOT Collector is a good fit for OTel-first Linux or Kubernetes+deployments, but Beacon's macOS endpoint v0 uses native Datadog Agent file log+collection because it is the supported host path for tailing local JSONL files.diff --git a/cli/beacon/internal/endpoint/datadog/pack/conf.yaml.tmpl b/cli/beacon/internal/endpoint/datadog/pack/conf.yaml.tmplnew file mode 100644index 00000000..99ed756b--- /dev/null+++ b/cli/beacon/internal/endpoint/datadog/pack/conf.yaml.tmpl@@ -0,0 +1,11 @@+# Beacon endpoint runtime events.+# Requires logs_enabled: true in /opt/datadog-agent/etc/datadog.yaml.+logs:+ - type: file+ path: "{{LOG_PATH}}"+ service: beacon-endpoint-agent+ source: beacon+ sourcecategory: agent-runtime+ tags:+ - vendor:beacon+ - product:endpoint-agentdiff --git a/collector-builder/exporter/beaconjsonexporter/exporter.go b/collector-builder/exporter/beaconjsonexporter/exporter.goindex 41a4a79b..60d66fe2 100644--- a/collector-builder/exporter/beaconjsonexporter/exporter.go+++ b/collector-builder/exporter/beaconjsonexporter/exporter.go@@ -264,11 +264,7 @@ func (e *beaconExporter) normalizeCodexLogEvent(event *beaconEvent, attrs map[stcase codexUserPrompt:event.Event.Action = "prompt.submitted"event.Event.Category = "prompt"- if prompt := firstString(attrs, "prompt", "gen_ai.prompt", "user_prompt", "input.prompt"); e.cfg.ContentRetention != "metadata" && prompt != "" {- event.Message = prompt- } else {- event.Message = "Codex prompt submitted"- }+ event.Message = "Codex prompt submitted"case codexToolDecision:decision := firstString(attrs, "decision")if strings.EqualFold(decision, "denied") || strings.EqualFold(decision, "deny") {diff --git a/collector-builder/exporter/beaconjsonexporter/exporter_test.go b/collector-builder/exporter/beaconjsonexporter/exporter_test.goindex e0a4ba9d..6a388765 100644--- a/collector-builder/exporter/beaconjsonexporter/exporter_test.go+++ b/collector-builder/exporter/beaconjsonexporter/exporter_test.go@@ -483,8 +483,8 @@ func TestConsumeLogsMapsCodexSemanticEventsAndDropsTransportNoise(t *testing.T)if event.Prompt == nil || event.Prompt.Text != "look up weather token=[REDACTED]" {t.Fatalf("prompt = %#v, want redacted typed prompt", event.Prompt)}- if event.Message != "look up weather token=[REDACTED]" {- t.Fatalf("prompt message = %q, want prompt text", event.Message)+ if event.Message != "Codex prompt submitted" {+ t.Fatalf("prompt message = %q, want generic placeholder", event.Message)}}}