The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu
Go agent-ai-cliagent-harnessesagent-securitybrowser-extensionciclaude-codecloudcodexcursordetection-engineeringjevknowledge-basememorymobile-device-managementobservabilitysecuritysecurity-information-and-event-managementsecurity-toolstelemetrytraces
add rapid7 support
Justin D'Souzabb2c84717b97
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.ymlindex e3ad4d5d..e2b45fd8 100644--- a/.github/workflows/ci.yml+++ b/.github/workflows/ci.yml@@ -32,6 +32,7 @@ jobs:./beacon endpoint hooks --help > /tmp/beacon-hooks-help.txt./beacon endpoint integrations claude-cowork --help > /tmp/beacon-cowork-help.txt./beacon endpoint wazuh --help > /tmp/beacon-wazuh-help.txt+ ./beacon endpoint rapid7 --help > /tmp/beacon-rapid7-help.txtfor forbidden in login init update claude cursor dep-scan config whoami; doif grep -E "^[[:space:]]+$forbidden([[:space:]]|$)" /tmp/beacon-help.txt; thenecho "forbidden command exposed: $forbidden"diff --git a/README.md b/README.mdindex 06f47d23..58d3c7a8 100644--- a/README.md+++ b/README.md@@ -99,6 +99,7 @@ enterprise-grade SIEMs.| Elastic | Filebeat or Elastic Agent content pack over local JSONL || Datadog | Datadog Agent custom log collection over local JSONL || Sumo Logic | HTTP Logs & Metrics Source content pack over local JSONL |+| Rapid7 InsightIDR | Custom Logs webhook content pack over local JSONL || Splunk HEC | Optional endpoint forwarding during install or repair || Customer-managed SIEM pipelines | Forwarding from local Beacon JSONL under customer control |diff --git a/cli/beacon/README.md b/cli/beacon/README.mdindex 79325a0a..ca3ee8ce 100644--- a/cli/beacon/README.md+++ b/cli/beacon/README.md@@ -73,6 +73,19 @@ The Sumo pack keeps Beacon as a local JSONL producer and documents forwardingMetrics Source. Use a tailing forwarder for production so offsets arecheckpointed and the whole file is not repeatedly uploaded.+## Rapid7 InsightIDR++```bash+./beacon endpoint rapid7 print-config+./beacon endpoint rapid7 install-pack --output ./beacon-rapid7-pack+./beacon endpoint rapid7 validate+```++The Rapid7 pack keeps Beacon as a local JSONL producer and documents forwarding+`runtime.jsonl` into a Rapid7 InsightIDR Custom Logs webhook as NDJSON. Store the+webhook URL in your customer-managed shipper or deployment tooling, not in+Beacon endpoint configuration.+## Optional Integrations```bashdiff --git a/cli/beacon/cmd/endpoint.go b/cli/beacon/cmd/endpoint.goindex 53d73767..aa9316e2 100644--- a/cli/beacon/cmd/endpoint.go+++ b/cli/beacon/cmd/endpoint.go@@ -22,6 +22,7 @@ import ("github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/integrations/cowork""github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/integrations/openclaw""github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/lifecycle"+ "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/rapid7""github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/schema""github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/sumo""github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/wazuh"@@ -169,6 +170,11 @@ var endpointSumoCmd = &cobra.Command{Short: "Manage Sumo Logic integration content",}+var endpointRapid7Cmd = &cobra.Command{+ Use: "rapid7",+ Short: "Manage Rapid7 InsightIDR integration content",+}+var endpointIntegrationsCmd = &cobra.Command{Use: "integrations",Short: "Manage admin-configured endpoint integrations",@@ -474,6 +480,29 @@ var endpointSumoValidateCmd = &cobra.Command{RunE: runEndpointSumoValidate,}+var endpointRapid7PrintConfigCmd = &cobra.Command{+ Use: "print-config",+ Short: "Print a Rapid7 Custom Logs webhook smoke-test uploader for Beacon endpoint events",+ Run: func(cmd *cobra.Command, args []string) {+ cfg := loadOrDefaultConfig()+ fmt.Print(rapid7.UploadSmokeTest(cfg.LogPath))+ },+}++var endpointRapid7InstallPackCmd = &cobra.Command{+ Use: "install-pack",+ Short: "Write Rapid7 InsightIDR Custom Logs forwarding content to a directory",+ SilenceUsage: true,+ RunE: runEndpointRapid7InstallPack,+}++var endpointRapid7ValidateCmd = &cobra.Command{+ Use: "validate",+ Short: "Write and describe a Rapid7 InsightIDR validation event",+ SilenceUsage: true,+ RunE: runEndpointRapid7Validate,+}+func init() {rootCmd.AddCommand(endpointCmd)rootCmd.AddCommand(topLevelDoctorCmd)@@ -494,6 +523,7 @@ func init() {endpointCmd.AddCommand(endpointElasticCmd)endpointCmd.AddCommand(endpointDatadogCmd)endpointCmd.AddCommand(endpointSumoCmd)+ endpointCmd.AddCommand(endpointRapid7Cmd)endpointCmd.AddCommand(endpointIntegrationsCmd)endpointCmd.AddCommand(endpointHooksCmd)endpointCmd.AddCommand(endpointConfigCmd)@@ -513,6 +543,9 @@ func init() {endpointSumoCmd.AddCommand(endpointSumoPrintConfigCmd)endpointSumoCmd.AddCommand(endpointSumoInstallPackCmd)endpointSumoCmd.AddCommand(endpointSumoValidateCmd)+ endpointRapid7Cmd.AddCommand(endpointRapid7PrintConfigCmd)+ endpointRapid7Cmd.AddCommand(endpointRapid7InstallPackCmd)+ endpointRapid7Cmd.AddCommand(endpointRapid7ValidateCmd)endpointIntegrationsCmd.AddCommand(endpointIntegrationsValidateCmd)endpointIntegrationsCmd.AddCommand(endpointCoworkCmd)endpointIntegrationsCmd.AddCommand(endpointOpenClawCmd)@@ -605,6 +638,12 @@ func init() {c.Flags().StringVar(&endpointOpts.logPath, "log-path", "", "Runtime JSONL log path")}endpointSumoInstallPackCmd.Flags().StringVar(&endpointOpts.outputDir, "output", "", "Output directory for Sumo Logic content pack")+ for _, c := range []*cobra.Command{endpointRapid7PrintConfigCmd, endpointRapid7InstallPackCmd, endpointRapid7ValidateCmd} {+ c.Flags().BoolVar(&endpointOpts.userMode, "user", true, "Use per-user endpoint paths")+ c.Flags().BoolVar(&endpointOpts.systemMode, "system", false, "Use system endpoint paths and launch daemon")+ c.Flags().StringVar(&endpointOpts.logPath, "log-path", "", "Runtime JSONL log path")+ }+ endpointRapid7InstallPackCmd.Flags().StringVar(&endpointOpts.outputDir, "output", "", "Output directory for Rapid7 InsightIDR content pack")for _, c := range []*cobra.Command{endpointCoworkPrintConfigCmd, endpointCoworkSetupCmd, endpointCoworkStatusCmd, endpointCoworkValidateCmd} {c.Flags().BoolVar(&endpointOpts.userMode, "user", true, "Use per-user endpoint paths")c.Flags().BoolVar(&endpointOpts.systemMode, "system", false, "Use system endpoint paths and launch daemon")@@ -925,6 +964,31 @@ func runEndpointSumoValidate(cmd *cobra.Command, args []string) error {return nil}+func runEndpointRapid7InstallPack(cmd *cobra.Command, args []string) error {+ cfg := loadOrDefaultConfig()+ outputDir := endpointOpts.outputDir+ if outputDir == "" {+ outputDir = rapid7.DefaultOutputDir+ }+ if err := rapid7.InstallPack(outputDir, cfg.LogPath); err != nil {+ return err+ }+ fmt.Printf("Rapid7 InsightIDR content pack written to %s\n", outputDir)+ return nil+}++func runEndpointRapid7Validate(cmd *cobra.Command, args []string) error {+ cfg := loadOrDefaultConfig()+ path, err := writeValidationEvent(cfg, "rapid7")+ if err != nil {+ return err+ }+ fmt.Printf("Validation event written to %s\n", path)+ fmt.Println("Expected Rapid7 fields: vendor=beacon product=endpoint-agent destination.type=rapid7")+ fmt.Println(`Expected validation query: "Beacon endpoint Rapid7 validation event"`)+ return nil+}+func runEndpointOpenClawValidate() error {cfg := loadOrDefaultConfig()setup := func() {diff --git a/cli/beacon/cmd/endpoint_diagnostics.go b/cli/beacon/cmd/endpoint_diagnostics.goindex 061d6879..33bec8ea 100644--- a/cli/beacon/cmd/endpoint_diagnostics.go+++ b/cli/beacon/cmd/endpoint_diagnostics.go@@ -548,6 +548,9 @@ func syntheticEvent(destination string) schema.Event {case "sumo":mode = "http_source_jsonl"message = "Beacon endpoint Sumo validation event"+ case "rapid7":+ mode = "custom_logs_webhook_ndjson"+ message = "Beacon endpoint Rapid7 validation event"}event := schema.NewEvent(schema.NewEventOptions{Action: "agent.detected",diff --git a/cli/beacon/cmd/endpoint_test.go b/cli/beacon/cmd/endpoint_test.goindex 793ed418..a36a5287 100644--- a/cli/beacon/cmd/endpoint_test.go+++ b/cli/beacon/cmd/endpoint_test.go@@ -163,6 +163,30 @@ func TestEndpointSumoCommandsRegistered(t *testing.T) {}}+func TestEndpointRapid7CommandsRegistered(t *testing.T) {+ for _, path := range [][]string{+ {"rapid7", "print-config"},+ {"rapid7", "install-pack"},+ {"rapid7", "validate"},+ } {+ cmd, _, err := endpointCmd.Find(path)+ if err != nil {+ t.Fatalf("Find %v returned error: %v", path, err)+ }+ if cmd == nil || cmd.Use != path[len(path)-1] {+ t.Fatalf("rapid7 command %v not registered: %#v", path, cmd)+ }+ for _, name := range []string{"user", "system", "log-path"} {+ if cmd.Flags().Lookup(name) == nil {+ t.Fatalf("rapid7 command %v missing --%s flag", path, name)+ }+ }+ }+ if endpointRapid7InstallPackCmd.Flags().Lookup("output") == nil {+ t.Fatal("rapid7 install-pack command missing --output flag")+ }+}+func TestEnsureElasticPackDoesNotOverwriteExistingPack(t *testing.T) {dir := t.TempDir()composePath := filepath.Join(dir, "docker-compose.yml")@@ -461,6 +485,14 @@ func TestSyntheticEventDestinations(t *testing.T) {if event.Destination.Type != "pipeline" || event.Destination.Mode != "local_jsonl" {t.Fatalf("pipeline destination = %#v json=%s", event.Destination, data)}++ rapid7Event := syntheticEvent("rapid7")+ if rapid7Event.Destination.Type != "rapid7" || rapid7Event.Destination.Mode != "custom_logs_webhook_ndjson" {+ t.Fatalf("rapid7 destination = %#v", rapid7Event.Destination)+ }+ if rapid7Event.Message != "Beacon endpoint Rapid7 validation event" {+ t.Fatalf("rapid7 message = %q", rapid7Event.Message)+ }}func captureStdout(t *testing.T, fn func() error) (string, error) {diff --git a/cli/beacon/internal/endpoint/rapid7/pack/README.md b/cli/beacon/internal/endpoint/rapid7/pack/README.mdnew file mode 100644index 00000000..232c1936--- /dev/null+++ b/cli/beacon/internal/endpoint/rapid7/pack/README.md@@ -0,0 +1,107 @@+# Beacon Endpoint Agent Rapid7 Pack++This pack forwards Beacon endpoint JSONL events into Rapid7 InsightIDR Custom+Logs through a webhook event source. Beacon still writes one local source of+truth: `runtime.jsonl`. Rapid7 webhook URLs stay in your log shipper or+deployment tooling, not in Beacon endpoint configuration.++## Prerequisites++- Beacon endpoint installed and writing local JSONL.+- A Rapid7 InsightIDR Custom Logs event source configured with the Webhook+ collection method.+- The generated webhook URL stored securely as `RAPID7_WEBHOOK_URL` for smoke+ testing or in your customer-managed forwarder.++Recommended Rapid7 setup:++```text+Data Collection > Setup Event Source > Add Event Source > Raw Data > Custom Logs+Collection Method: Webhook+Name: Asymptote Agent Beacon+JSON Events Key: leave blank for Beacon NDJSON payloads+```++Rapid7 webhook URLs are bearer destinations. Protect them like credentials and+avoid committing them to endpoint configuration or source control.++## Install++Generate this pack:++```bash+beacon endpoint rapid7 install-pack --output ./beacon-rapid7-pack+```++The generated smoke-test script points at the Beacon log path selected by the+CLI:++- User mode: `~/.beacon/endpoint/logs/runtime.jsonl`+- System mode: `/var/log/beacon-agent/runtime.jsonl`+- Custom mode: the value passed with `--log-path`++For MDM or managed endpoint deployment, prefer Beacon system mode so your+customer-managed log shipper can tail `/var/log/beacon-agent/runtime.jsonl`+without per-user home directory ACLs.++## One-Shot Smoke Test++Use the generated script to upload the current file once. This is only for+validation; do not use it as production forwarding because it re-uploads the+whole file every time.++```bash+export RAPID7_WEBHOOK_URL="https://..."+./beacon-rapid7-pack/rapid7-upload-smoke-test.sh+```++You can override the Beacon log path:++```bash+export BEACON_LOG="$HOME/.beacon/endpoint/logs/runtime.jsonl"+```++The script sends `Content-Type: application/x-ndjson` and preserves one Beacon+event per line. Rapid7 InsightIDR Custom Logs treats each NDJSON line as an+individual event.++## Production Forwarding++For production, use your fleet's existing log shipper or a customer-managed+collector to tail `runtime.jsonl` and POST batches to the Rapid7 Custom Logs+webhook. The forwarder should:++- checkpoint file offsets,+- batch newline-delimited JSON records,+- preserve one Beacon event per line,+- retry transient failures without duplicating the whole file,+- keep the Rapid7 webhook URL outside Beacon endpoint configuration.++## Validate++Write a fresh Beacon validation event:++```bash+beacon endpoint rapid7 validate+```++Run the one-shot smoke test or wait for your production forwarder to ship the+new line. In Rapid7 Log Search, start with the Custom Logs event source and+search for:++```text+"Beacon endpoint Rapid7 validation event"+```++You can also confirm normalized Beacon fields are present:++```text+vendor=beacon product=endpoint-agent destination.type=rapid7+```++## Content Retention++Beacon content retention defaults to `full`, so prompt text, tool input, command+output, raw tool payloads, and other retained content may be forwarded to+Rapid7. Use Beacon's `metadata` or `redacted` content retention modes for+stricter deployments.diff --git a/cli/beacon/internal/endpoint/rapid7/pack/rapid7-upload-smoke-test.sh.tmpl b/cli/beacon/internal/endpoint/rapid7/pack/rapid7-upload-smoke-test.sh.tmplnew file mode 100644index 00000000..19eb9fc9--- /dev/null+++ b/cli/beacon/internal/endpoint/rapid7/pack/rapid7-upload-smoke-test.sh.tmpl@@ -0,0 +1,19 @@+#!/bin/sh+set -eu++BEACON_LOG="${BEACON_LOG:-{{LOG_PATH}}}"++if [ -z "${RAPID7_WEBHOOK_URL:-}" ]; then+ echo "RAPID7_WEBHOOK_URL is required. Use the Custom Logs webhook URL from Rapid7 InsightIDR." >&2+ exit 2+fi++if [ ! -r "$BEACON_LOG" ]; then+ echo "Beacon runtime log is not readable: $BEACON_LOG" >&2+ exit 2+fi++curl -X POST \+ -H "Content-Type: application/x-ndjson" \+ --data-binary @"$BEACON_LOG" \+ "$RAPID7_WEBHOOK_URL"diff --git a/cli/beacon/internal/endpoint/rapid7/rapid7.go b/cli/beacon/internal/endpoint/rapid7/rapid7.gonew file mode 100644index 00000000..9863b5a1--- /dev/null+++ b/cli/beacon/internal/endpoint/rapid7/rapid7.go@@ -0,0 +1,71 @@+package rapid7++import (+ "embed"+ "fmt"+ "os"+ "path/filepath"+ "strings"+)++//go:embed pack/*+var packFS embed.FS++const (+ DefaultLogPath = "/var/log/beacon-agent/runtime.jsonl"+ DefaultOutputDir = "beacon-rapid7-pack"+)++type File struct {+ Name string+ Content string+}++func Files() []File {+ return []File{+ {Name: "README.md", Content: mustRead("pack/README.md")},+ {Name: "rapid7-upload-smoke-test.sh", Content: UploadSmokeTest(DefaultLogPath)},+ {Name: "sample-event.jsonl", Content: mustRead("pack/sample-event.jsonl")},+ }+}++func UploadSmokeTest(logPath string) string {+ if logPath == "" {+ logPath = DefaultLogPath+ }+ return strings.ReplaceAll(mustRead("pack/rapid7-upload-smoke-test.sh.tmpl"), "{{LOG_PATH}}", logPath)+}++func InstallPack(outputDir, logPath string) error {+ if outputDir == "" {+ outputDir = DefaultOutputDir+ }+ if err := os.MkdirAll(outputDir, 0755); err != nil {+ return err+ }+ for _, file := range Files() {+ content := file.Content+ if file.Name == "rapid7-upload-smoke-test.sh" {+ content = UploadSmokeTest(logPath)+ }+ if err := os.WriteFile(filepath.Join(outputDir, file.Name), []byte(content), fileMode(file.Name)); err != nil {+ return err+ }+ }+ return nil+}++func fileMode(name string) os.FileMode {+ if strings.HasSuffix(name, ".sh") {+ return 0755+ }+ return 0644+}++func mustRead(path string) string {+ data, err := packFS.ReadFile(path)+ if err != nil {+ panic(fmt.Sprintf("rapid7 pack asset %s: %v", path, err))+ }+ return string(data)+}diff --git a/cli/beacon/internal/endpoint/rapid7/rapid7_test.go b/cli/beacon/internal/endpoint/rapid7/rapid7_test.gonew file mode 100644index 00000000..294c4823--- /dev/null+++ b/cli/beacon/internal/endpoint/rapid7/rapid7_test.go@@ -0,0 +1,104 @@+package rapid7++import (+ "bufio"+ "encoding/json"+ "os"+ "path/filepath"+ "strings"+ "testing"+)++func TestUploadSmokeTestUsesConfiguredPath(t *testing.T) {+ got := UploadSmokeTest("/tmp/beacon/runtime.jsonl")+ if !strings.Contains(got, "/tmp/beacon/runtime.jsonl") {+ t.Fatalf("script did not include configured path: %s", got)+ }+ if strings.Contains(got, "{{LOG_PATH}}") {+ t.Fatalf("script still contains template token: %s", got)+ }+ for _, want := range []string{+ "RAPID7_WEBHOOK_URL",+ "BEACON_LOG",+ "Content-Type: application/x-ndjson",+ "curl -X POST",+ "--data-binary @\"$BEACON_LOG\"",+ } {+ if !strings.Contains(got, want) {+ t.Fatalf("script missing %q: %s", want, got)+ }+ }+}++func TestInstallPackWritesExpectedFiles(t *testing.T) {+ dir := t.TempDir()+ if err := InstallPack(dir, "/tmp/beacon/runtime.jsonl"); err != nil {+ t.Fatalf("InstallPack returned error: %v", err)+ }+ for _, name := range []string{"README.md", "rapid7-upload-smoke-test.sh", "sample-event.jsonl"} {+ if _, err := os.Stat(filepath.Join(dir, name)); err != nil {+ t.Fatalf("expected %s: %v", name, err)+ }+ }+ scriptPath := filepath.Join(dir, "rapid7-upload-smoke-test.sh")+ script, err := os.ReadFile(scriptPath)+ if err != nil {+ t.Fatal(err)+ }+ if !strings.Contains(string(script), "/tmp/beacon/runtime.jsonl") {+ t.Fatalf("generated script missing configured log path: %s", script)+ }+ info, err := os.Stat(scriptPath)+ if err != nil {+ t.Fatal(err)+ }+ if info.Mode().Perm()&0111 == 0 {+ t.Fatalf("generated script should be executable, mode=%s", info.Mode())+ }+}++func TestSampleEventsCoverValidationHookAndOTelShapes(t *testing.T) {+ scanner := bufio.NewScanner(strings.NewReader(mustRead("pack/sample-event.jsonl")))+ var sawValidation, sawHook, sawOTel bool+ for scanner.Scan() {+ var doc map[string]interface{}+ if err := json.Unmarshal(scanner.Bytes(), &doc); err != nil {+ t.Fatalf("sample-event.jsonl is not valid JSONL: %v", err)+ }+ if destination, ok := doc["destination"].(map[string]interface{}); ok && destination["type"] == "rapid7" {+ sawValidation = true+ }+ if _, ok := doc["tool"].(map[string]interface{}); ok {+ sawHook = true+ }+ if raw, ok := doc["raw"].(map[string]interface{}); ok && raw["otel_signal"] != nil {+ sawOTel = true+ }+ }+ if err := scanner.Err(); err != nil {+ t.Fatal(err)+ }+ if !sawValidation || !sawHook || !sawOTel {+ t.Fatalf("sample events should include validation, hook-rich, and OTel shapes; validation=%t hook=%t otel=%t", sawValidation, sawHook, sawOTel)+ }+}++func TestPackREADMEMentionsRapid7SetupAndProductionForwarding(t *testing.T) {+ readme := mustRead("pack/README.md")+ for _, want := range []string{+ "beacon endpoint rapid7 validate",+ "Rapid7 InsightIDR Custom Logs",+ "Webhook",+ "JSON Events Key",+ "RAPID7_WEBHOOK_URL",+ "application/x-ndjson",+ "Beacon endpoint Rapid7 validation event",+ "vendor=beacon product=endpoint-agent destination.type=rapid7",+ "content retention",+ "/var/log/beacon-agent/runtime.jsonl",+ } {+ if !strings.Contains(readme, want) {+ t.Fatalf("pack README missing %q", want)+ }+ }+}diff --git a/packaging/macos/README.md b/packaging/macos/README.mdindex ee4d0309..82658063 100644--- a/packaging/macos/README.md+++ b/packaging/macos/README.md@@ -100,6 +100,21 @@ name `agentbeacon`, and fieldsuse a tailing shipper that checkpoints offsets, batches JSONL records, sendsgzip-compressed POST payloads, and avoids repeatedly uploading the whole file.+For Rapid7 InsightIDR, use the same local JSONL producer model and deploy a+customer-managed shipper that tails `/var/log/beacon-agent/runtime.jsonl` into a+Custom Logs webhook as NDJSON. Generate Beacon's Rapid7 content pack for setup+guidance, a one-shot smoke test, and sample events:++```bash+/opt/beacon/bin/beacon endpoint rapid7 install-pack --system --output ./beacon-rapid7-pack+/opt/beacon/bin/beacon endpoint rapid7 validate --system+```++Keep the Rapid7 webhook URL in deployment tooling or the forwarder environment,+not in Beacon endpoint configuration. For production forwarding, use a tailing+shipper that checkpoints offsets, batches JSONL records, and avoids repeatedly+uploading the whole file.+Environment variables take precedence, followed by MDM script parameters:```text