The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu
Go agent-ai-cliagent-harnessesagent-securitybrowser-extensionciclaude-codecloudcodexcursordetection-engineeringjevknowledge-basememorymobile-device-managementobservabilitysecuritysecurity-information-and-event-managementsecurity-toolstelemetrytraces
add s3 support
Justin D'Souza8006f31a4284
diff --git a/README.md b/README.mdindex 6c0568d0..809a9852 100644--- a/README.md+++ b/README.md@@ -106,6 +106,7 @@ enterprise-grade SIEMs.| [Local JSONL](https://docs.asymptotelabs.ai/cli/local-testing-logs) | Default endpoint log and local dashboard source || [Microsoft Sentinel](https://docs.asymptotelabs.ai/cli/siem-forwarding-microsoft-sentinel) | Azure Monitor Agent and Data Collection Rule content pack over local JSONL || [Rapid7 InsightIDR](https://docs.asymptotelabs.ai/cli/siem-forwarding-rapid7) | Custom Logs webhook content pack over local JSONL |+| AWS S3 | Vector content pack over local JSONL using customer-managed AWS credentials || [Splunk HEC](https://docs.asymptotelabs.ai/cli/siem-forwarding-splunk) | Optional endpoint forwarding during install or repair || [Sumo Logic](https://docs.asymptotelabs.ai/cli/siem-forwarding-sumo) | HTTP Logs & Metrics Source content pack over local JSONL || [Wazuh](https://docs.asymptotelabs.ai/cli/siem-forwarding-wazuh) | Localfile configuration and Beacon Wazuh content pack |diff --git a/cli/beacon/README.md b/cli/beacon/README.mdindex df57b6cc..72391ca0 100644--- a/cli/beacon/README.md+++ b/cli/beacon/README.md@@ -141,6 +141,20 @@ The Rapid7 pack keeps Beacon as a local JSONL producer and documents forwardingwebhook URL in your customer-managed shipper or deployment tooling, not inBeacon endpoint configuration.+## AWS S3++```bash+./beacon endpoint s3 print-config+./beacon endpoint s3 install-pack --output ./beacon-s3-pack+./beacon endpoint s3 validate+```++The S3 pack keeps Beacon as a local JSONL producer and documents forwarding+`runtime.jsonl` into an AWS S3 bucket with a customer-managed Vector host agent.+Store AWS credentials, profiles, IAM roles, bucket policies, lifecycle rules,+and encryption settings in AWS, Vector, or deployment tooling, not in Beacon+endpoint configuration.+## Microsoft Sentinel```bashdiff --git a/cli/beacon/cmd/endpoint.go b/cli/beacon/cmd/endpoint.goindex 89390c1e..42ce83db 100644--- a/cli/beacon/cmd/endpoint.go+++ b/cli/beacon/cmd/endpoint.go@@ -24,6 +24,7 @@ import ("github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/integrations/vscode""github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/lifecycle""github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/rapid7"+ "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/s3""github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/schema""github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/sentinel""github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/sumo"@@ -188,6 +189,11 @@ var endpointRapid7Cmd = &cobra.Command{Short: "Manage Rapid7 InsightIDR integration content",}+var endpointS3Cmd = &cobra.Command{+ Use: "s3",+ Short: "Manage AWS S3 forwarding content",+}+var endpointSentinelCmd = &cobra.Command{Use: "sentinel",Short: "Manage Microsoft Sentinel integration content",@@ -606,6 +612,35 @@ var endpointRapid7ValidateCmd = &cobra.Command{RunE: runEndpointRapid7Validate,}+var endpointS3PrintConfigCmd = &cobra.Command{+ Use: "print-config",+ Short: "Print an AWS CLI S3 smoke-test uploader for Beacon endpoint events",+ SilenceUsage: true,+ RunE: func(cmd *cobra.Command, args []string) error {+ cfg := loadOrDefaultConfig()+ snippet, err := s3.UploadSmokeTest(cfg.LogPath)+ if err != nil {+ return err+ }+ fmt.Print(snippet)+ return nil+ },+}++var endpointS3InstallPackCmd = &cobra.Command{+ Use: "install-pack",+ Short: "Write AWS S3 forwarding content to a directory",+ SilenceUsage: true,+ RunE: runEndpointS3InstallPack,+}++var endpointS3ValidateCmd = &cobra.Command{+ Use: "validate",+ Short: "Write and describe an AWS S3 validation event",+ SilenceUsage: true,+ RunE: runEndpointS3Validate,+}+var endpointSentinelPrintConfigCmd = &cobra.Command{Use: "print-config",Short: "Print a Sentinel DCR transform for Beacon endpoint events",@@ -651,6 +686,7 @@ func init() {endpointCmd.AddCommand(endpointDatadogCmd)endpointCmd.AddCommand(endpointSumoCmd)endpointCmd.AddCommand(endpointRapid7Cmd)+ endpointCmd.AddCommand(endpointS3Cmd)endpointCmd.AddCommand(endpointSentinelCmd)endpointCmd.AddCommand(endpointIntegrationsCmd)endpointCmd.AddCommand(endpointHooksCmd)@@ -674,6 +710,9 @@ func init() {endpointRapid7Cmd.AddCommand(endpointRapid7PrintConfigCmd)endpointRapid7Cmd.AddCommand(endpointRapid7InstallPackCmd)endpointRapid7Cmd.AddCommand(endpointRapid7ValidateCmd)+ endpointS3Cmd.AddCommand(endpointS3PrintConfigCmd)+ endpointS3Cmd.AddCommand(endpointS3InstallPackCmd)+ endpointS3Cmd.AddCommand(endpointS3ValidateCmd)endpointSentinelCmd.AddCommand(endpointSentinelPrintConfigCmd)endpointSentinelCmd.AddCommand(endpointSentinelInstallPackCmd)endpointSentinelCmd.AddCommand(endpointSentinelValidateCmd)@@ -782,6 +821,12 @@ func init() {c.Flags().StringVar(&endpointOpts.logPath, "log-path", "", "Runtime JSONL log path")}endpointRapid7InstallPackCmd.Flags().StringVar(&endpointOpts.outputDir, "output", "", "Output directory for Rapid7 InsightIDR content pack")+ for _, c := range []*cobra.Command{endpointS3PrintConfigCmd, endpointS3InstallPackCmd, endpointS3ValidateCmd} {+ c.Flags().BoolVar(&endpointOpts.userMode, "user", true, "Use per-user endpoint paths")+ c.Flags().BoolVar(&endpointOpts.systemMode, "system", false, "Use system endpoint paths and launch daemon")+ c.Flags().StringVar(&endpointOpts.logPath, "log-path", "", "Runtime JSONL log path")+ }+ endpointS3InstallPackCmd.Flags().StringVar(&endpointOpts.outputDir, "output", "", "Output directory for AWS S3 content pack")for _, c := range []*cobra.Command{endpointSentinelPrintConfigCmd, endpointSentinelInstallPackCmd, endpointSentinelValidateCmd} {c.Flags().BoolVar(&endpointOpts.userMode, "user", true, "Use per-user endpoint paths")c.Flags().BoolVar(&endpointOpts.systemMode, "system", false, "Use system endpoint paths and launch daemon")@@ -1234,6 +1279,32 @@ func runEndpointRapid7Validate(cmd *cobra.Command, args []string) error {return nil}+func runEndpointS3InstallPack(cmd *cobra.Command, args []string) error {+ cfg := loadOrDefaultConfig()+ outputDir := endpointOpts.outputDir+ if outputDir == "" {+ outputDir = s3.DefaultOutputDir+ }+ if err := s3.InstallPack(outputDir, cfg.LogPath); err != nil {+ return err+ }+ fmt.Printf("AWS S3 content pack written to %s\n", outputDir)+ return nil+}++func runEndpointS3Validate(cmd *cobra.Command, args []string) error {+ cfg := loadOrDefaultConfig()+ path, err := writeValidationEvent(cfg, "s3")+ if err != nil {+ return err+ }+ fmt.Printf("Validation event written to %s\n", path)+ fmt.Println("Expected S3 fields: vendor=beacon product=endpoint-agent destination.type=s3 destination.mode=aws_s3_jsonl")+ fmt.Println(`Confirm delivery with AWS CLI: aws s3 ls "s3://${BEACON_S3_BUCKET}/${BEACON_S3_PREFIX}/" --recursive`)+ fmt.Println(`Inspect an object with AWS CLI: aws s3 cp "s3://${BEACON_S3_BUCKET}/${BEACON_S3_PREFIX}/date=<date>/<object>.jsonl.gz" - | gzip -dc | grep "Beacon endpoint S3 validation event"`)+ return nil+}+func runEndpointSentinelInstallPack(cmd *cobra.Command, args []string) error {cfg := loadOrDefaultConfig()outputDir := endpointOpts.outputDirdiff --git a/cli/beacon/cmd/endpoint_diagnostics.go b/cli/beacon/cmd/endpoint_diagnostics.goindex ae6510ad..973d6596 100644--- a/cli/beacon/cmd/endpoint_diagnostics.go+++ b/cli/beacon/cmd/endpoint_diagnostics.go@@ -573,6 +573,9 @@ func syntheticEvent(destination string) schema.Event {case "rapid7":mode = "custom_logs_webhook_ndjson"message = "Beacon endpoint Rapid7 validation event"+ case "s3":+ mode = "aws_s3_jsonl"+ message = "Beacon endpoint S3 validation event"case "sentinel":mode = "azure_monitor_agent_custom_json_logs"message = "Beacon endpoint Sentinel validation event"diff --git a/cli/beacon/cmd/endpoint_test.go b/cli/beacon/cmd/endpoint_test.goindex e913d7a5..26d88f8d 100644--- a/cli/beacon/cmd/endpoint_test.go+++ b/cli/beacon/cmd/endpoint_test.go@@ -257,6 +257,30 @@ func TestEndpointRapid7CommandsRegistered(t *testing.T) {}}+func TestEndpointS3CommandsRegistered(t *testing.T) {+ for _, path := range [][]string{+ {"s3", "print-config"},+ {"s3", "install-pack"},+ {"s3", "validate"},+ } {+ cmd, _, err := endpointCmd.Find(path)+ if err != nil {+ t.Fatalf("Find %v returned error: %v", path, err)+ }+ if cmd == nil || cmd.Use != path[len(path)-1] {+ t.Fatalf("s3 command %v not registered: %#v", path, cmd)+ }+ for _, name := range []string{"user", "system", "log-path"} {+ if cmd.Flags().Lookup(name) == nil {+ t.Fatalf("s3 command %v missing --%s flag", path, name)+ }+ }+ }+ if endpointS3InstallPackCmd.Flags().Lookup("output") == nil {+ t.Fatal("s3 install-pack command missing --output flag")+ }+}+func TestEndpointSentinelCommandsRegistered(t *testing.T) {for _, path := range [][]string{{"sentinel", "print-config"},@@ -602,6 +626,49 @@ func TestSyntheticEventDestinations(t *testing.T) {if rapid7Event.Message != "Beacon endpoint Rapid7 validation event" {t.Fatalf("rapid7 message = %q", rapid7Event.Message)}++ s3Event := syntheticEvent("s3")+ if s3Event.Destination.Type != "s3" || s3Event.Destination.Mode != "aws_s3_jsonl" {+ t.Fatalf("s3 destination = %#v", s3Event.Destination)+ }+ if s3Event.Message != "Beacon endpoint S3 validation event" {+ t.Fatalf("s3 message = %q", s3Event.Message)+ }+}++func TestEndpointS3ValidatePrintsAWSCLIInspectionGuidance(t *testing.T) {+ oldLogPath := endpointOpts.logPath+ oldUserMode := endpointOpts.userMode+ oldSystemMode := endpointOpts.systemMode+ endpointOpts.logPath = filepath.Join(t.TempDir(), "runtime.jsonl")+ endpointOpts.userMode = true+ endpointOpts.systemMode = false+ t.Cleanup(func() {+ endpointOpts.logPath = oldLogPath+ endpointOpts.userMode = oldUserMode+ endpointOpts.systemMode = oldSystemMode+ })++ output, err := captureStdout(t, func() error {+ return runEndpointS3Validate(endpointS3ValidateCmd, nil)+ })+ if err != nil {+ t.Fatalf("runEndpointS3Validate returned error: %v", err)+ }+ for _, want := range []string{+ "Validation event written to",+ "destination.type=s3 destination.mode=aws_s3_jsonl",+ "aws s3 ls",+ "aws s3 cp",+ "Beacon endpoint S3 validation event",+ } {+ if !strings.Contains(output, want) {+ t.Fatalf("S3 validation output missing %q: %s", want, output)+ }+ }+ if strings.Contains(output, "verified delivery") {+ t.Fatalf("S3 validation output should not claim remote delivery verification: %s", output)+ }}func captureStdout(t *testing.T, fn func() error) (string, error) {diff --git a/cli/beacon/internal/endpoint/s3/pack/README.md b/cli/beacon/internal/endpoint/s3/pack/README.mdnew file mode 100644index 00000000..6dba692d--- /dev/null+++ b/cli/beacon/internal/endpoint/s3/pack/README.md@@ -0,0 +1,163 @@+# Beacon Endpoint Agent AWS S3 Pack++This pack forwards Beacon endpoint JSONL events into an AWS S3 bucket. Beacon+still writes one local source of truth: `runtime.jsonl`. AWS credentials,+profiles, IAM roles, bucket policies, object lifecycle rules, and server-side+encryption stay in AWS, Vector, or customer-managed deployment tooling, not in+Beacon endpoint configuration.++## Prerequisites++- Beacon endpoint installed and writing local JSONL.+- An AWS S3 bucket for Beacon runtime logs.+- An IAM role or credentials available through the standard AWS credential+ provider chain for the process running Vector or the AWS CLI smoke test.++Recommended S3 layout:++```text+s3://example-security-logs/beacon/runtime/date=YYYY-MM-DD/<timestamp>-<uuid>.jsonl.gz+```++Example least-privilege IAM policy for a dedicated Beacon prefix:++```json+{+ "Version": "2012-10-17",+ "Statement": [+ {+ "Effect": "Allow",+ "Action": ["s3:PutObject"],+ "Resource": "arn:aws:s3:::example-security-logs/beacon/runtime/*"+ }+ ]+}+```++Add `s3:PutObjectTagging`, KMS permissions, or bucket-specific conditions only+if your AWS controls require them. Configure bucket lifecycle, retention,+server-side encryption, and access logging in AWS.++## Install++Generate this pack:++```bash+beacon endpoint s3 install-pack --output ./beacon-s3-pack+```++The generated smoke-test script points at the Beacon log path selected by the+CLI:++- User mode: `~/.beacon/endpoint/logs/runtime.jsonl`+- System mode: `/var/log/beacon-agent/runtime.jsonl`+- Custom mode: the value passed with `--log-path`++For MDM or managed endpoint deployment, prefer Beacon system mode so your+customer-managed log shipper can tail `/var/log/beacon-agent/runtime.jsonl`+without per-user home directory ACLs.++The generated `vector.toml` uses the same selected Beacon log path.++## One-Shot Smoke Test++Use the generated script to upload the current file once. This is only for+validation; do not use it as production forwarding because it re-uploads the+whole file every time.++```bash+export BEACON_S3_BUCKET="example-security-logs"+export BEACON_S3_PREFIX="beacon/runtime"+export AWS_REGION="us-east-1"+./beacon-s3-pack/s3-upload-smoke-test.sh+```++The script uses `aws s3 cp` and the standard AWS credential provider chain. You+can provide credentials with an instance profile, SSO/profile configuration,+environment variables, or your managed endpoint secret tooling. Beacon does not+store AWS credentials.++Beacon does not store AWS credentials in endpoint configuration or generated+runtime state.++Confirm the uploaded object:++```bash+aws s3 ls "s3://${BEACON_S3_BUCKET}/${BEACON_S3_PREFIX}/smoke-tests/" --region "$AWS_REGION"+aws s3 cp "s3://${BEACON_S3_BUCKET}/${BEACON_S3_PREFIX}/smoke-tests/<object>.jsonl" - --region "$AWS_REGION" | grep "Beacon endpoint S3 validation event"+```++## Production Forwarding++For production, use the generated Vector config as a customer-managed host-agent+forwarding template. Beacon remains the local JSONL producer; Vector tails+`runtime.jsonl`, checkpoints file offsets in its `data_dir`, batches Beacon+events, and writes gzip-compressed newline-delimited JSON objects into AWS S3.++Install Vector using your normal endpoint management tooling, then copy the+generated config into Vector's config directory. On a macOS system-mode Beacon+deployment, the generated config tails `/var/log/beacon-agent/runtime.jsonl`:++```bash+sudo mkdir -p /etc/vector+sudo cp ./beacon-s3-pack/vector.toml /etc/vector/beacon-s3.toml+export BEACON_S3_BUCKET="example-security-logs"+export BEACON_S3_PREFIX="beacon/runtime"+export AWS_REGION="us-east-1"+vector validate /etc/vector/beacon-s3.toml+vector --config /etc/vector/beacon-s3.toml+```++In managed deployments, provide `BEACON_S3_BUCKET`, optional+`BEACON_S3_PREFIX`, `AWS_REGION`, and any AWS credential-provider settings+through the Vector service environment, host identity, or MDM/secret tooling.+Do not store AWS destination secrets in Beacon endpoint configuration.++The Vector template is intentionally simple and expects a Vector version with+the `file` source, `remap` transform, and `aws_s3` sink. It parses each Beacon+JSONL line and re-encodes the original Beacon event as NDJSON so S3 receives one+Beacon event per line, without a Vector wrapper.++The template uses date-partitioned `key_prefix`, `filename_time_format = "%s"`,+and `filename_append_uuid = true` so production forwarding does not overwrite+previous S3 objects. It also sets `compression = "gzip"`,+`content_encoding = "gzip"`, and `content_type = "application/x-ndjson"`.++If you adapt the config or use another forwarder, it should:++- checkpoint file offsets,+- batch newline-delimited JSON records,+- use non-overwriting object keys,+- retry transient failures without duplicating the whole file,+- keep AWS credentials, IAM roles, bucket policy, lifecycle, and encryption+ outside Beacon endpoint configuration.++## Validate++Write a fresh Beacon validation event:++```bash+beacon endpoint s3 validate+```++Run the one-shot smoke test or wait for your production forwarder to ship the+new line. Beacon can write the local validation event, but remote delivery must+be confirmed with AWS tooling:++```bash+aws s3 ls "s3://${BEACON_S3_BUCKET}/${BEACON_S3_PREFIX}/" --recursive --region "$AWS_REGION"+aws s3 cp "s3://${BEACON_S3_BUCKET}/${BEACON_S3_PREFIX}/date=<date>/<object>.jsonl.gz" - --region "$AWS_REGION" | gzip -dc | grep "Beacon endpoint S3 validation event"+```++Expected validation fields:++```text+vendor=beacon product=endpoint-agent destination.type=s3 destination.mode=aws_s3_jsonl+```++## Content Retention++Beacon content retention defaults to `full`, so prompt text, tool input, command+output, raw tool payloads, and other retained content may be forwarded to S3.+Use Beacon's `metadata` or `redacted` content retention modes for stricter+deployments.diff --git a/cli/beacon/internal/endpoint/s3/pack/s3-upload-smoke-test.sh.tmpl b/cli/beacon/internal/endpoint/s3/pack/s3-upload-smoke-test.sh.tmplnew file mode 100644index 00000000..c5b7de09--- /dev/null+++ b/cli/beacon/internal/endpoint/s3/pack/s3-upload-smoke-test.sh.tmpl@@ -0,0 +1,33 @@+#!/bin/sh+set -eu++BEACON_LOG="${BEACON_LOG:-{{LOG_PATH}}}"+BEACON_S3_PREFIX="${BEACON_S3_PREFIX:-beacon/runtime}"+AWS_REGION="${AWS_REGION:-us-east-1}"++if [ -z "${BEACON_S3_BUCKET:-}" ]; then+ echo "BEACON_S3_BUCKET is required." >&2+ exit 2+fi++if ! command -v aws >/dev/null 2>&1; then+ echo "aws CLI is required for this smoke test." >&2+ exit 2+fi++if [ ! -r "$BEACON_LOG" ]; then+ echo "Beacon runtime log is not readable: $BEACON_LOG" >&2+ exit 2+fi++prefix="${BEACON_S3_PREFIX%/}"+timestamp="$(date -u +%Y%m%dT%H%M%SZ)"+key="${prefix}/smoke-tests/beacon-runtime-${timestamp}.jsonl"++aws s3 cp "$BEACON_LOG" "s3://${BEACON_S3_BUCKET}/${key}" \+ --region "$AWS_REGION" \+ --content-type "application/x-ndjson"++echo "Uploaded one-shot Beacon log copy to s3://${BEACON_S3_BUCKET}/${key}"+echo "AWS credentials were resolved by the standard AWS credential provider chain."+echo "This script is for validation only; use vector.toml for production forwarding."diff --git a/cli/beacon/internal/endpoint/s3/pack/vector.toml.tmpl b/cli/beacon/internal/endpoint/s3/pack/vector.toml.tmplnew file mode 100644index 00000000..dbbf445c--- /dev/null+++ b/cli/beacon/internal/endpoint/s3/pack/vector.toml.tmpl@@ -0,0 +1,48 @@+# Beacon Endpoint Agent -> AWS S3+#+# Run Vector as a customer-managed host agent. Keep AWS credentials, profiles,+# instance roles, and bucket policy in AWS or deployment tooling, not in Beacon+# config. This template relies on the standard AWS credential provider chain.++data_dir = "/var/lib/vector/beacon-s3"++[sources.beacon_runtime]+type = "file"+include = ["{{LOG_PATH}}"]+read_from = "end"++[transforms.beacon_json]+type = "remap"+inputs = ["beacon_runtime"]+source = '''+. = parse_json!(.message)+'''++[sinks.beacon_s3]+type = "aws_s3"+inputs = ["beacon_json"]+bucket = "${BEACON_S3_BUCKET}"+region = "${AWS_REGION:-us-east-1}"+key_prefix = "${BEACON_S3_PREFIX:-beacon/runtime}/date=%F/"+filename_time_format = "%s"+filename_append_uuid = true+filename_extension = "jsonl.gz"+compression = "gzip"+content_encoding = "gzip"+content_type = "application/x-ndjson"+storage_class = "${BEACON_S3_STORAGE_CLASS:-STANDARD}"++[sinks.beacon_s3.encoding]+codec = "json"++[sinks.beacon_s3.framing]+method = "newline_delimited"++[sinks.beacon_s3.batch]+max_bytes = 10000000+timeout_secs = 300++[sinks.beacon_s3.request]+retry_attempts = 10+retry_initial_backoff_secs = 1+retry_max_duration_secs = 300diff --git a/cli/beacon/internal/endpoint/s3/s3.go b/cli/beacon/internal/endpoint/s3/s3.gonew file mode 100644index 00000000..4cc215ce--- /dev/null+++ b/cli/beacon/internal/endpoint/s3/s3.go@@ -0,0 +1,113 @@+package s3++import (+ "embed"+ "fmt"+ "io/fs"++ "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/siempack"+)++//go:embed pack/*+var packFS embed.FS++const (+ DefaultLogPath = "/var/log/beacon-agent/runtime.jsonl"+ DefaultOutputDir = "beacon-s3-pack"+)++type File struct {+ Name string+ Content string+ TemplateLogPath bool+ JSONEscape bool+}++// readAsset reads an embedded asset using the module-level packFS and returns its+// contents or an error.+func readAsset(path string) (string, error) {+ data, err := siempack.ReadFile(packFS, path)+ if err != nil {+ return "", fmt.Errorf("s3 pack asset %q: %w", path, err)+ }+ return data, nil+}++// mustRead returns the embedded asset at path or panics. Retained for test use+// where the embedded FS is always present; all production code uses readAsset.+func mustRead(path string) string {+ s, err := readAsset(path)+ if err != nil {+ panic(err.Error())+ }+ return s+}++// uploadSmokeTestFromFS renders the S3 upload smoke-test script using the supplied FS.+func uploadSmokeTestFromFS(fsys fs.FS, logPath string) (string, error) {+ if logPath == "" {+ logPath = DefaultLogPath+ }+ content, err := siempack.ReadFile(fsys, "pack/s3-upload-smoke-test.sh.tmpl")+ if err != nil {+ return "", fmt.Errorf("s3 pack asset %q: %w", "pack/s3-upload-smoke-test.sh.tmpl", err)+ }+ return siempack.RenderLogPath(content, logPath), nil+}++// filesFromFS builds the full file list from the supplied FS, propagating any+// read errors instead of panicking. Template substitution of LOG_PATH is+// deferred to siempack.Install via the TemplateLogPath field.+func filesFromFS(fsys fs.FS) ([]File, error) {+ readme, err := siempack.ReadFile(fsys, "pack/README.md")+ if err != nil {+ return nil, fmt.Errorf("s3 pack asset %q: %w", "pack/README.md", err)+ }+ smokeTest, err := siempack.ReadFile(fsys, "pack/s3-upload-smoke-test.sh.tmpl")+ if err != nil {+ return nil, fmt.Errorf("s3 pack asset %q: %w", "pack/s3-upload-smoke-test.sh.tmpl", err)+ }+ sample, err := siempack.ReadFile(fsys, "pack/sample-event.jsonl")+ if err != nil {+ return nil, fmt.Errorf("s3 pack asset %q: %w", "pack/sample-event.jsonl", err)+ }+ vector, err := siempack.ReadFile(fsys, "pack/vector.toml.tmpl")+ if err != nil {+ return nil, fmt.Errorf("s3 pack asset %q: %w", "pack/vector.toml.tmpl", err)+ }+ return []File{+ {Name: "README.md", Content: readme},+ {Name: "s3-upload-smoke-test.sh", Content: smokeTest, TemplateLogPath: true},+ {Name: "sample-event.jsonl", Content: sample},+ {Name: "vector.toml", Content: vector, TemplateLogPath: true},+ }, nil+}++// Files returns all pack files, propagating any embedded asset read error.+func Files() ([]File, error) {+ return filesFromFS(packFS)+}++// UploadSmokeTest returns the S3 upload smoke-test script with logPath substituted.+func UploadSmokeTest(logPath string) (string, error) {+ return uploadSmokeTestFromFS(packFS, logPath)+}++// InstallPack writes the pack files to outputDir with logPath substituted.+func InstallPack(outputDir, logPath string) error {+ if outputDir == "" {+ outputDir = DefaultOutputDir+ }+ if logPath == "" {+ logPath = DefaultLogPath+ }+ files, err := filesFromFS(packFS)+ if err != nil {+ return err+ }+ sfiles := make([]siempack.File, 0, len(files))+ for _, file := range files {+ sfiles = append(sfiles, siempack.File(file))+ }+ return siempack.Install(outputDir, sfiles, logPath)+}diff --git a/cli/beacon/internal/endpoint/s3/s3_test.go b/cli/beacon/internal/endpoint/s3/s3_test.gonew file mode 100644index 00000000..4f4cab61--- /dev/null+++ b/cli/beacon/internal/endpoint/s3/s3_test.go@@ -0,0 +1,236 @@+package s3++import (+ "bufio"+ "encoding/json"+ "os"+ "path/filepath"+ "strings"+ "testing"+ "testing/fstest"+)++func TestUploadSmokeTestUsesConfiguredPath(t *testing.T) {+ got, err := UploadSmokeTest("/tmp/beacon/runtime.jsonl")+ if err != nil {+ t.Fatalf("UploadSmokeTest returned unexpected error: %v", err)+ }+ if !strings.Contains(got, "/tmp/beacon/runtime.jsonl") {+ t.Fatalf("script did not include configured path: %s", got)+ }+ if strings.Contains(got, "{{LOG_PATH}}") {+ t.Fatalf("script still contains template token: %s", got)+ }+ for _, want := range []string{+ "BEACON_S3_BUCKET",+ "BEACON_S3_PREFIX",+ "AWS_REGION",+ "aws s3 cp",+ "--content-type \"application/x-ndjson\"",+ "standard AWS credential provider chain",+ "validation only",+ } {+ if !strings.Contains(got, want) {+ t.Fatalf("script missing %q: %s", want, got)+ }+ }+}++func TestInstallPackWritesExpectedFiles(t *testing.T) {+ dir := t.TempDir()+ if err := InstallPack(dir, "/tmp/beacon/runtime.jsonl"); err != nil {+ t.Fatalf("InstallPack returned error: %v", err)+ }+ for _, name := range []string{"README.md", "s3-upload-smoke-test.sh", "sample-event.jsonl", "vector.toml"} {+ if _, err := os.Stat(filepath.Join(dir, name)); err != nil {+ t.Fatalf("expected %s: %v", name, err)+ }+ }+ scriptPath := filepath.Join(dir, "s3-upload-smoke-test.sh")+ script, err := os.ReadFile(scriptPath)+ if err != nil {+ t.Fatal(err)+ }+ if !strings.Contains(string(script), "/tmp/beacon/runtime.jsonl") {+ t.Fatalf("generated script missing configured log path: %s", script)+ }+ info, err := os.Stat(scriptPath)+ if err != nil {+ t.Fatal(err)+ }+ if info.Mode().Perm()&0111 == 0 {+ t.Fatalf("generated script should be executable, mode=%s", info.Mode())+ }+ vectorPath := filepath.Join(dir, "vector.toml")+ vectorConfig, err := os.ReadFile(vectorPath)+ if err != nil {+ t.Fatal(err)+ }+ if !strings.Contains(string(vectorConfig), "/tmp/beacon/runtime.jsonl") {+ t.Fatalf("generated vector config missing configured log path: %s", vectorConfig)+ }+ info, err = os.Stat(vectorPath)+ if err != nil {+ t.Fatal(err)+ }+ if info.Mode().Perm() != 0644 {+ t.Fatalf("generated vector config should be 0644, mode=%s", info.Mode().Perm())+ }+}++func TestVectorConfigUsesAWSS3SinkAndPreservesJSONShape(t *testing.T) {+ got := mustRead("pack/vector.toml.tmpl")+ for _, want := range []string{+ `include = ["{{LOG_PATH}}"]`,+ `read_from = "end"`,+ `. = parse_json!(.message)`,+ `type = "aws_s3"`,+ `bucket = "${BEACON_S3_BUCKET}"`,+ `region = "${AWS_REGION:-us-east-1}"`,+ `key_prefix = "${BEACON_S3_PREFIX:-beacon/runtime}/date=%F/"`,+ `filename_time_format = "%s"`,+ `filename_append_uuid = true`,+ `filename_extension = "jsonl.gz"`,+ `compression = "gzip"`,+ `content_encoding = "gzip"`,+ `content_type = "application/x-ndjson"`,+ `codec = "json"`,+ `method = "newline_delimited"`,+ `max_bytes = 10000000`,+ `retry_attempts = 10`,+ } {+ if !strings.Contains(got, want) {+ t.Fatalf("vector config missing %q: %s", want, got)+ }+ }+}++func TestSampleEventsCoverValidationHookAndOTelShapes(t *testing.T) {+ scanner := bufio.NewScanner(strings.NewReader(mustRead("pack/sample-event.jsonl")))+ var sawValidation, sawHook, sawOTel bool+ for scanner.Scan() {+ var doc map[string]interface{}+ if err := json.Unmarshal(scanner.Bytes(), &doc); err != nil {+ t.Fatalf("sample-event.jsonl is not valid JSONL: %v", err)+ }+ if destination, ok := doc["destination"].(map[string]interface{}); ok && destination["type"] == "s3" {+ sawValidation = true+ }+ if _, ok := doc["tool"].(map[string]interface{}); ok {+ sawHook = true+ }+ if raw, ok := doc["raw"].(map[string]interface{}); ok && raw["otel_signal"] != nil {+ sawOTel = true+ }+ }+ if err := scanner.Err(); err != nil {+ t.Fatal(err)+ }+ if !sawValidation || !sawHook || !sawOTel {+ t.Fatalf("sample events should include validation, hook-rich, and OTel shapes; validation=%t hook=%t otel=%t", sawValidation, sawHook, sawOTel)+ }+}++func TestPackREADMEMentionsS3SetupAndProductionForwarding(t *testing.T) {+ readme := mustRead("pack/README.md")+ for _, want := range []string{+ "beacon endpoint s3 validate",+ "AWS S3 bucket",+ "standard AWS credential provider chain",+ "least-privilege IAM policy",+ "BEACON_S3_BUCKET",+ "BEACON_S3_PREFIX",+ "aws s3 ls",+ "aws s3 cp",+ "Beacon endpoint S3 validation event",+ "vendor=beacon product=endpoint-agent destination.type=s3 destination.mode=aws_s3_jsonl",+ "vector.toml",+ "customer-managed host-agent",+ "without a Vector wrapper",+ "content retention",+ "/var/log/beacon-agent/runtime.jsonl",+ "Beacon does not store AWS credentials",+ "server-side encryption",+ } {+ if !strings.Contains(readme, want) {+ t.Fatalf("pack README missing %q", want)+ }+ }+}++func TestFiles_NoError(t *testing.T) {+ files, err := Files()+ if err != nil {+ t.Fatalf("Files() returned unexpected error: %v", err)+ }+ if len(files) == 0 {+ t.Fatal("Files() returned no files")+ }+}++func TestFiles_ContainsAllRequiredNames(t *testing.T) {+ files, err := Files()+ if err != nil {+ t.Fatal(err)+ }+ names := make(map[string]bool)+ for _, f := range files {+ names[f.Name] = true+ }+ for _, required := range []string{+ "README.md", "s3-upload-smoke-test.sh", "sample-event.jsonl", "vector.toml",+ } {+ if !names[required] {+ t.Errorf("Files() missing required file %q", required)+ }+ }+}++func TestFilesFromFS_PropagatesReadError(t *testing.T) {+ emptyFS := fstest.MapFS{}+ _, err := filesFromFS(emptyFS)+ if err == nil {+ t.Fatal("filesFromFS with empty FS should return an error")+ }+ if !strings.Contains(err.Error(), "s3 pack asset") {+ t.Fatalf("error should identify the pack source, got: %v", err)+ }+}++func TestUploadSmokeTestFromFS_ErrorOnMissingAsset(t *testing.T) {+ emptyFS := fstest.MapFS{}+ _, err := uploadSmokeTestFromFS(emptyFS, "/some/path.jsonl")+ if err == nil {+ t.Fatal("uploadSmokeTestFromFS with empty FS should return error")+ }+ if !strings.Contains(err.Error(), "s3 pack asset") {+ t.Fatalf("error should identify the pack source, got: %v", err)+ }+}++func TestInstallPack_ErrorOnWriteFailure(t *testing.T) {+ if os.Getuid() == 0 {+ t.Skip("running as root: filesystem permission restrictions do not apply")+ }+ dir := t.TempDir()+ if err := os.Chmod(dir, 0555); err != nil {+ t.Skip("cannot set directory permissions:", err)+ }+ defer os.Chmod(dir, 0755)++ subdir := filepath.Join(dir, "output")+ err := InstallPack(subdir, DefaultLogPath)+ if err == nil {+ t.Fatal("InstallPack into read-only directory should return error")+ }+}++func TestUploadSmokeTest_DefaultLogPath(t *testing.T) {+ got, err := UploadSmokeTest("")+ if err != nil {+ t.Fatalf("UploadSmokeTest with empty path returned error: %v", err)+ }+ if !strings.Contains(got, DefaultLogPath) {+ t.Fatalf("UploadSmokeTest with empty logPath should use DefaultLogPath %q, got: %s", DefaultLogPath, got)+ }+}