Skip to content

The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu

add s3 support

Justin D'Souza8006f31a4284
diff --git a/README.md b/README.md
index 6c0568d0..809a9852 100644
--- a/README.md
+++ b/README.md
@@ -106,6 +106,7 @@ enterprise-grade SIEMs.
| [Local JSONL](https://docs.asymptotelabs.ai/cli/local-testing-logs) | Default endpoint log and local dashboard source |
| [Microsoft Sentinel](https://docs.asymptotelabs.ai/cli/siem-forwarding-microsoft-sentinel) | Azure Monitor Agent and Data Collection Rule content pack over local JSONL |
| [Rapid7 InsightIDR](https://docs.asymptotelabs.ai/cli/siem-forwarding-rapid7) | Custom Logs webhook content pack over local JSONL |
+| AWS S3 | Vector content pack over local JSONL using customer-managed AWS credentials |
| [Splunk HEC](https://docs.asymptotelabs.ai/cli/siem-forwarding-splunk) | Optional endpoint forwarding during install or repair |
| [Sumo Logic](https://docs.asymptotelabs.ai/cli/siem-forwarding-sumo) | HTTP Logs & Metrics Source content pack over local JSONL |
| [Wazuh](https://docs.asymptotelabs.ai/cli/siem-forwarding-wazuh) | Localfile configuration and Beacon Wazuh content pack |
diff --git a/cli/beacon/README.md b/cli/beacon/README.md
index df57b6cc..72391ca0 100644
--- a/cli/beacon/README.md
+++ b/cli/beacon/README.md
@@ -141,6 +141,20 @@ The Rapid7 pack keeps Beacon as a local JSONL producer and documents forwarding
webhook URL in your customer-managed shipper or deployment tooling, not in
Beacon endpoint configuration.
+## AWS S3
+
+```bash
+./beacon endpoint s3 print-config
+./beacon endpoint s3 install-pack --output ./beacon-s3-pack
+./beacon endpoint s3 validate
+```
+
+The S3 pack keeps Beacon as a local JSONL producer and documents forwarding
+`runtime.jsonl` into an AWS S3 bucket with a customer-managed Vector host agent.
+Store AWS credentials, profiles, IAM roles, bucket policies, lifecycle rules,
+and encryption settings in AWS, Vector, or deployment tooling, not in Beacon
+endpoint configuration.
+
## Microsoft Sentinel
```bash
diff --git a/cli/beacon/cmd/endpoint.go b/cli/beacon/cmd/endpoint.go
index 89390c1e..42ce83db 100644
--- a/cli/beacon/cmd/endpoint.go
+++ b/cli/beacon/cmd/endpoint.go
@@ -24,6 +24,7 @@ import (
"github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/integrations/vscode"
"github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/lifecycle"
"github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/rapid7"
+ "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/s3"
"github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/schema"
"github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/sentinel"
"github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/sumo"
@@ -188,6 +189,11 @@ var endpointRapid7Cmd = &cobra.Command{
Short: "Manage Rapid7 InsightIDR integration content",
}
+var endpointS3Cmd = &cobra.Command{
+ Use: "s3",
+ Short: "Manage AWS S3 forwarding content",
+}
+
var endpointSentinelCmd = &cobra.Command{
Use: "sentinel",
Short: "Manage Microsoft Sentinel integration content",
@@ -606,6 +612,35 @@ var endpointRapid7ValidateCmd = &cobra.Command{
RunE: runEndpointRapid7Validate,
}
+var endpointS3PrintConfigCmd = &cobra.Command{
+ Use: "print-config",
+ Short: "Print an AWS CLI S3 smoke-test uploader for Beacon endpoint events",
+ SilenceUsage: true,
+ RunE: func(cmd *cobra.Command, args []string) error {
+ cfg := loadOrDefaultConfig()
+ snippet, err := s3.UploadSmokeTest(cfg.LogPath)
+ if err != nil {
+ return err
+ }
+ fmt.Print(snippet)
+ return nil
+ },
+}
+
+var endpointS3InstallPackCmd = &cobra.Command{
+ Use: "install-pack",
+ Short: "Write AWS S3 forwarding content to a directory",
+ SilenceUsage: true,
+ RunE: runEndpointS3InstallPack,
+}
+
+var endpointS3ValidateCmd = &cobra.Command{
+ Use: "validate",
+ Short: "Write and describe an AWS S3 validation event",
+ SilenceUsage: true,
+ RunE: runEndpointS3Validate,
+}
+
var endpointSentinelPrintConfigCmd = &cobra.Command{
Use: "print-config",
Short: "Print a Sentinel DCR transform for Beacon endpoint events",
@@ -651,6 +686,7 @@ func init() {
endpointCmd.AddCommand(endpointDatadogCmd)
endpointCmd.AddCommand(endpointSumoCmd)
endpointCmd.AddCommand(endpointRapid7Cmd)
+ endpointCmd.AddCommand(endpointS3Cmd)
endpointCmd.AddCommand(endpointSentinelCmd)
endpointCmd.AddCommand(endpointIntegrationsCmd)
endpointCmd.AddCommand(endpointHooksCmd)
@@ -674,6 +710,9 @@ func init() {
endpointRapid7Cmd.AddCommand(endpointRapid7PrintConfigCmd)
endpointRapid7Cmd.AddCommand(endpointRapid7InstallPackCmd)
endpointRapid7Cmd.AddCommand(endpointRapid7ValidateCmd)
+ endpointS3Cmd.AddCommand(endpointS3PrintConfigCmd)
+ endpointS3Cmd.AddCommand(endpointS3InstallPackCmd)
+ endpointS3Cmd.AddCommand(endpointS3ValidateCmd)
endpointSentinelCmd.AddCommand(endpointSentinelPrintConfigCmd)
endpointSentinelCmd.AddCommand(endpointSentinelInstallPackCmd)
endpointSentinelCmd.AddCommand(endpointSentinelValidateCmd)
@@ -782,6 +821,12 @@ func init() {
c.Flags().StringVar(&endpointOpts.logPath, "log-path", "", "Runtime JSONL log path")
}
endpointRapid7InstallPackCmd.Flags().StringVar(&endpointOpts.outputDir, "output", "", "Output directory for Rapid7 InsightIDR content pack")
+ for _, c := range []*cobra.Command{endpointS3PrintConfigCmd, endpointS3InstallPackCmd, endpointS3ValidateCmd} {
+ c.Flags().BoolVar(&endpointOpts.userMode, "user", true, "Use per-user endpoint paths")
+ c.Flags().BoolVar(&endpointOpts.systemMode, "system", false, "Use system endpoint paths and launch daemon")
+ c.Flags().StringVar(&endpointOpts.logPath, "log-path", "", "Runtime JSONL log path")
+ }
+ endpointS3InstallPackCmd.Flags().StringVar(&endpointOpts.outputDir, "output", "", "Output directory for AWS S3 content pack")
for _, c := range []*cobra.Command{endpointSentinelPrintConfigCmd, endpointSentinelInstallPackCmd, endpointSentinelValidateCmd} {
c.Flags().BoolVar(&endpointOpts.userMode, "user", true, "Use per-user endpoint paths")
c.Flags().BoolVar(&endpointOpts.systemMode, "system", false, "Use system endpoint paths and launch daemon")
@@ -1234,6 +1279,32 @@ func runEndpointRapid7Validate(cmd *cobra.Command, args []string) error {
return nil
}
+func runEndpointS3InstallPack(cmd *cobra.Command, args []string) error {
+ cfg := loadOrDefaultConfig()
+ outputDir := endpointOpts.outputDir
+ if outputDir == "" {
+ outputDir = s3.DefaultOutputDir
+ }
+ if err := s3.InstallPack(outputDir, cfg.LogPath); err != nil {
+ return err
+ }
+ fmt.Printf("AWS S3 content pack written to %s\n", outputDir)
+ return nil
+}
+
+func runEndpointS3Validate(cmd *cobra.Command, args []string) error {
+ cfg := loadOrDefaultConfig()
+ path, err := writeValidationEvent(cfg, "s3")
+ if err != nil {
+ return err
+ }
+ fmt.Printf("Validation event written to %s\n", path)
+ fmt.Println("Expected S3 fields: vendor=beacon product=endpoint-agent destination.type=s3 destination.mode=aws_s3_jsonl")
+ fmt.Println(`Confirm delivery with AWS CLI: aws s3 ls "s3://${BEACON_S3_BUCKET}/${BEACON_S3_PREFIX}/" --recursive`)
+ fmt.Println(`Inspect an object with AWS CLI: aws s3 cp "s3://${BEACON_S3_BUCKET}/${BEACON_S3_PREFIX}/date=<date>/<object>.jsonl.gz" - | gzip -dc | grep "Beacon endpoint S3 validation event"`)
+ return nil
+}
+
func runEndpointSentinelInstallPack(cmd *cobra.Command, args []string) error {
cfg := loadOrDefaultConfig()
outputDir := endpointOpts.outputDir
diff --git a/cli/beacon/cmd/endpoint_diagnostics.go b/cli/beacon/cmd/endpoint_diagnostics.go
index ae6510ad..973d6596 100644
--- a/cli/beacon/cmd/endpoint_diagnostics.go
+++ b/cli/beacon/cmd/endpoint_diagnostics.go
@@ -573,6 +573,9 @@ func syntheticEvent(destination string) schema.Event {
case "rapid7":
mode = "custom_logs_webhook_ndjson"
message = "Beacon endpoint Rapid7 validation event"
+ case "s3":
+ mode = "aws_s3_jsonl"
+ message = "Beacon endpoint S3 validation event"
case "sentinel":
mode = "azure_monitor_agent_custom_json_logs"
message = "Beacon endpoint Sentinel validation event"
diff --git a/cli/beacon/cmd/endpoint_test.go b/cli/beacon/cmd/endpoint_test.go
index e913d7a5..26d88f8d 100644
--- a/cli/beacon/cmd/endpoint_test.go
+++ b/cli/beacon/cmd/endpoint_test.go
@@ -257,6 +257,30 @@ func TestEndpointRapid7CommandsRegistered(t *testing.T) {
}
}
+func TestEndpointS3CommandsRegistered(t *testing.T) {
+ for _, path := range [][]string{
+ {"s3", "print-config"},
+ {"s3", "install-pack"},
+ {"s3", "validate"},
+ } {
+ cmd, _, err := endpointCmd.Find(path)
+ if err != nil {
+ t.Fatalf("Find %v returned error: %v", path, err)
+ }
+ if cmd == nil || cmd.Use != path[len(path)-1] {
+ t.Fatalf("s3 command %v not registered: %#v", path, cmd)
+ }
+ for _, name := range []string{"user", "system", "log-path"} {
+ if cmd.Flags().Lookup(name) == nil {
+ t.Fatalf("s3 command %v missing --%s flag", path, name)
+ }
+ }
+ }
+ if endpointS3InstallPackCmd.Flags().Lookup("output") == nil {
+ t.Fatal("s3 install-pack command missing --output flag")
+ }
+}
+
func TestEndpointSentinelCommandsRegistered(t *testing.T) {
for _, path := range [][]string{
{"sentinel", "print-config"},
@@ -602,6 +626,49 @@ func TestSyntheticEventDestinations(t *testing.T) {
if rapid7Event.Message != "Beacon endpoint Rapid7 validation event" {
t.Fatalf("rapid7 message = %q", rapid7Event.Message)
}
+
+ s3Event := syntheticEvent("s3")
+ if s3Event.Destination.Type != "s3" || s3Event.Destination.Mode != "aws_s3_jsonl" {
+ t.Fatalf("s3 destination = %#v", s3Event.Destination)
+ }
+ if s3Event.Message != "Beacon endpoint S3 validation event" {
+ t.Fatalf("s3 message = %q", s3Event.Message)
+ }
+}
+
+func TestEndpointS3ValidatePrintsAWSCLIInspectionGuidance(t *testing.T) {
+ oldLogPath := endpointOpts.logPath
+ oldUserMode := endpointOpts.userMode
+ oldSystemMode := endpointOpts.systemMode
+ endpointOpts.logPath = filepath.Join(t.TempDir(), "runtime.jsonl")
+ endpointOpts.userMode = true
+ endpointOpts.systemMode = false
+ t.Cleanup(func() {
+ endpointOpts.logPath = oldLogPath
+ endpointOpts.userMode = oldUserMode
+ endpointOpts.systemMode = oldSystemMode
+ })
+
+ output, err := captureStdout(t, func() error {
+ return runEndpointS3Validate(endpointS3ValidateCmd, nil)
+ })
+ if err != nil {
+ t.Fatalf("runEndpointS3Validate returned error: %v", err)
+ }
+ for _, want := range []string{
+ "Validation event written to",
+ "destination.type=s3 destination.mode=aws_s3_jsonl",
+ "aws s3 ls",
+ "aws s3 cp",
+ "Beacon endpoint S3 validation event",
+ } {
+ if !strings.Contains(output, want) {
+ t.Fatalf("S3 validation output missing %q: %s", want, output)
+ }
+ }
+ if strings.Contains(output, "verified delivery") {
+ t.Fatalf("S3 validation output should not claim remote delivery verification: %s", output)
+ }
}
func captureStdout(t *testing.T, fn func() error) (string, error) {
diff --git a/cli/beacon/internal/endpoint/s3/pack/README.md b/cli/beacon/internal/endpoint/s3/pack/README.md
new file mode 100644
index 00000000..6dba692d
--- /dev/null
+++ b/cli/beacon/internal/endpoint/s3/pack/README.md
@@ -0,0 +1,163 @@
+# Beacon Endpoint Agent AWS S3 Pack
+
+This pack forwards Beacon endpoint JSONL events into an AWS S3 bucket. Beacon
+still writes one local source of truth: `runtime.jsonl`. AWS credentials,
+profiles, IAM roles, bucket policies, object lifecycle rules, and server-side
+encryption stay in AWS, Vector, or customer-managed deployment tooling, not in
+Beacon endpoint configuration.
+
+## Prerequisites
+
+- Beacon endpoint installed and writing local JSONL.
+- An AWS S3 bucket for Beacon runtime logs.
+- An IAM role or credentials available through the standard AWS credential
+ provider chain for the process running Vector or the AWS CLI smoke test.
+
+Recommended S3 layout:
+
+```text
+s3://example-security-logs/beacon/runtime/date=YYYY-MM-DD/<timestamp>-<uuid>.jsonl.gz
+```
+
+Example least-privilege IAM policy for a dedicated Beacon prefix:
+
+```json
+{
+ "Version": "2012-10-17",
+ "Statement": [
+ {
+ "Effect": "Allow",
+ "Action": ["s3:PutObject"],
+ "Resource": "arn:aws:s3:::example-security-logs/beacon/runtime/*"
+ }
+ ]
+}
+```
+
+Add `s3:PutObjectTagging`, KMS permissions, or bucket-specific conditions only
+if your AWS controls require them. Configure bucket lifecycle, retention,
+server-side encryption, and access logging in AWS.
+
+## Install
+
+Generate this pack:
+
+```bash
+beacon endpoint s3 install-pack --output ./beacon-s3-pack
+```
+
+The generated smoke-test script points at the Beacon log path selected by the
+CLI:
+
+- User mode: `~/.beacon/endpoint/logs/runtime.jsonl`
+- System mode: `/var/log/beacon-agent/runtime.jsonl`
+- Custom mode: the value passed with `--log-path`
+
+For MDM or managed endpoint deployment, prefer Beacon system mode so your
+customer-managed log shipper can tail `/var/log/beacon-agent/runtime.jsonl`
+without per-user home directory ACLs.
+
+The generated `vector.toml` uses the same selected Beacon log path.
+
+## One-Shot Smoke Test
+
+Use the generated script to upload the current file once. This is only for
+validation; do not use it as production forwarding because it re-uploads the
+whole file every time.
+
+```bash
+export BEACON_S3_BUCKET="example-security-logs"
+export BEACON_S3_PREFIX="beacon/runtime"
+export AWS_REGION="us-east-1"
+./beacon-s3-pack/s3-upload-smoke-test.sh
+```
+
+The script uses `aws s3 cp` and the standard AWS credential provider chain. You
+can provide credentials with an instance profile, SSO/profile configuration,
+environment variables, or your managed endpoint secret tooling. Beacon does not
+store AWS credentials.
+
+Beacon does not store AWS credentials in endpoint configuration or generated
+runtime state.
+
+Confirm the uploaded object:
+
+```bash
+aws s3 ls "s3://${BEACON_S3_BUCKET}/${BEACON_S3_PREFIX}/smoke-tests/" --region "$AWS_REGION"
+aws s3 cp "s3://${BEACON_S3_BUCKET}/${BEACON_S3_PREFIX}/smoke-tests/<object>.jsonl" - --region "$AWS_REGION" | grep "Beacon endpoint S3 validation event"
+```
+
+## Production Forwarding
+
+For production, use the generated Vector config as a customer-managed host-agent
+forwarding template. Beacon remains the local JSONL producer; Vector tails
+`runtime.jsonl`, checkpoints file offsets in its `data_dir`, batches Beacon
+events, and writes gzip-compressed newline-delimited JSON objects into AWS S3.
+
+Install Vector using your normal endpoint management tooling, then copy the
+generated config into Vector's config directory. On a macOS system-mode Beacon
+deployment, the generated config tails `/var/log/beacon-agent/runtime.jsonl`:
+
+```bash
+sudo mkdir -p /etc/vector
+sudo cp ./beacon-s3-pack/vector.toml /etc/vector/beacon-s3.toml
+export BEACON_S3_BUCKET="example-security-logs"
+export BEACON_S3_PREFIX="beacon/runtime"
+export AWS_REGION="us-east-1"
+vector validate /etc/vector/beacon-s3.toml
+vector --config /etc/vector/beacon-s3.toml
+```
+
+In managed deployments, provide `BEACON_S3_BUCKET`, optional
+`BEACON_S3_PREFIX`, `AWS_REGION`, and any AWS credential-provider settings
+through the Vector service environment, host identity, or MDM/secret tooling.
+Do not store AWS destination secrets in Beacon endpoint configuration.
+
+The Vector template is intentionally simple and expects a Vector version with
+the `file` source, `remap` transform, and `aws_s3` sink. It parses each Beacon
+JSONL line and re-encodes the original Beacon event as NDJSON so S3 receives one
+Beacon event per line, without a Vector wrapper.
+
+The template uses date-partitioned `key_prefix`, `filename_time_format = "%s"`,
+and `filename_append_uuid = true` so production forwarding does not overwrite
+previous S3 objects. It also sets `compression = "gzip"`,
+`content_encoding = "gzip"`, and `content_type = "application/x-ndjson"`.
+
+If you adapt the config or use another forwarder, it should:
+
+- checkpoint file offsets,
+- batch newline-delimited JSON records,
+- use non-overwriting object keys,
+- retry transient failures without duplicating the whole file,
+- keep AWS credentials, IAM roles, bucket policy, lifecycle, and encryption
+ outside Beacon endpoint configuration.
+
+## Validate
+
+Write a fresh Beacon validation event:
+
+```bash
+beacon endpoint s3 validate
+```
+
+Run the one-shot smoke test or wait for your production forwarder to ship the
+new line. Beacon can write the local validation event, but remote delivery must
+be confirmed with AWS tooling:
+
+```bash
+aws s3 ls "s3://${BEACON_S3_BUCKET}/${BEACON_S3_PREFIX}/" --recursive --region "$AWS_REGION"
+aws s3 cp "s3://${BEACON_S3_BUCKET}/${BEACON_S3_PREFIX}/date=<date>/<object>.jsonl.gz" - --region "$AWS_REGION" | gzip -dc | grep "Beacon endpoint S3 validation event"
+```
+
+Expected validation fields:
+
+```text
+vendor=beacon product=endpoint-agent destination.type=s3 destination.mode=aws_s3_jsonl
+```
+
+## Content Retention
+
+Beacon content retention defaults to `full`, so prompt text, tool input, command
+output, raw tool payloads, and other retained content may be forwarded to S3.
+Use Beacon's `metadata` or `redacted` content retention modes for stricter
+deployments.
diff --git a/cli/beacon/internal/endpoint/s3/pack/s3-upload-smoke-test.sh.tmpl b/cli/beacon/internal/endpoint/s3/pack/s3-upload-smoke-test.sh.tmpl
new file mode 100644
index 00000000..c5b7de09
--- /dev/null
+++ b/cli/beacon/internal/endpoint/s3/pack/s3-upload-smoke-test.sh.tmpl
@@ -0,0 +1,33 @@
+#!/bin/sh
+set -eu
+
+BEACON_LOG="${BEACON_LOG:-{{LOG_PATH}}}"
+BEACON_S3_PREFIX="${BEACON_S3_PREFIX:-beacon/runtime}"
+AWS_REGION="${AWS_REGION:-us-east-1}"
+
+if [ -z "${BEACON_S3_BUCKET:-}" ]; then
+ echo "BEACON_S3_BUCKET is required." >&2
+ exit 2
+fi
+
+if ! command -v aws >/dev/null 2>&1; then
+ echo "aws CLI is required for this smoke test." >&2
+ exit 2
+fi
+
+if [ ! -r "$BEACON_LOG" ]; then
+ echo "Beacon runtime log is not readable: $BEACON_LOG" >&2
+ exit 2
+fi
+
+prefix="${BEACON_S3_PREFIX%/}"
+timestamp="$(date -u +%Y%m%dT%H%M%SZ)"
+key="${prefix}/smoke-tests/beacon-runtime-${timestamp}.jsonl"
+
+aws s3 cp "$BEACON_LOG" "s3://${BEACON_S3_BUCKET}/${key}" \
+ --region "$AWS_REGION" \
+ --content-type "application/x-ndjson"
+
+echo "Uploaded one-shot Beacon log copy to s3://${BEACON_S3_BUCKET}/${key}"
+echo "AWS credentials were resolved by the standard AWS credential provider chain."
+echo "This script is for validation only; use vector.toml for production forwarding."
diff --git a/cli/beacon/internal/endpoint/s3/pack/vector.toml.tmpl b/cli/beacon/internal/endpoint/s3/pack/vector.toml.tmpl
new file mode 100644
index 00000000..dbbf445c
--- /dev/null
+++ b/cli/beacon/internal/endpoint/s3/pack/vector.toml.tmpl
@@ -0,0 +1,48 @@
+# Beacon Endpoint Agent -> AWS S3
+#
+# Run Vector as a customer-managed host agent. Keep AWS credentials, profiles,
+# instance roles, and bucket policy in AWS or deployment tooling, not in Beacon
+# config. This template relies on the standard AWS credential provider chain.
+
+data_dir = "/var/lib/vector/beacon-s3"
+
+[sources.beacon_runtime]
+type = "file"
+include = ["{{LOG_PATH}}"]
+read_from = "end"
+
+[transforms.beacon_json]
+type = "remap"
+inputs = ["beacon_runtime"]
+source = '''
+. = parse_json!(.message)
+'''
+
+[sinks.beacon_s3]
+type = "aws_s3"
+inputs = ["beacon_json"]
+bucket = "${BEACON_S3_BUCKET}"
+region = "${AWS_REGION:-us-east-1}"
+key_prefix = "${BEACON_S3_PREFIX:-beacon/runtime}/date=%F/"
+filename_time_format = "%s"
+filename_append_uuid = true
+filename_extension = "jsonl.gz"
+compression = "gzip"
+content_encoding = "gzip"
+content_type = "application/x-ndjson"
+storage_class = "${BEACON_S3_STORAGE_CLASS:-STANDARD}"
+
+[sinks.beacon_s3.encoding]
+codec = "json"
+
+[sinks.beacon_s3.framing]
+method = "newline_delimited"
+
+[sinks.beacon_s3.batch]
+max_bytes = 10000000
+timeout_secs = 300
+
+[sinks.beacon_s3.request]
+retry_attempts = 10
+retry_initial_backoff_secs = 1
+retry_max_duration_secs = 300
diff --git a/cli/beacon/internal/endpoint/s3/s3.go b/cli/beacon/internal/endpoint/s3/s3.go
new file mode 100644
index 00000000..4cc215ce
--- /dev/null
+++ b/cli/beacon/internal/endpoint/s3/s3.go
@@ -0,0 +1,113 @@
+package s3
+
+import (
+ "embed"
+ "fmt"
+ "io/fs"
+
+ "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/siempack"
+)
+
+//go:embed pack/*
+var packFS embed.FS
+
+const (
+ DefaultLogPath = "/var/log/beacon-agent/runtime.jsonl"
+ DefaultOutputDir = "beacon-s3-pack"
+)
+
+type File struct {
+ Name string
+ Content string
+ TemplateLogPath bool
+ JSONEscape bool
+}
+
+// readAsset reads an embedded asset using the module-level packFS and returns its
+// contents or an error.
+func readAsset(path string) (string, error) {
+ data, err := siempack.ReadFile(packFS, path)
+ if err != nil {
+ return "", fmt.Errorf("s3 pack asset %q: %w", path, err)
+ }
+ return data, nil
+}
+
+// mustRead returns the embedded asset at path or panics. Retained for test use
+// where the embedded FS is always present; all production code uses readAsset.
+func mustRead(path string) string {
+ s, err := readAsset(path)
+ if err != nil {
+ panic(err.Error())
+ }
+ return s
+}
+
+// uploadSmokeTestFromFS renders the S3 upload smoke-test script using the supplied FS.
+func uploadSmokeTestFromFS(fsys fs.FS, logPath string) (string, error) {
+ if logPath == "" {
+ logPath = DefaultLogPath
+ }
+ content, err := siempack.ReadFile(fsys, "pack/s3-upload-smoke-test.sh.tmpl")
+ if err != nil {
+ return "", fmt.Errorf("s3 pack asset %q: %w", "pack/s3-upload-smoke-test.sh.tmpl", err)
+ }
+ return siempack.RenderLogPath(content, logPath), nil
+}
+
+// filesFromFS builds the full file list from the supplied FS, propagating any
+// read errors instead of panicking. Template substitution of LOG_PATH is
+// deferred to siempack.Install via the TemplateLogPath field.
+func filesFromFS(fsys fs.FS) ([]File, error) {
+ readme, err := siempack.ReadFile(fsys, "pack/README.md")
+ if err != nil {
+ return nil, fmt.Errorf("s3 pack asset %q: %w", "pack/README.md", err)
+ }
+ smokeTest, err := siempack.ReadFile(fsys, "pack/s3-upload-smoke-test.sh.tmpl")
+ if err != nil {
+ return nil, fmt.Errorf("s3 pack asset %q: %w", "pack/s3-upload-smoke-test.sh.tmpl", err)
+ }
+ sample, err := siempack.ReadFile(fsys, "pack/sample-event.jsonl")
+ if err != nil {
+ return nil, fmt.Errorf("s3 pack asset %q: %w", "pack/sample-event.jsonl", err)
+ }
+ vector, err := siempack.ReadFile(fsys, "pack/vector.toml.tmpl")
+ if err != nil {
+ return nil, fmt.Errorf("s3 pack asset %q: %w", "pack/vector.toml.tmpl", err)
+ }
+ return []File{
+ {Name: "README.md", Content: readme},
+ {Name: "s3-upload-smoke-test.sh", Content: smokeTest, TemplateLogPath: true},
+ {Name: "sample-event.jsonl", Content: sample},
+ {Name: "vector.toml", Content: vector, TemplateLogPath: true},
+ }, nil
+}
+
+// Files returns all pack files, propagating any embedded asset read error.
+func Files() ([]File, error) {
+ return filesFromFS(packFS)
+}
+
+// UploadSmokeTest returns the S3 upload smoke-test script with logPath substituted.
+func UploadSmokeTest(logPath string) (string, error) {
+ return uploadSmokeTestFromFS(packFS, logPath)
+}
+
+// InstallPack writes the pack files to outputDir with logPath substituted.
+func InstallPack(outputDir, logPath string) error {
+ if outputDir == "" {
+ outputDir = DefaultOutputDir
+ }
+ if logPath == "" {
+ logPath = DefaultLogPath
+ }
+ files, err := filesFromFS(packFS)
+ if err != nil {
+ return err
+ }
+ sfiles := make([]siempack.File, 0, len(files))
+ for _, file := range files {
+ sfiles = append(sfiles, siempack.File(file))
+ }
+ return siempack.Install(outputDir, sfiles, logPath)
+}
diff --git a/cli/beacon/internal/endpoint/s3/s3_test.go b/cli/beacon/internal/endpoint/s3/s3_test.go
new file mode 100644
index 00000000..4f4cab61
--- /dev/null
+++ b/cli/beacon/internal/endpoint/s3/s3_test.go
@@ -0,0 +1,236 @@
+package s3
+
+import (
+ "bufio"
+ "encoding/json"
+ "os"
+ "path/filepath"
+ "strings"
+ "testing"
+ "testing/fstest"
+)
+
+func TestUploadSmokeTestUsesConfiguredPath(t *testing.T) {
+ got, err := UploadSmokeTest("/tmp/beacon/runtime.jsonl")
+ if err != nil {
+ t.Fatalf("UploadSmokeTest returned unexpected error: %v", err)
+ }
+ if !strings.Contains(got, "/tmp/beacon/runtime.jsonl") {
+ t.Fatalf("script did not include configured path: %s", got)
+ }
+ if strings.Contains(got, "{{LOG_PATH}}") {
+ t.Fatalf("script still contains template token: %s", got)
+ }
+ for _, want := range []string{
+ "BEACON_S3_BUCKET",
+ "BEACON_S3_PREFIX",
+ "AWS_REGION",
+ "aws s3 cp",
+ "--content-type \"application/x-ndjson\"",
+ "standard AWS credential provider chain",
+ "validation only",
+ } {
+ if !strings.Contains(got, want) {
+ t.Fatalf("script missing %q: %s", want, got)
+ }
+ }
+}
+
+func TestInstallPackWritesExpectedFiles(t *testing.T) {
+ dir := t.TempDir()
+ if err := InstallPack(dir, "/tmp/beacon/runtime.jsonl"); err != nil {
+ t.Fatalf("InstallPack returned error: %v", err)
+ }
+ for _, name := range []string{"README.md", "s3-upload-smoke-test.sh", "sample-event.jsonl", "vector.toml"} {
+ if _, err := os.Stat(filepath.Join(dir, name)); err != nil {
+ t.Fatalf("expected %s: %v", name, err)
+ }
+ }
+ scriptPath := filepath.Join(dir, "s3-upload-smoke-test.sh")
+ script, err := os.ReadFile(scriptPath)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !strings.Contains(string(script), "/tmp/beacon/runtime.jsonl") {
+ t.Fatalf("generated script missing configured log path: %s", script)
+ }
+ info, err := os.Stat(scriptPath)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if info.Mode().Perm()&0111 == 0 {
+ t.Fatalf("generated script should be executable, mode=%s", info.Mode())
+ }
+ vectorPath := filepath.Join(dir, "vector.toml")
+ vectorConfig, err := os.ReadFile(vectorPath)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !strings.Contains(string(vectorConfig), "/tmp/beacon/runtime.jsonl") {
+ t.Fatalf("generated vector config missing configured log path: %s", vectorConfig)
+ }
+ info, err = os.Stat(vectorPath)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if info.Mode().Perm() != 0644 {
+ t.Fatalf("generated vector config should be 0644, mode=%s", info.Mode().Perm())
+ }
+}
+
+func TestVectorConfigUsesAWSS3SinkAndPreservesJSONShape(t *testing.T) {
+ got := mustRead("pack/vector.toml.tmpl")
+ for _, want := range []string{
+ `include = ["{{LOG_PATH}}"]`,
+ `read_from = "end"`,
+ `. = parse_json!(.message)`,
+ `type = "aws_s3"`,
+ `bucket = "${BEACON_S3_BUCKET}"`,
+ `region = "${AWS_REGION:-us-east-1}"`,
+ `key_prefix = "${BEACON_S3_PREFIX:-beacon/runtime}/date=%F/"`,
+ `filename_time_format = "%s"`,
+ `filename_append_uuid = true`,
+ `filename_extension = "jsonl.gz"`,
+ `compression = "gzip"`,
+ `content_encoding = "gzip"`,
+ `content_type = "application/x-ndjson"`,
+ `codec = "json"`,
+ `method = "newline_delimited"`,
+ `max_bytes = 10000000`,
+ `retry_attempts = 10`,
+ } {
+ if !strings.Contains(got, want) {
+ t.Fatalf("vector config missing %q: %s", want, got)
+ }
+ }
+}
+
+func TestSampleEventsCoverValidationHookAndOTelShapes(t *testing.T) {
+ scanner := bufio.NewScanner(strings.NewReader(mustRead("pack/sample-event.jsonl")))
+ var sawValidation, sawHook, sawOTel bool
+ for scanner.Scan() {
+ var doc map[string]interface{}
+ if err := json.Unmarshal(scanner.Bytes(), &doc); err != nil {
+ t.Fatalf("sample-event.jsonl is not valid JSONL: %v", err)
+ }
+ if destination, ok := doc["destination"].(map[string]interface{}); ok && destination["type"] == "s3" {
+ sawValidation = true
+ }
+ if _, ok := doc["tool"].(map[string]interface{}); ok {
+ sawHook = true
+ }
+ if raw, ok := doc["raw"].(map[string]interface{}); ok && raw["otel_signal"] != nil {
+ sawOTel = true
+ }
+ }
+ if err := scanner.Err(); err != nil {
+ t.Fatal(err)
+ }
+ if !sawValidation || !sawHook || !sawOTel {
+ t.Fatalf("sample events should include validation, hook-rich, and OTel shapes; validation=%t hook=%t otel=%t", sawValidation, sawHook, sawOTel)
+ }
+}
+
+func TestPackREADMEMentionsS3SetupAndProductionForwarding(t *testing.T) {
+ readme := mustRead("pack/README.md")
+ for _, want := range []string{
+ "beacon endpoint s3 validate",
+ "AWS S3 bucket",
+ "standard AWS credential provider chain",
+ "least-privilege IAM policy",
+ "BEACON_S3_BUCKET",
+ "BEACON_S3_PREFIX",
+ "aws s3 ls",
+ "aws s3 cp",
+ "Beacon endpoint S3 validation event",
+ "vendor=beacon product=endpoint-agent destination.type=s3 destination.mode=aws_s3_jsonl",
+ "vector.toml",
+ "customer-managed host-agent",
+ "without a Vector wrapper",
+ "content retention",
+ "/var/log/beacon-agent/runtime.jsonl",
+ "Beacon does not store AWS credentials",
+ "server-side encryption",
+ } {
+ if !strings.Contains(readme, want) {
+ t.Fatalf("pack README missing %q", want)
+ }
+ }
+}
+
+func TestFiles_NoError(t *testing.T) {
+ files, err := Files()
+ if err != nil {
+ t.Fatalf("Files() returned unexpected error: %v", err)
+ }
+ if len(files) == 0 {
+ t.Fatal("Files() returned no files")
+ }
+}
+
+func TestFiles_ContainsAllRequiredNames(t *testing.T) {
+ files, err := Files()
+ if err != nil {
+ t.Fatal(err)
+ }
+ names := make(map[string]bool)
+ for _, f := range files {
+ names[f.Name] = true
+ }
+ for _, required := range []string{
+ "README.md", "s3-upload-smoke-test.sh", "sample-event.jsonl", "vector.toml",
+ } {
+ if !names[required] {
+ t.Errorf("Files() missing required file %q", required)
+ }
+ }
+}
+
+func TestFilesFromFS_PropagatesReadError(t *testing.T) {
+ emptyFS := fstest.MapFS{}
+ _, err := filesFromFS(emptyFS)
+ if err == nil {
+ t.Fatal("filesFromFS with empty FS should return an error")
+ }
+ if !strings.Contains(err.Error(), "s3 pack asset") {
+ t.Fatalf("error should identify the pack source, got: %v", err)
+ }
+}
+
+func TestUploadSmokeTestFromFS_ErrorOnMissingAsset(t *testing.T) {
+ emptyFS := fstest.MapFS{}
+ _, err := uploadSmokeTestFromFS(emptyFS, "/some/path.jsonl")
+ if err == nil {
+ t.Fatal("uploadSmokeTestFromFS with empty FS should return error")
+ }
+ if !strings.Contains(err.Error(), "s3 pack asset") {
+ t.Fatalf("error should identify the pack source, got: %v", err)
+ }
+}
+
+func TestInstallPack_ErrorOnWriteFailure(t *testing.T) {
+ if os.Getuid() == 0 {
+ t.Skip("running as root: filesystem permission restrictions do not apply")
+ }
+ dir := t.TempDir()
+ if err := os.Chmod(dir, 0555); err != nil {
+ t.Skip("cannot set directory permissions:", err)
+ }
+ defer os.Chmod(dir, 0755)
+
+ subdir := filepath.Join(dir, "output")
+ err := InstallPack(subdir, DefaultLogPath)
+ if err == nil {
+ t.Fatal("InstallPack into read-only directory should return error")
+ }
+}
+
+func TestUploadSmokeTest_DefaultLogPath(t *testing.T) {
+ got, err := UploadSmokeTest("")
+ if err != nil {
+ t.Fatalf("UploadSmokeTest with empty path returned error: %v", err)
+ }
+ if !strings.Contains(got, DefaultLogPath) {
+ t.Fatalf("UploadSmokeTest with empty logPath should use DefaultLogPath %q, got: %s", DefaultLogPath, got)
+ }
+}