Skip to content

The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu

add-managed-ingest-support

Justin D'Souzad5ca52109e68
diff --git a/cli/beacon/cmd/endpoint_install.go b/cli/beacon/cmd/endpoint_install.go
index edb49738..564b9b73 100644
--- a/cli/beacon/cmd/endpoint_install.go
+++ b/cli/beacon/cmd/endpoint_install.go
@@ -157,6 +157,7 @@ func runEndpointStatus(cmd *cobra.Command, args []string) error {
} else {
fmt.Println("Last event: present")
}
+ printIngestStatus("Endpoint ingest", status.ManagedUpload)
return nil
}
diff --git a/cli/beacon/cmd/ingest.go b/cli/beacon/cmd/ingest.go
new file mode 100644
index 00000000..7e8d90b9
--- /dev/null
+++ b/cli/beacon/cmd/ingest.go
@@ -0,0 +1,146 @@
+package cmd
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "os"
+ "time"
+
+ beaconauth "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/auth"
+ endpointconfig "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/config"
+ "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/lifecycle"
+ "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/ingest"
+ endpointingest "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/ingest/endpoint"
+ "github.com/spf13/cobra"
+)
+
+var ingestCmd = &cobra.Command{
+ Use: "ingest",
+ Short: "Upload Beacon telemetry to configured ingest destinations",
+}
+
+var ingestStatusCmd = &cobra.Command{
+ Use: "status",
+ Short: "Show configured ingest status",
+ SilenceUsage: true,
+ RunE: func(cmd *cobra.Command, args []string) error {
+ state := endpointIngestStatus()
+ if endpointOpts.jsonOutput {
+ return json.NewEncoder(os.Stdout).Encode(map[string]ingest.State{
+ "endpoint": state,
+ })
+ }
+ fmt.Println("Beacon ingest")
+ printIngestStatus("Endpoint", state)
+ return nil
+ },
+}
+
+var ingestEndpointCmd = &cobra.Command{
+ Use: "endpoint",
+ Short: "Manage endpoint telemetry ingest",
+}
+
+var ingestEndpointStatusCmd = &cobra.Command{
+ Use: "status",
+ Short: "Show endpoint telemetry upload status",
+ SilenceUsage: true,
+ RunE: func(cmd *cobra.Command, args []string) error {
+ state := endpointIngestStatus()
+ if endpointOpts.jsonOutput {
+ return json.NewEncoder(os.Stdout).Encode(state)
+ }
+ printIngestStatus("Endpoint ingest", state)
+ return nil
+ },
+}
+
+var ingestEndpointUploadCmd = &cobra.Command{
+ Use: "upload",
+ Short: "Upload endpoint telemetry to managed ingest",
+ SilenceUsage: true,
+ RunE: func(cmd *cobra.Command, args []string) error {
+ res, err := uploadEndpointIngest(cmd.Context())
+ if endpointOpts.jsonOutput {
+ if encodeErr := json.NewEncoder(os.Stdout).Encode(res.State); encodeErr != nil {
+ return encodeErr
+ }
+ } else {
+ printIngestStatus("Endpoint ingest", res.State)
+ }
+ if err != nil {
+ return err
+ }
+ return nil
+ },
+}
+
+func init() {
+ rootCmd.AddCommand(ingestCmd)
+ ingestCmd.AddCommand(ingestStatusCmd)
+ ingestCmd.AddCommand(ingestEndpointCmd)
+ ingestEndpointCmd.AddCommand(ingestEndpointStatusCmd)
+ ingestEndpointCmd.AddCommand(ingestEndpointUploadCmd)
+
+ for _, c := range []*cobra.Command{ingestStatusCmd, ingestEndpointStatusCmd, ingestEndpointUploadCmd} {
+ c.Flags().BoolVar(&endpointOpts.userMode, "user", true, "Use per-user endpoint paths")
+ c.Flags().BoolVar(&endpointOpts.systemMode, "system", false, "Use system endpoint paths and launch daemon")
+ c.Flags().StringVar(&endpointOpts.logPath, "log-path", "", "Runtime JSONL log path")
+ c.Flags().BoolVar(&endpointOpts.jsonOutput, "json", false, "Print output as JSON")
+ }
+}
+
+func endpointIngestStatus() ingest.State {
+ cfg, effectiveUserMode, logPath := endpointIngestConfig()
+ creds, _ := beaconauth.LoadCredentials()
+ return ingest.Status(
+ endpointingest.Settings(cfg, logPath, effectiveUserMode),
+ endpointingest.Store(effectiveUserMode),
+ creds,
+ )
+}
+
+func uploadEndpointIngest(ctx context.Context) (ingest.Result, error) {
+ cfg, effectiveUserMode, logPath := endpointIngestConfig()
+ creds, _ := beaconauth.LoadCredentials()
+ uploadCtx, cancel := context.WithTimeout(ctx, 30*time.Second)
+ defer cancel()
+ res := ingest.Upload(uploadCtx, ingest.Options{
+ Settings: endpointingest.Settings(cfg, logPath, effectiveUserMode),
+ Creds: creds,
+ Store: endpointingest.Store(effectiveUserMode),
+ Source: endpointingest.NewSource(cfg, logPath, effectiveUserMode),
+ })
+ if res.State.Enabled && res.State.Managed && res.State.LastError != "" {
+ return res, fmt.Errorf("%s", res.State.LastError)
+ }
+ return res, nil
+}
+
+func endpointIngestConfig() (endpointconfig.Config, bool, string) {
+ cfg := loadOrDefaultConfig()
+ runtimeLog := lifecycle.ResolveRuntimeLog(endpointUserMode(), endpointOpts.logPath)
+ effectiveCfg := cfg
+ effectiveCfg.UserMode = runtimeLog.EffectiveUserMode
+ if runtimeLog.EffectiveUserMode != cfg.UserMode {
+ effectiveCfg = loadConfigForMode(runtimeLog.EffectiveUserMode, runtimeLog.EffectiveLogPath)
+ }
+ effectiveCfg.LogPath = runtimeLog.EffectiveLogPath
+ return effectiveCfg, effectiveCfg.UserMode, effectiveCfg.LogPath
+}
+
+func printIngestStatus(label string, state ingest.State) {
+ fmt.Printf("%s: enabled=%t managed=%t logged_in=%t", label, state.Enabled, state.Managed, state.LoggedIn)
+ if state.LastUploadAt != "" {
+ fmt.Printf(" last_upload=%s", state.LastUploadAt)
+ }
+ if state.LastCursor.Offset > 0 {
+ fmt.Printf(" cursor_offset=%d", state.LastCursor.Offset)
+ }
+ fmt.Printf(" accepted=%d rejected=%d", state.AcceptedCount, state.RejectedCount)
+ if state.LastError != "" {
+ fmt.Printf(" error=%s", state.LastError)
+ }
+ fmt.Println()
+}
diff --git a/cli/beacon/internal/endpoint/config/config.go b/cli/beacon/internal/endpoint/config/config.go
index 01e133da..340a5d3e 100644
--- a/cli/beacon/internal/endpoint/config/config.go
+++ b/cli/beacon/internal/endpoint/config/config.go
@@ -22,12 +22,13 @@ const (
)
type Config struct {
- UserMode bool `json:"user_mode"`
- LogPath string `json:"log_path"`
- Collector Collector `json:"collector"`
- Harnesses []string `json:"harnesses"`
- EventCategories []string `json:"event_categories,omitempty"`
- Destinations *Destinations `json:"destinations,omitempty"`
+ UserMode bool `json:"user_mode"`
+ LogPath string `json:"log_path"`
+ Collector Collector `json:"collector"`
+ Harnesses []string `json:"harnesses"`
+ EventCategories []string `json:"event_categories,omitempty"`
+ Destinations *Destinations `json:"destinations,omitempty"`
+ ManagedUpload *ManagedUpload `json:"managed_upload,omitempty"`
}
type Collector struct {
@@ -45,6 +46,14 @@ type Destinations struct {
FalconHEC *FalconHEC `json:"falcon_hec,omitempty"`
}
+type ManagedUpload struct {
+ Enabled bool `json:"enabled,omitempty"`
+ Managed bool `json:"managed,omitempty"`
+ IngestURL string `json:"ingest_url,omitempty"`
+ SourceID string `json:"source_id,omitempty"`
+ ContentRetention string `json:"content_retention,omitempty"`
+}
+
type SplunkHEC struct {
Enabled bool `json:"enabled,omitempty"`
Endpoint string `json:"endpoint,omitempty"`
diff --git a/cli/beacon/internal/endpoint/lifecycle/lifecycle.go b/cli/beacon/internal/endpoint/lifecycle/lifecycle.go
index 7e46c154..d67111eb 100644
--- a/cli/beacon/internal/endpoint/lifecycle/lifecycle.go
+++ b/cli/beacon/internal/endpoint/lifecycle/lifecycle.go
@@ -8,6 +8,7 @@ import (
"runtime"
"time"
+ beaconauth "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/auth"
endpointcollector "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/collector"
endpointconfig "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/config"
"github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/diagnostics"
@@ -15,6 +16,8 @@ import (
"github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/schema"
"github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/service"
"github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/writer"
+ "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/ingest"
+ endpointingest "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/ingest/endpoint"
"github.com/asymptote-labs/agent-beacon/cli/beacon/internal/version"
)
@@ -55,16 +58,17 @@ type InstallResult struct {
}
type Status struct {
- Version string `json:"version"`
- ConfigPath string `json:"config_path"`
- LogPath string `json:"log_path"`
- RuntimeLog RuntimeLogSource `json:"runtime_log"`
- Collector endpointcollector.Status `json:"collector"`
- Service service.Status `json:"service"`
- Harnesses []harness.Harness `json:"harnesses"`
- Diagnostics []diagnostics.Check `json:"diagnostics"`
- LastEvent string `json:"last_event,omitempty"`
- Destinations DestinationStatus `json:"destinations"`
+ Version string `json:"version"`
+ ConfigPath string `json:"config_path"`
+ LogPath string `json:"log_path"`
+ RuntimeLog RuntimeLogSource `json:"runtime_log"`
+ Collector endpointcollector.Status `json:"collector"`
+ Service service.Status `json:"service"`
+ Harnesses []harness.Harness `json:"harnesses"`
+ Diagnostics []diagnostics.Check `json:"diagnostics"`
+ LastEvent string `json:"last_event,omitempty"`
+ Destinations DestinationStatus `json:"destinations"`
+ ManagedUpload ingest.State `json:"managed_upload"`
}
type DestinationStatus struct {
@@ -293,17 +297,24 @@ func GetStatus(userMode bool, logPath string) Status {
Message: runtimeLog.Warning,
})
}
+ creds, _ := beaconauth.LoadCredentials()
+ managedUpload := ingest.Status(
+ endpointingest.Settings(effectiveCfg, effectiveCfg.LogPath, effectiveCfg.UserMode),
+ endpointingest.Store(effectiveCfg.UserMode),
+ creds,
+ )
return Status{
- Version: version.GetVersion(),
- ConfigPath: endpointconfig.ConfigPath(effectiveCfg.UserMode),
- LogPath: effectiveCfg.LogPath,
- RuntimeLog: runtimeLog,
- Collector: endpointcollector.CheckStatus(effectiveCfg),
- Service: service.Manager{UserMode: effectiveCfg.UserMode}.Status(),
- Harnesses: harness.DiscoverAll(),
- Diagnostics: checks,
- LastEvent: last,
- Destinations: destinationStatus(effectiveCfg),
+ Version: version.GetVersion(),
+ ConfigPath: endpointconfig.ConfigPath(effectiveCfg.UserMode),
+ LogPath: effectiveCfg.LogPath,
+ RuntimeLog: runtimeLog,
+ Collector: endpointcollector.CheckStatus(effectiveCfg),
+ Service: service.Manager{UserMode: effectiveCfg.UserMode}.Status(),
+ Harnesses: harness.DiscoverAll(),
+ Diagnostics: checks,
+ LastEvent: last,
+ Destinations: destinationStatus(effectiveCfg),
+ ManagedUpload: managedUpload,
}
}
diff --git a/cli/beacon/internal/endpoint/lifecycle/lifecycle_test.go b/cli/beacon/internal/endpoint/lifecycle/lifecycle_test.go
index 653075e8..49bb459f 100644
--- a/cli/beacon/internal/endpoint/lifecycle/lifecycle_test.go
+++ b/cli/beacon/internal/endpoint/lifecycle/lifecycle_test.go
@@ -3,12 +3,15 @@ package lifecycle
import (
"errors"
"net"
+ "net/http"
+ "net/http/httptest"
"os"
"path/filepath"
"runtime"
"strings"
"testing"
+ beaconauth "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/auth"
endpointconfig "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/config"
"github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/harness"
"github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/schema"
@@ -159,6 +162,47 @@ func TestConfigureHarnessesRejectsUnsupportedHarness(t *testing.T) {
}
}
+func TestGetStatusDoesNotUploadManagedTelemetry(t *testing.T) {
+ home := t.TempDir()
+ t.Setenv("HOME", home)
+ logPath := filepath.Join(home, "runtime.jsonl")
+ if err := os.WriteFile(logPath, []byte("{\"event\":\"ok\"}\n"), 0600); err != nil {
+ t.Fatal(err)
+ }
+ called := false
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ called = true
+ http.Error(w, "status should not upload", http.StatusInternalServerError)
+ }))
+ defer server.Close()
+
+ _, err := endpointconfig.Save(endpointconfig.Config{
+ UserMode: true,
+ LogPath: logPath,
+ Harnesses: []string{},
+ ManagedUpload: &endpointconfig.ManagedUpload{
+ Enabled: true,
+ Managed: true,
+ IngestURL: server.URL,
+ SourceID: "source-1",
+ },
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := beaconauth.SaveCredentials(&beaconauth.Credentials{Token: "token", UserID: "user"}); err != nil {
+ t.Fatal(err)
+ }
+
+ status := GetStatus(true, logPath)
+ if !status.ManagedUpload.Enabled || !status.ManagedUpload.Managed || !status.ManagedUpload.LoggedIn {
+ t.Fatalf("unexpected managed upload status: %#v", status.ManagedUpload)
+ }
+ if called {
+ t.Fatal("GetStatus performed a managed upload network call")
+ }
+}
+
func TestConfigureHarnessesRejectsFactoryAsMDMManaged(t *testing.T) {
cfg := endpointconfig.Config{
Harnesses: []string{"factory"},
diff --git a/cli/beacon/internal/ingest/client.go b/cli/beacon/internal/ingest/client.go
new file mode 100644
index 00000000..e94c0977
--- /dev/null
+++ b/cli/beacon/internal/ingest/client.go
@@ -0,0 +1,65 @@
+package ingest
+
+import (
+ "bytes"
+ "context"
+ "encoding/json"
+ "fmt"
+ "io"
+ "net/http"
+ "strings"
+ "time"
+
+ beaconauth "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/auth"
+ "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/version"
+)
+
+const defaultIngestPath = "/api/v1/ingest/beacon/events"
+
+type Client struct {
+ URL string
+ HTTPClient *http.Client
+}
+
+func NewClient(ingestURL string, httpClient *http.Client) Client {
+ if strings.TrimSpace(ingestURL) == "" {
+ ingestURL = beaconauth.ResolveDashboardURL("") + defaultIngestPath
+ }
+ if httpClient == nil {
+ httpClient = &http.Client{Timeout: 15 * time.Second}
+ }
+ return Client{
+ URL: strings.TrimRight(ingestURL, "/"),
+ HTTPClient: httpClient,
+ }
+}
+
+func (c Client) UploadBatch(ctx context.Context, token string, reqBody uploadRequest) (uploadResponse, error) {
+ data, err := json.Marshal(reqBody)
+ if err != nil {
+ return uploadResponse{}, err
+ }
+ req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.URL, bytes.NewReader(data))
+ if err != nil {
+ return uploadResponse{}, err
+ }
+ req.Header.Set("Authorization", "Bearer "+token)
+ req.Header.Set("Content-Type", "application/json")
+ req.Header.Set("User-Agent", "beacon/"+version.GetVersion())
+
+ resp, err := c.HTTPClient.Do(req)
+ if err != nil {
+ return uploadResponse{}, err
+ }
+ defer resp.Body.Close()
+
+ respBody, _ := io.ReadAll(io.LimitReader(resp.Body, 4096))
+ if resp.StatusCode < 200 || resp.StatusCode >= 300 {
+ return uploadResponse{}, fmt.Errorf("ingest upload failed: HTTP %d: %s", resp.StatusCode, strings.TrimSpace(string(respBody)))
+ }
+ var parsed uploadResponse
+ if err := json.Unmarshal(respBody, &parsed); err != nil && len(respBody) > 0 {
+ return uploadResponse{}, fmt.Errorf("ingest upload response was invalid: %w", err)
+ }
+ return parsed, nil
+}
diff --git a/cli/beacon/internal/ingest/endpoint/reader.go b/cli/beacon/internal/ingest/endpoint/reader.go
new file mode 100644
index 00000000..d8d75092
--- /dev/null
+++ b/cli/beacon/internal/ingest/endpoint/reader.go
@@ -0,0 +1,142 @@
+package endpoint
+
+import (
+ "bufio"
+ "bytes"
+ "encoding/json"
+ "os"
+ "path/filepath"
+ "sort"
+ "strings"
+
+ "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/ingest"
+)
+
+func (s Source) Batches(state ingest.State, maxEvents int, maxBytes int) ([]ingest.Batch, error) {
+ if maxEvents <= 0 {
+ maxEvents = ingest.DefaultBatchEvents
+ }
+ if maxBytes <= 0 {
+ maxBytes = ingest.DefaultBatchBytes
+ }
+ files, err := logFiles(s.logPath)
+ if err != nil {
+ return nil, err
+ }
+ var batches []ingest.Batch
+ for _, file := range files {
+ fileBatches, err := readFileBatches(file, s.logPath, state, maxEvents, maxBytes)
+ if err != nil {
+ return nil, err
+ }
+ batches = append(batches, fileBatches...)
+ }
+ return batches, nil
+}
+
+func readFileBatches(path string, activePath string, state ingest.State, maxEvents int, maxBytes int) ([]ingest.Batch, error) {
+ startOffset := state.FileOffsets[path]
+ file, err := os.Open(path)
+ if err != nil {
+ if os.IsNotExist(err) {
+ return nil, nil
+ }
+ return nil, err
+ }
+ defer file.Close()
+
+ if startOffset > 0 {
+ if _, err := file.Seek(startOffset, 0); err != nil {
+ return nil, err
+ }
+ }
+
+ scanner := bufio.NewScanner(file)
+ scanner.Buffer(make([]byte, 0, 64*1024), maxBytes)
+ var (
+ batches []ingest.Batch
+ events []json.RawMessage
+ batchSize int
+ batchRejected int
+ batchEndOffset int64
+ batchLine int
+ line int
+ offset = startOffset
+ )
+ flush := func() {
+ if len(events) == 0 && batchRejected == 0 {
+ return
+ }
+ batches = append(batches, ingest.Batch{
+ Cursor: ingest.Cursor{
+ LogPath: path,
+ Offset: batchEndOffset,
+ Line: batchLine,
+ Archive: archiveName(activePath, path),
+ },
+ Events: events,
+ Rejected: batchRejected,
+ })
+ events = nil
+ batchSize = 0
+ batchRejected = 0
+ }
+
+ for scanner.Scan() {
+ line++
+ raw := append([]byte(nil), scanner.Bytes()...)
+ nextOffset := offset + int64(len(raw)) + 1
+ trimmed := bytes.TrimSpace(raw)
+ if len(trimmed) == 0 {
+ batchEndOffset = nextOffset
+ batchLine = line
+ offset = nextOffset
+ continue
+ }
+ if len(events) > 0 && (len(events) >= maxEvents || batchSize+len(trimmed) > maxBytes) {
+ flush()
+ }
+ if !json.Valid(trimmed) {
+ batchRejected++
+ batchEndOffset = nextOffset
+ batchLine = line
+ offset = nextOffset
+ continue
+ }
+ events = append(events, json.RawMessage(trimmed))
+ batchSize += len(trimmed)
+ batchEndOffset = nextOffset
+ batchLine = line
+ offset = nextOffset
+ }
+ if err := scanner.Err(); err != nil {
+ return nil, err
+ }
+ flush()
+ return batches, nil
+}
+
+func logFiles(path string) ([]string, error) {
+ dir := filepath.Dir(path)
+ base := filepath.Base(path)
+ entries, err := os.ReadDir(dir)
+ if err != nil {
+ return nil, err
+ }
+ var archives []string
+ for _, entry := range entries {
+ name := entry.Name()
+ if strings.HasPrefix(name, base+".") {
+ archives = append(archives, filepath.Join(dir, name))
+ }
+ }
+ sort.Strings(archives)
+ return append(archives, path), nil
+}
+
+func archiveName(activePath string, path string) string {
+ if activePath == path {
+ return ""
+ }
+ return filepath.Base(path)
+}
diff --git a/cli/beacon/internal/ingest/endpoint/reader_test.go b/cli/beacon/internal/ingest/endpoint/reader_test.go
new file mode 100644
index 00000000..3ce94082
--- /dev/null
+++ b/cli/beacon/internal/ingest/endpoint/reader_test.go
@@ -0,0 +1,61 @@
+package endpoint
+
+import (
+ "os"
+ "path/filepath"
+ "testing"
+
+ endpointconfig "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/config"
+ "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/ingest"
+)
+
+func TestEndpointSourceBatchesValidEventsAndMalformedRejects(t *testing.T) {
+ dir := t.TempDir()
+ logPath := filepath.Join(dir, "runtime.jsonl")
+ content := []byte("{\"event\":\"one\"}\nnot-json\n{\"event\":\"two\"}\n")
+ if err := os.WriteFile(logPath, content, 0600); err != nil {
+ t.Fatal(err)
+ }
+
+ source := NewSource(endpointconfig.Config{UserMode: true, LogPath: logPath}, logPath, true)
+ batches, err := source.Batches(ingest.State{FileOffsets: map[string]int64{}}, 500, 1024)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if len(batches) != 1 {
+ t.Fatalf("len(batches) = %d, want 1", len(batches))
+ }
+ if got := len(batches[0].Events); got != 2 {
+ t.Fatalf("events = %d, want 2", got)
+ }
+ if batches[0].Rejected != 1 {
+ t.Fatalf("Rejected = %d, want 1", batches[0].Rejected)
+ }
+ if batches[0].Cursor.Offset != int64(len(content)) {
+ t.Fatalf("Offset = %d, want %d", batches[0].Cursor.Offset, len(content))
+ }
+}
+
+func TestEndpointSourceHonorsSavedOffset(t *testing.T) {
+ dir := t.TempDir()
+ logPath := filepath.Join(dir, "runtime.jsonl")
+ first := "{\"event\":\"one\"}\n"
+ second := "{\"event\":\"two\"}\n"
+ if err := os.WriteFile(logPath, []byte(first+second), 0600); err != nil {
+ t.Fatal(err)
+ }
+
+ source := NewSource(endpointconfig.Config{UserMode: true, LogPath: logPath}, logPath, true)
+ batches, err := source.Batches(ingest.State{FileOffsets: map[string]int64{logPath: int64(len(first))}}, 500, 1024)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if len(batches) != 1 || len(batches[0].Events) != 1 {
+ t.Fatalf("unexpected batches: %#v", batches)
+ }
+ if string(batches[0].Events[0]) != "{\"event\":\"two\"}" {
+ t.Fatalf("event = %s", batches[0].Events[0])
+ }
+}
diff --git a/cli/beacon/internal/ingest/endpoint/source.go b/cli/beacon/internal/ingest/endpoint/source.go
new file mode 100644
index 00000000..4c5a2f25
--- /dev/null
+++ b/cli/beacon/internal/ingest/endpoint/source.go
@@ -0,0 +1,90 @@
+package endpoint
+
+import (
+ "crypto/sha256"
+ "encoding/hex"
+ "fmt"
+ "os"
+ "path/filepath"
+ "strings"
+
+ endpointconfig "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/config"
+ "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/ingest"
+)
+
+const stateFileName = "upload-state.json"
+
+type Source struct {
+ cfg endpointconfig.Config
+ logPath string
+ userMode bool
+}
+
+func NewSource(cfg endpointconfig.Config, logPath string, userMode bool) Source {
+ return Source{
+ cfg: cfg,
+ logPath: logPath,
+ userMode: userMode,
+ }
+}
+
+func Settings(cfg endpointconfig.Config, logPath string, userMode bool) ingest.Settings {
+ managed := cfg.ManagedUpload
+ settings := ingest.Settings{
+ SourceID: sourceID(cfg, logPath, userMode),
+ ContentRetention: contentRetention(managed),
+ }
+ if managed == nil {
+ return settings
+ }
+ settings.Enabled = managed.Enabled
+ settings.Managed = managed.Managed
+ settings.IngestURL = strings.TrimSpace(managed.IngestURL)
+ return settings
+}
+
+func Store(userMode bool) ingest.Store {
+ return ingest.Store{Path: filepath.Join(endpointconfig.BaseDir(userMode), "managed", stateFileName)}
+}
+
+func (s Source) Metadata() ingest.SourceMetadata {
+ settings := Settings(s.cfg, s.logPath, s.userMode)
+ return ingest.SourceMetadata{
+ SourceID: settings.SourceID,
+ Hostname: hostname(),
+ EndpointMode: endpointMode(s.userMode),
+ LogPath: s.logPath,
+ ContentRetention: settings.ContentRetention,
+ ManagedMode: settings.Managed,
+ }
+}
+
+func sourceID(cfg endpointconfig.Config, logPath string, userMode bool) string {
+ if cfg.ManagedUpload != nil && strings.TrimSpace(cfg.ManagedUpload.SourceID) != "" {
+ return strings.TrimSpace(cfg.ManagedUpload.SourceID)
+ }
+ sum := sha256.Sum256([]byte(fmt.Sprintf("%t|%s", userMode, logPath)))
+ return "beacon-local-" + hex.EncodeToString(sum[:])[:32]
+}
+
+func hostname() string {
+ name, err := os.Hostname()
+ if err != nil {
+ return ""
+ }
+ return name
+}
+
+func endpointMode(userMode bool) string {
+ if userMode {
+ return "user"
+ }
+ return "system"
+}
+
+func contentRetention(cfg *endpointconfig.ManagedUpload) string {
+ if cfg == nil || strings.TrimSpace(cfg.ContentRetention) == "" {
+ return "redacted"
+ }
+ return strings.TrimSpace(cfg.ContentRetention)
+}
diff --git a/cli/beacon/internal/ingest/status.go b/cli/beacon/internal/ingest/status.go
new file mode 100644
index 00000000..d0912831
--- /dev/null
+++ b/cli/beacon/internal/ingest/status.go
@@ -0,0 +1,23 @@
+package ingest
+
+import beaconauth "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/auth"
+
+func Status(settings Settings, store Store, creds *beaconauth.Credentials) State {
+ state := store.Load()
+ applyRuntimeState(&state, settings, creds)
+ return state
+}
+
+func applyRuntimeState(state *State, settings Settings, creds *beaconauth.Credentials) {
+ state.Enabled = settings.Enabled
+ state.Managed = settings.Managed
+ state.SourceID = settings.SourceID
+ state.LoggedIn = creds != nil && creds.Token != "" && !creds.IsExpired()
+ if creds != nil {
+ state.UserEmail = creds.Email
+ state.OrgName = creds.OrgName
+ }
+ if state.FileOffsets == nil {
+ state.FileOffsets = map[string]int64{}
+ }
+}
diff --git a/cli/beacon/internal/ingest/store.go b/cli/beacon/internal/ingest/store.go
new file mode 100644
index 00000000..a984ff9e
--- /dev/null
+++ b/cli/beacon/internal/ingest/store.go
@@ -0,0 +1,44 @@
+package ingest
+
+import (
+ "encoding/json"
+ "os"
+ "path/filepath"
+)
+
+type Store struct {
+ Path string
+}
+
+func (s Store) Load() State {
+ data, err := os.ReadFile(s.Path)
+ if err != nil {
+ return emptyState()
+ }
+ var state State
+ if err := json.Unmarshal(data, &state); err != nil {
+ return emptyState()
+ }
+ if state.FileOffsets == nil {
+ state.FileOffsets = map[string]int64{}
+ }
+ return state
+}
+
+func (s Store) Save(state State) error {
+ if state.FileOffsets == nil {
+ state.FileOffsets = map[string]int64{}
+ }
+ if err := os.MkdirAll(filepath.Dir(s.Path), 0700); err != nil {
+ return err
+ }
+ data, err := json.MarshalIndent(state, "", " ")
+ if err != nil {
+ return err
+ }
+ return os.WriteFile(s.Path, data, 0600)
+}
+
+func emptyState() State {
+ return State{FileOffsets: map[string]int64{}}
+}
diff --git a/cli/beacon/internal/ingest/types.go b/cli/beacon/internal/ingest/types.go
new file mode 100644
index 00000000..43bc4cae
--- /dev/null
+++ b/cli/beacon/internal/ingest/types.go
@@ -0,0 +1,100 @@
+package ingest
+
+import (
+ "encoding/json"
+ "net/http"
+
+ beaconauth "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/auth"
+)
+
+const (
+ DefaultBatchEvents = 500
+ DefaultBatchBytes = 1024 * 1024
+)
+
+type Settings struct {
+ Enabled bool
+ Managed bool
+ IngestURL string
+ SourceID string
+ ContentRetention string
+}
+
+type State struct {
+ Enabled bool `json:"enabled"`
+ Managed bool `json:"managed"`
+ LoggedIn bool `json:"logged_in"`
+ UserEmail string `json:"user_email,omitempty"`
+ OrgName string `json:"org_name,omitempty"`
+ SourceID string `json:"source_id,omitempty"`
+ LastUploadAt string `json:"last_upload_at,omitempty"`
+ LastEventAt string `json:"last_event_at,omitempty"`
+ LastCursor Cursor `json:"last_cursor,omitempty"`
+ AcceptedCount int `json:"accepted_count"`
+ RejectedCount int `json:"rejected_count"`
+ LastError string `json:"last_error,omitempty"`
+ FileOffsets map[string]int64 `json:"file_offsets,omitempty"`
+ UpdatedAt string `json:"updated_at,omitempty"`
+}
+
+type Cursor struct {
+ LogPath string `json:"log_path,omitempty"`
+ Offset int64 `json:"offset,omitempty"`
+ Line int `json:"line,omitempty"`
+ Archive string `json:"archive,omitempty"`
+}
+
+type SourceMetadata struct {
+ SourceID string
+ Hostname string
+ EndpointMode string
+ LogPath string
+ ContentRetention string
+ ManagedMode bool
+}
+
+type Batch struct {
+ Cursor Cursor
+ Events []json.RawMessage
+ Rejected int
+}
+
+type Source interface {
+ Metadata() SourceMetadata
+ Batches(state State, maxEvents int, maxBytes int) ([]Batch, error)
+}
+
+type Options struct {
+ Settings Settings
+ Creds *beaconauth.Credentials
+ Store Store
+ Client Client
+ Source Source
+ HTTPClient *http.Client
+}
+
+type Result struct {
+ State State
+ Uploaded bool
+}
+
+type uploadRequest struct {
+ UploadID string `json:"upload_id"`
+ Source uploadSource `json:"source"`
+ Cursor Cursor `json:"cursor"`
+ Events []json.RawMessage `json:"events"`
+}
+
+type uploadSource struct {
+ SourceID string `json:"source_id,omitempty"`
+ Hostname string `json:"hostname,omitempty"`
+ EndpointMode string `json:"endpoint_mode"`
+ LogPath string `json:"log_path,omitempty"`
+ ContentRetention string `json:"content_retention"`
+ ManagedMode bool `json:"managed_mode"`
+}
+
+type uploadResponse struct {
+ Accepted int `json:"accepted"`
+ Rejected int `json:"rejected"`
+}
diff --git a/cli/beacon/internal/ingest/upload.go b/cli/beacon/internal/ingest/upload.go
new file mode 100644
index 00000000..eca76816
--- /dev/null
+++ b/cli/beacon/internal/ingest/upload.go
@@ -0,0 +1,89 @@
+package ingest
+
+import (
+ "context"
+ "crypto/sha256"
+ "encoding/hex"
+ "fmt"
+ "time"
+)
+
+func Upload(ctx context.Context, opts Options) Result {
+ state := opts.Store.Load()
+ applyRuntimeState(&state, opts.Settings, opts.Creds)
+
+ if !state.Enabled || !state.Managed {
+ state.LastError = ""
+ _ = opts.Store.Save(state)
+ return Result{State: state}
+ }
+ if opts.Creds == nil || opts.Creds.Token == "" || opts.Creds.IsExpired() {
+ state.LastError = "run beacon login before endpoint ingest upload"
+ _ = opts.Store.Save(state)
+ return Result{State: state}
+ }
+ if opts.Source == nil {
+ state.LastError = "endpoint ingest source is not configured"
+ _ = opts.Store.Save(state)
+ return Result{State: state}
+ }
+
+ client := opts.Client
+ if client.URL == "" || client.HTTPClient == nil {
+ client = NewClient(opts.Settings.IngestURL, opts.HTTPClient)
+ }
+
+ batches, err := opts.Source.Batches(state, DefaultBatchEvents, DefaultBatchBytes)
+ if err != nil {
+ state.LastError = err.Error()
+ _ = opts.Store.Save(state)
+ return Result{State: state}
+ }
+
+ uploaded := false
+ for _, batch := range batches {
+ state.RejectedCount += batch.Rejected
+ if len(batch.Events) == 0 {
+ state.FileOffsets[batch.Cursor.LogPath] = batch.Cursor.Offset
+ continue
+ }
+ response, err := client.UploadBatch(ctx, opts.Creds.Token, uploadRequest{
+ UploadID: uploadID(state.SourceID, batch.Cursor.LogPath, batch.Cursor.Offset),
+ Source: uploadSourceFromMetadata(opts.Source.Metadata()),
+ Cursor: batch.Cursor,
+ Events: batch.Events,
+ })
+ if err != nil {
+ state.LastError = err.Error()
+ break
+ }
+ now := time.Now().UTC().Format(time.RFC3339)
+ state.AcceptedCount += response.Accepted
+ state.RejectedCount += response.Rejected
+ state.LastUploadAt = now
+ state.LastEventAt = now
+ state.LastCursor = batch.Cursor
+ state.FileOffsets[batch.Cursor.LogPath] = batch.Cursor.Offset
+ state.LastError = ""
+ uploaded = true
+ }
+ state.UpdatedAt = time.Now().UTC().Format(time.RFC3339)
+ _ = opts.Store.Save(state)
+ return Result{State: state, Uploaded: uploaded}
+}
+
+func uploadSourceFromMetadata(metadata SourceMetadata) uploadSource {
+ return uploadSource{
+ SourceID: metadata.SourceID,
+ Hostname: metadata.Hostname,
+ EndpointMode: metadata.EndpointMode,
+ LogPath: metadata.LogPath,
+ ContentRetention: metadata.ContentRetention,
+ ManagedMode: metadata.ManagedMode,
+ }
+}
+
+func uploadID(sourceID, path string, offset int64) string {
+ sum := sha256.Sum256([]byte(fmt.Sprintf("%s|%s|%d", sourceID, path, offset)))
+ return hex.EncodeToString(sum[:])
+}
diff --git a/cli/beacon/internal/ingest/upload_test.go b/cli/beacon/internal/ingest/upload_test.go
new file mode 100644
index 00000000..7cca85eb
--- /dev/null
+++ b/cli/beacon/internal/ingest/upload_test.go
@@ -0,0 +1,147 @@
+package ingest
+
+import (
+ "context"
+ "encoding/json"
+ "net/http"
+ "net/http/httptest"
+ "os"
+ "path/filepath"
+ "testing"
+
+ beaconauth "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/auth"
+)
+
+type fakeSource struct {
+ metadata SourceMetadata
+ batches []Batch
+ called bool
+}
+
+func (s *fakeSource) Metadata() SourceMetadata {
+ return s.metadata
+}
+
+func (s *fakeSource) Batches(state State, maxEvents int, maxBytes int) ([]Batch, error) {
+ s.called = true
+ return s.batches, nil
+}
+
+func TestStatusIsReadOnly(t *testing.T) {
+ statePath := filepath.Join(t.TempDir(), "upload-state.json")
+ state := Status(Settings{Enabled: true, Managed: true, SourceID: "source-1"}, Store{Path: statePath}, &beaconauth.Credentials{
+ Token: "token",
+ UserID: "user",
+ Email: "[email protected]",
+ })
+
+ if !state.Enabled || !state.Managed || !state.LoggedIn || state.SourceID != "source-1" {
+ t.Fatalf("unexpected status: %#v", state)
+ }
+ if _, err := os.Stat(statePath); !os.IsNotExist(err) {
+ t.Fatalf("Status wrote state file, err=%v", err)
+ }
+}
+
+func TestUploadIsInertUnlessManagedEnabled(t *testing.T) {
+ source := &fakeSource{}
+ res := Upload(context.Background(), Options{
+ Settings: Settings{Enabled: false, Managed: false},
+ Store: Store{Path: filepath.Join(t.TempDir(), "upload-state.json")},
+ Creds: &beaconauth.Credentials{Token: "token", UserID: "user"},
+ Source: source,
+ })
+
+ if source.called {
+ t.Fatal("disabled upload should not read source batches")
+ }
+ if res.Uploaded || res.State.AcceptedCount != 0 || res.State.LastUploadAt != "" {
+ t.Fatalf("disabled upload should not send events: %#v", res.State)
+ }
+}
+
+func TestUploadRequiresLogin(t *testing.T) {
+ source := &fakeSource{}
+ res := Upload(context.Background(), Options{
+ Settings: Settings{Enabled: true, Managed: true, SourceID: "source-1"},
+ Store: Store{Path: filepath.Join(t.TempDir(), "upload-state.json")},
+ Source: source,
+ })
+
+ if source.called {
+ t.Fatal("upload without login should not read source batches")
+ }
+ if res.State.LastError == "" {
+ t.Fatalf("expected login error: %#v", res.State)
+ }
+}
+
+func TestUploadPostsBatchAndPersistsCursor(t *testing.T) {
+ var received uploadRequest
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if got := r.Header.Get("Authorization"); got != "Bearer token" {
+ t.Fatalf("Authorization = %q, want bearer token", got)
+ }
+ if err := json.NewDecoder(r.Body).Decode(&received); err != nil {
+ t.Fatal(err)
+ }
+ _, _ = w.Write([]byte(`{"accepted":1,"rejected":0}`))
+ }))
+ defer server.Close()
+
+ statePath := filepath.Join(t.TempDir(), "upload-state.json")
+ source := &fakeSource{
+ metadata: SourceMetadata{SourceID: "source-1", EndpointMode: "user", LogPath: "runtime.jsonl", ContentRetention: "redacted", ManagedMode: true},
+ batches: []Batch{{
+ Cursor: Cursor{LogPath: "runtime.jsonl", Offset: 128, Line: 1},
+ Events: []json.RawMessage{json.RawMessage(`{"event":"ok"}`)},
+ }},
+ }
+ res := Upload(context.Background(), Options{
+ Settings: Settings{Enabled: true, Managed: true, SourceID: "source-1", IngestURL: server.URL},
+ Store: Store{Path: statePath},
+ Creds: &beaconauth.Credentials{Token: "token", UserID: "user"},
+ Source: source,
+ })
+
+ if res.State.LastError != "" {
+ t.Fatalf("LastError = %q", res.State.LastError)
+ }
+ if !res.Uploaded || res.State.AcceptedCount != 1 || res.State.FileOffsets["runtime.jsonl"] != 128 {
+ t.Fatalf("unexpected upload state: %#v", res.State)
+ }
+ if received.Source.SourceID != "source-1" || !received.Source.ManagedMode || len(received.Events) != 1 {
+ t.Fatalf("unexpected request: %#v", received)
+ }
+ if _, err := os.Stat(statePath); err != nil {
+ t.Fatalf("state file not written: %v", err)
+ }
+}
+
+func TestUploadDoesNotAdvanceCursorOnFailedPost(t *testing.T) {
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ http.Error(w, "nope", http.StatusInternalServerError)
+ }))
+ defer server.Close()
+
+ source := &fakeSource{
+ metadata: SourceMetadata{SourceID: "source-1", EndpointMode: "user", LogPath: "runtime.jsonl", ContentRetention: "redacted", ManagedMode: true},
+ batches: []Batch{{
+ Cursor: Cursor{LogPath: "runtime.jsonl", Offset: 128, Line: 1},
+ Events: []json.RawMessage{json.RawMessage(`{"event":"ok"}`)},
+ }},
+ }
+ res := Upload(context.Background(), Options{
+ Settings: Settings{Enabled: true, Managed: true, SourceID: "source-1", IngestURL: server.URL},
+ Store: Store{Path: filepath.Join(t.TempDir(), "upload-state.json")},
+ Creds: &beaconauth.Credentials{Token: "token", UserID: "user"},
+ Source: source,
+ })
+
+ if res.State.LastError == "" {
+ t.Fatal("expected upload error")
+ }
+ if res.State.FileOffsets["runtime.jsonl"] != 0 || res.State.LastCursor.Offset != 0 {
+ t.Fatalf("cursor advanced on failed post: %#v", res.State)
+ }
+}