The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu
Go agent-ai-cliagent-harnessesagent-securitybrowser-extensionciclaude-codecloudcodexcursordetection-engineeringjevknowledge-basememorymobile-device-managementobservabilitysecuritysecurity-information-and-event-managementsecurity-toolstelemetrytraces
add-managed-ingest-support
Justin D'Souzad5ca52109e68
diff --git a/cli/beacon/cmd/endpoint_install.go b/cli/beacon/cmd/endpoint_install.goindex edb49738..564b9b73 100644--- a/cli/beacon/cmd/endpoint_install.go+++ b/cli/beacon/cmd/endpoint_install.go@@ -157,6 +157,7 @@ func runEndpointStatus(cmd *cobra.Command, args []string) error {} else {fmt.Println("Last event: present")}+ printIngestStatus("Endpoint ingest", status.ManagedUpload)return nil}diff --git a/cli/beacon/cmd/ingest.go b/cli/beacon/cmd/ingest.gonew file mode 100644index 00000000..7e8d90b9--- /dev/null+++ b/cli/beacon/cmd/ingest.go@@ -0,0 +1,146 @@+package cmd++import (+ "context"+ "encoding/json"+ "fmt"+ "os"+ "time"++ beaconauth "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/auth"+ endpointconfig "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/config"+ "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/lifecycle"+ "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/ingest"+ endpointingest "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/ingest/endpoint"+ "github.com/spf13/cobra"+)++var ingestCmd = &cobra.Command{+ Use: "ingest",+ Short: "Upload Beacon telemetry to configured ingest destinations",+}++var ingestStatusCmd = &cobra.Command{+ Use: "status",+ Short: "Show configured ingest status",+ SilenceUsage: true,+ RunE: func(cmd *cobra.Command, args []string) error {+ state := endpointIngestStatus()+ if endpointOpts.jsonOutput {+ return json.NewEncoder(os.Stdout).Encode(map[string]ingest.State{+ "endpoint": state,+ })+ }+ fmt.Println("Beacon ingest")+ printIngestStatus("Endpoint", state)+ return nil+ },+}++var ingestEndpointCmd = &cobra.Command{+ Use: "endpoint",+ Short: "Manage endpoint telemetry ingest",+}++var ingestEndpointStatusCmd = &cobra.Command{+ Use: "status",+ Short: "Show endpoint telemetry upload status",+ SilenceUsage: true,+ RunE: func(cmd *cobra.Command, args []string) error {+ state := endpointIngestStatus()+ if endpointOpts.jsonOutput {+ return json.NewEncoder(os.Stdout).Encode(state)+ }+ printIngestStatus("Endpoint ingest", state)+ return nil+ },+}++var ingestEndpointUploadCmd = &cobra.Command{+ Use: "upload",+ Short: "Upload endpoint telemetry to managed ingest",+ SilenceUsage: true,+ RunE: func(cmd *cobra.Command, args []string) error {+ res, err := uploadEndpointIngest(cmd.Context())+ if endpointOpts.jsonOutput {+ if encodeErr := json.NewEncoder(os.Stdout).Encode(res.State); encodeErr != nil {+ return encodeErr+ }+ } else {+ printIngestStatus("Endpoint ingest", res.State)+ }+ if err != nil {+ return err+ }+ return nil+ },+}++func init() {+ rootCmd.AddCommand(ingestCmd)+ ingestCmd.AddCommand(ingestStatusCmd)+ ingestCmd.AddCommand(ingestEndpointCmd)+ ingestEndpointCmd.AddCommand(ingestEndpointStatusCmd)+ ingestEndpointCmd.AddCommand(ingestEndpointUploadCmd)++ for _, c := range []*cobra.Command{ingestStatusCmd, ingestEndpointStatusCmd, ingestEndpointUploadCmd} {+ c.Flags().BoolVar(&endpointOpts.userMode, "user", true, "Use per-user endpoint paths")+ c.Flags().BoolVar(&endpointOpts.systemMode, "system", false, "Use system endpoint paths and launch daemon")+ c.Flags().StringVar(&endpointOpts.logPath, "log-path", "", "Runtime JSONL log path")+ c.Flags().BoolVar(&endpointOpts.jsonOutput, "json", false, "Print output as JSON")+ }+}++func endpointIngestStatus() ingest.State {+ cfg, effectiveUserMode, logPath := endpointIngestConfig()+ creds, _ := beaconauth.LoadCredentials()+ return ingest.Status(+ endpointingest.Settings(cfg, logPath, effectiveUserMode),+ endpointingest.Store(effectiveUserMode),+ creds,+ )+}++func uploadEndpointIngest(ctx context.Context) (ingest.Result, error) {+ cfg, effectiveUserMode, logPath := endpointIngestConfig()+ creds, _ := beaconauth.LoadCredentials()+ uploadCtx, cancel := context.WithTimeout(ctx, 30*time.Second)+ defer cancel()+ res := ingest.Upload(uploadCtx, ingest.Options{+ Settings: endpointingest.Settings(cfg, logPath, effectiveUserMode),+ Creds: creds,+ Store: endpointingest.Store(effectiveUserMode),+ Source: endpointingest.NewSource(cfg, logPath, effectiveUserMode),+ })+ if res.State.Enabled && res.State.Managed && res.State.LastError != "" {+ return res, fmt.Errorf("%s", res.State.LastError)+ }+ return res, nil+}++func endpointIngestConfig() (endpointconfig.Config, bool, string) {+ cfg := loadOrDefaultConfig()+ runtimeLog := lifecycle.ResolveRuntimeLog(endpointUserMode(), endpointOpts.logPath)+ effectiveCfg := cfg+ effectiveCfg.UserMode = runtimeLog.EffectiveUserMode+ if runtimeLog.EffectiveUserMode != cfg.UserMode {+ effectiveCfg = loadConfigForMode(runtimeLog.EffectiveUserMode, runtimeLog.EffectiveLogPath)+ }+ effectiveCfg.LogPath = runtimeLog.EffectiveLogPath+ return effectiveCfg, effectiveCfg.UserMode, effectiveCfg.LogPath+}++func printIngestStatus(label string, state ingest.State) {+ fmt.Printf("%s: enabled=%t managed=%t logged_in=%t", label, state.Enabled, state.Managed, state.LoggedIn)+ if state.LastUploadAt != "" {+ fmt.Printf(" last_upload=%s", state.LastUploadAt)+ }+ if state.LastCursor.Offset > 0 {+ fmt.Printf(" cursor_offset=%d", state.LastCursor.Offset)+ }+ fmt.Printf(" accepted=%d rejected=%d", state.AcceptedCount, state.RejectedCount)+ if state.LastError != "" {+ fmt.Printf(" error=%s", state.LastError)+ }+ fmt.Println()+}diff --git a/cli/beacon/internal/endpoint/config/config.go b/cli/beacon/internal/endpoint/config/config.goindex 01e133da..340a5d3e 100644--- a/cli/beacon/internal/endpoint/config/config.go+++ b/cli/beacon/internal/endpoint/config/config.go@@ -22,12 +22,13 @@ const ()type Config struct {- UserMode bool `json:"user_mode"`- LogPath string `json:"log_path"`- Collector Collector `json:"collector"`- Harnesses []string `json:"harnesses"`- EventCategories []string `json:"event_categories,omitempty"`- Destinations *Destinations `json:"destinations,omitempty"`+ UserMode bool `json:"user_mode"`+ LogPath string `json:"log_path"`+ Collector Collector `json:"collector"`+ Harnesses []string `json:"harnesses"`+ EventCategories []string `json:"event_categories,omitempty"`+ Destinations *Destinations `json:"destinations,omitempty"`+ ManagedUpload *ManagedUpload `json:"managed_upload,omitempty"`}type Collector struct {@@ -45,6 +46,14 @@ type Destinations struct {FalconHEC *FalconHEC `json:"falcon_hec,omitempty"`}+type ManagedUpload struct {+ Enabled bool `json:"enabled,omitempty"`+ Managed bool `json:"managed,omitempty"`+ IngestURL string `json:"ingest_url,omitempty"`+ SourceID string `json:"source_id,omitempty"`+ ContentRetention string `json:"content_retention,omitempty"`+}+type SplunkHEC struct {Enabled bool `json:"enabled,omitempty"`Endpoint string `json:"endpoint,omitempty"`diff --git a/cli/beacon/internal/endpoint/lifecycle/lifecycle.go b/cli/beacon/internal/endpoint/lifecycle/lifecycle.goindex 7e46c154..d67111eb 100644--- a/cli/beacon/internal/endpoint/lifecycle/lifecycle.go+++ b/cli/beacon/internal/endpoint/lifecycle/lifecycle.go@@ -8,6 +8,7 @@ import ("runtime""time"+ beaconauth "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/auth"endpointcollector "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/collector"endpointconfig "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/config""github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/diagnostics"@@ -15,6 +16,8 @@ import ("github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/schema""github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/service""github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/writer"+ "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/ingest"+ endpointingest "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/ingest/endpoint""github.com/asymptote-labs/agent-beacon/cli/beacon/internal/version")@@ -55,16 +58,17 @@ type InstallResult struct {}type Status struct {- Version string `json:"version"`- ConfigPath string `json:"config_path"`- LogPath string `json:"log_path"`- RuntimeLog RuntimeLogSource `json:"runtime_log"`- Collector endpointcollector.Status `json:"collector"`- Service service.Status `json:"service"`- Harnesses []harness.Harness `json:"harnesses"`- Diagnostics []diagnostics.Check `json:"diagnostics"`- LastEvent string `json:"last_event,omitempty"`- Destinations DestinationStatus `json:"destinations"`+ Version string `json:"version"`+ ConfigPath string `json:"config_path"`+ LogPath string `json:"log_path"`+ RuntimeLog RuntimeLogSource `json:"runtime_log"`+ Collector endpointcollector.Status `json:"collector"`+ Service service.Status `json:"service"`+ Harnesses []harness.Harness `json:"harnesses"`+ Diagnostics []diagnostics.Check `json:"diagnostics"`+ LastEvent string `json:"last_event,omitempty"`+ Destinations DestinationStatus `json:"destinations"`+ ManagedUpload ingest.State `json:"managed_upload"`}type DestinationStatus struct {@@ -293,17 +297,24 @@ func GetStatus(userMode bool, logPath string) Status {Message: runtimeLog.Warning,})}+ creds, _ := beaconauth.LoadCredentials()+ managedUpload := ingest.Status(+ endpointingest.Settings(effectiveCfg, effectiveCfg.LogPath, effectiveCfg.UserMode),+ endpointingest.Store(effectiveCfg.UserMode),+ creds,+ )return Status{- Version: version.GetVersion(),- ConfigPath: endpointconfig.ConfigPath(effectiveCfg.UserMode),- LogPath: effectiveCfg.LogPath,- RuntimeLog: runtimeLog,- Collector: endpointcollector.CheckStatus(effectiveCfg),- Service: service.Manager{UserMode: effectiveCfg.UserMode}.Status(),- Harnesses: harness.DiscoverAll(),- Diagnostics: checks,- LastEvent: last,- Destinations: destinationStatus(effectiveCfg),+ Version: version.GetVersion(),+ ConfigPath: endpointconfig.ConfigPath(effectiveCfg.UserMode),+ LogPath: effectiveCfg.LogPath,+ RuntimeLog: runtimeLog,+ Collector: endpointcollector.CheckStatus(effectiveCfg),+ Service: service.Manager{UserMode: effectiveCfg.UserMode}.Status(),+ Harnesses: harness.DiscoverAll(),+ Diagnostics: checks,+ LastEvent: last,+ Destinations: destinationStatus(effectiveCfg),+ ManagedUpload: managedUpload,}}diff --git a/cli/beacon/internal/endpoint/lifecycle/lifecycle_test.go b/cli/beacon/internal/endpoint/lifecycle/lifecycle_test.goindex 653075e8..49bb459f 100644--- a/cli/beacon/internal/endpoint/lifecycle/lifecycle_test.go+++ b/cli/beacon/internal/endpoint/lifecycle/lifecycle_test.go@@ -3,12 +3,15 @@ package lifecycleimport ("errors""net"+ "net/http"+ "net/http/httptest""os""path/filepath""runtime""strings""testing"+ beaconauth "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/auth"endpointconfig "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/config""github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/harness""github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/schema"@@ -159,6 +162,47 @@ func TestConfigureHarnessesRejectsUnsupportedHarness(t *testing.T) {}}+func TestGetStatusDoesNotUploadManagedTelemetry(t *testing.T) {+ home := t.TempDir()+ t.Setenv("HOME", home)+ logPath := filepath.Join(home, "runtime.jsonl")+ if err := os.WriteFile(logPath, []byte("{\"event\":\"ok\"}\n"), 0600); err != nil {+ t.Fatal(err)+ }+ called := false+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ called = true+ http.Error(w, "status should not upload", http.StatusInternalServerError)+ }))+ defer server.Close()++ _, err := endpointconfig.Save(endpointconfig.Config{+ UserMode: true,+ LogPath: logPath,+ Harnesses: []string{},+ ManagedUpload: &endpointconfig.ManagedUpload{+ Enabled: true,+ Managed: true,+ IngestURL: server.URL,+ SourceID: "source-1",+ },+ })+ if err != nil {+ t.Fatal(err)+ }+ if err := beaconauth.SaveCredentials(&beaconauth.Credentials{Token: "token", UserID: "user"}); err != nil {+ t.Fatal(err)+ }++ status := GetStatus(true, logPath)+ if !status.ManagedUpload.Enabled || !status.ManagedUpload.Managed || !status.ManagedUpload.LoggedIn {+ t.Fatalf("unexpected managed upload status: %#v", status.ManagedUpload)+ }+ if called {+ t.Fatal("GetStatus performed a managed upload network call")+ }+}+func TestConfigureHarnessesRejectsFactoryAsMDMManaged(t *testing.T) {cfg := endpointconfig.Config{Harnesses: []string{"factory"},diff --git a/cli/beacon/internal/ingest/client.go b/cli/beacon/internal/ingest/client.gonew file mode 100644index 00000000..e94c0977--- /dev/null+++ b/cli/beacon/internal/ingest/client.go@@ -0,0 +1,65 @@+package ingest++import (+ "bytes"+ "context"+ "encoding/json"+ "fmt"+ "io"+ "net/http"+ "strings"+ "time"++ beaconauth "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/auth"+ "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/version"+)++const defaultIngestPath = "/api/v1/ingest/beacon/events"++type Client struct {+ URL string+ HTTPClient *http.Client+}++func NewClient(ingestURL string, httpClient *http.Client) Client {+ if strings.TrimSpace(ingestURL) == "" {+ ingestURL = beaconauth.ResolveDashboardURL("") + defaultIngestPath+ }+ if httpClient == nil {+ httpClient = &http.Client{Timeout: 15 * time.Second}+ }+ return Client{+ URL: strings.TrimRight(ingestURL, "/"),+ HTTPClient: httpClient,+ }+}++func (c Client) UploadBatch(ctx context.Context, token string, reqBody uploadRequest) (uploadResponse, error) {+ data, err := json.Marshal(reqBody)+ if err != nil {+ return uploadResponse{}, err+ }+ req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.URL, bytes.NewReader(data))+ if err != nil {+ return uploadResponse{}, err+ }+ req.Header.Set("Authorization", "Bearer "+token)+ req.Header.Set("Content-Type", "application/json")+ req.Header.Set("User-Agent", "beacon/"+version.GetVersion())++ resp, err := c.HTTPClient.Do(req)+ if err != nil {+ return uploadResponse{}, err+ }+ defer resp.Body.Close()++ respBody, _ := io.ReadAll(io.LimitReader(resp.Body, 4096))+ if resp.StatusCode < 200 || resp.StatusCode >= 300 {+ return uploadResponse{}, fmt.Errorf("ingest upload failed: HTTP %d: %s", resp.StatusCode, strings.TrimSpace(string(respBody)))+ }+ var parsed uploadResponse+ if err := json.Unmarshal(respBody, &parsed); err != nil && len(respBody) > 0 {+ return uploadResponse{}, fmt.Errorf("ingest upload response was invalid: %w", err)+ }+ return parsed, nil+}diff --git a/cli/beacon/internal/ingest/endpoint/reader.go b/cli/beacon/internal/ingest/endpoint/reader.gonew file mode 100644index 00000000..d8d75092--- /dev/null+++ b/cli/beacon/internal/ingest/endpoint/reader.go@@ -0,0 +1,142 @@+package endpoint++import (+ "bufio"+ "bytes"+ "encoding/json"+ "os"+ "path/filepath"+ "sort"+ "strings"++ "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/ingest"+)++func (s Source) Batches(state ingest.State, maxEvents int, maxBytes int) ([]ingest.Batch, error) {+ if maxEvents <= 0 {+ maxEvents = ingest.DefaultBatchEvents+ }+ if maxBytes <= 0 {+ maxBytes = ingest.DefaultBatchBytes+ }+ files, err := logFiles(s.logPath)+ if err != nil {+ return nil, err+ }+ var batches []ingest.Batch+ for _, file := range files {+ fileBatches, err := readFileBatches(file, s.logPath, state, maxEvents, maxBytes)+ if err != nil {+ return nil, err+ }+ batches = append(batches, fileBatches...)+ }+ return batches, nil+}++func readFileBatches(path string, activePath string, state ingest.State, maxEvents int, maxBytes int) ([]ingest.Batch, error) {+ startOffset := state.FileOffsets[path]+ file, err := os.Open(path)+ if err != nil {+ if os.IsNotExist(err) {+ return nil, nil+ }+ return nil, err+ }+ defer file.Close()++ if startOffset > 0 {+ if _, err := file.Seek(startOffset, 0); err != nil {+ return nil, err+ }+ }++ scanner := bufio.NewScanner(file)+ scanner.Buffer(make([]byte, 0, 64*1024), maxBytes)+ var (+ batches []ingest.Batch+ events []json.RawMessage+ batchSize int+ batchRejected int+ batchEndOffset int64+ batchLine int+ line int+ offset = startOffset+ )+ flush := func() {+ if len(events) == 0 && batchRejected == 0 {+ return+ }+ batches = append(batches, ingest.Batch{+ Cursor: ingest.Cursor{+ LogPath: path,+ Offset: batchEndOffset,+ Line: batchLine,+ Archive: archiveName(activePath, path),+ },+ Events: events,+ Rejected: batchRejected,+ })+ events = nil+ batchSize = 0+ batchRejected = 0+ }++ for scanner.Scan() {+ line+++ raw := append([]byte(nil), scanner.Bytes()...)+ nextOffset := offset + int64(len(raw)) + 1+ trimmed := bytes.TrimSpace(raw)+ if len(trimmed) == 0 {+ batchEndOffset = nextOffset+ batchLine = line+ offset = nextOffset+ continue+ }+ if len(events) > 0 && (len(events) >= maxEvents || batchSize+len(trimmed) > maxBytes) {+ flush()+ }+ if !json.Valid(trimmed) {+ batchRejected+++ batchEndOffset = nextOffset+ batchLine = line+ offset = nextOffset+ continue+ }+ events = append(events, json.RawMessage(trimmed))+ batchSize += len(trimmed)+ batchEndOffset = nextOffset+ batchLine = line+ offset = nextOffset+ }+ if err := scanner.Err(); err != nil {+ return nil, err+ }+ flush()+ return batches, nil+}++func logFiles(path string) ([]string, error) {+ dir := filepath.Dir(path)+ base := filepath.Base(path)+ entries, err := os.ReadDir(dir)+ if err != nil {+ return nil, err+ }+ var archives []string+ for _, entry := range entries {+ name := entry.Name()+ if strings.HasPrefix(name, base+".") {+ archives = append(archives, filepath.Join(dir, name))+ }+ }+ sort.Strings(archives)+ return append(archives, path), nil+}++func archiveName(activePath string, path string) string {+ if activePath == path {+ return ""+ }+ return filepath.Base(path)+}diff --git a/cli/beacon/internal/ingest/endpoint/reader_test.go b/cli/beacon/internal/ingest/endpoint/reader_test.gonew file mode 100644index 00000000..3ce94082--- /dev/null+++ b/cli/beacon/internal/ingest/endpoint/reader_test.go@@ -0,0 +1,61 @@+package endpoint++import (+ "os"+ "path/filepath"+ "testing"++ endpointconfig "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/config"+ "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/ingest"+)++func TestEndpointSourceBatchesValidEventsAndMalformedRejects(t *testing.T) {+ dir := t.TempDir()+ logPath := filepath.Join(dir, "runtime.jsonl")+ content := []byte("{\"event\":\"one\"}\nnot-json\n{\"event\":\"two\"}\n")+ if err := os.WriteFile(logPath, content, 0600); err != nil {+ t.Fatal(err)+ }++ source := NewSource(endpointconfig.Config{UserMode: true, LogPath: logPath}, logPath, true)+ batches, err := source.Batches(ingest.State{FileOffsets: map[string]int64{}}, 500, 1024)+ if err != nil {+ t.Fatal(err)+ }++ if len(batches) != 1 {+ t.Fatalf("len(batches) = %d, want 1", len(batches))+ }+ if got := len(batches[0].Events); got != 2 {+ t.Fatalf("events = %d, want 2", got)+ }+ if batches[0].Rejected != 1 {+ t.Fatalf("Rejected = %d, want 1", batches[0].Rejected)+ }+ if batches[0].Cursor.Offset != int64(len(content)) {+ t.Fatalf("Offset = %d, want %d", batches[0].Cursor.Offset, len(content))+ }+}++func TestEndpointSourceHonorsSavedOffset(t *testing.T) {+ dir := t.TempDir()+ logPath := filepath.Join(dir, "runtime.jsonl")+ first := "{\"event\":\"one\"}\n"+ second := "{\"event\":\"two\"}\n"+ if err := os.WriteFile(logPath, []byte(first+second), 0600); err != nil {+ t.Fatal(err)+ }++ source := NewSource(endpointconfig.Config{UserMode: true, LogPath: logPath}, logPath, true)+ batches, err := source.Batches(ingest.State{FileOffsets: map[string]int64{logPath: int64(len(first))}}, 500, 1024)+ if err != nil {+ t.Fatal(err)+ }++ if len(batches) != 1 || len(batches[0].Events) != 1 {+ t.Fatalf("unexpected batches: %#v", batches)+ }+ if string(batches[0].Events[0]) != "{\"event\":\"two\"}" {+ t.Fatalf("event = %s", batches[0].Events[0])+ }+}diff --git a/cli/beacon/internal/ingest/endpoint/source.go b/cli/beacon/internal/ingest/endpoint/source.gonew file mode 100644index 00000000..4c5a2f25--- /dev/null+++ b/cli/beacon/internal/ingest/endpoint/source.go@@ -0,0 +1,90 @@+package endpoint++import (+ "crypto/sha256"+ "encoding/hex"+ "fmt"+ "os"+ "path/filepath"+ "strings"++ endpointconfig "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/config"+ "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/ingest"+)++const stateFileName = "upload-state.json"++type Source struct {+ cfg endpointconfig.Config+ logPath string+ userMode bool+}++func NewSource(cfg endpointconfig.Config, logPath string, userMode bool) Source {+ return Source{+ cfg: cfg,+ logPath: logPath,+ userMode: userMode,+ }+}++func Settings(cfg endpointconfig.Config, logPath string, userMode bool) ingest.Settings {+ managed := cfg.ManagedUpload+ settings := ingest.Settings{+ SourceID: sourceID(cfg, logPath, userMode),+ ContentRetention: contentRetention(managed),+ }+ if managed == nil {+ return settings+ }+ settings.Enabled = managed.Enabled+ settings.Managed = managed.Managed+ settings.IngestURL = strings.TrimSpace(managed.IngestURL)+ return settings+}++func Store(userMode bool) ingest.Store {+ return ingest.Store{Path: filepath.Join(endpointconfig.BaseDir(userMode), "managed", stateFileName)}+}++func (s Source) Metadata() ingest.SourceMetadata {+ settings := Settings(s.cfg, s.logPath, s.userMode)+ return ingest.SourceMetadata{+ SourceID: settings.SourceID,+ Hostname: hostname(),+ EndpointMode: endpointMode(s.userMode),+ LogPath: s.logPath,+ ContentRetention: settings.ContentRetention,+ ManagedMode: settings.Managed,+ }+}++func sourceID(cfg endpointconfig.Config, logPath string, userMode bool) string {+ if cfg.ManagedUpload != nil && strings.TrimSpace(cfg.ManagedUpload.SourceID) != "" {+ return strings.TrimSpace(cfg.ManagedUpload.SourceID)+ }+ sum := sha256.Sum256([]byte(fmt.Sprintf("%t|%s", userMode, logPath)))+ return "beacon-local-" + hex.EncodeToString(sum[:])[:32]+}++func hostname() string {+ name, err := os.Hostname()+ if err != nil {+ return ""+ }+ return name+}++func endpointMode(userMode bool) string {+ if userMode {+ return "user"+ }+ return "system"+}++func contentRetention(cfg *endpointconfig.ManagedUpload) string {+ if cfg == nil || strings.TrimSpace(cfg.ContentRetention) == "" {+ return "redacted"+ }+ return strings.TrimSpace(cfg.ContentRetention)+}diff --git a/cli/beacon/internal/ingest/status.go b/cli/beacon/internal/ingest/status.gonew file mode 100644index 00000000..d0912831--- /dev/null+++ b/cli/beacon/internal/ingest/status.go@@ -0,0 +1,23 @@+package ingest++import beaconauth "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/auth"++func Status(settings Settings, store Store, creds *beaconauth.Credentials) State {+ state := store.Load()+ applyRuntimeState(&state, settings, creds)+ return state+}++func applyRuntimeState(state *State, settings Settings, creds *beaconauth.Credentials) {+ state.Enabled = settings.Enabled+ state.Managed = settings.Managed+ state.SourceID = settings.SourceID+ state.LoggedIn = creds != nil && creds.Token != "" && !creds.IsExpired()+ if creds != nil {+ state.UserEmail = creds.Email+ state.OrgName = creds.OrgName+ }+ if state.FileOffsets == nil {+ state.FileOffsets = map[string]int64{}+ }+}diff --git a/cli/beacon/internal/ingest/store.go b/cli/beacon/internal/ingest/store.gonew file mode 100644index 00000000..a984ff9e--- /dev/null+++ b/cli/beacon/internal/ingest/store.go@@ -0,0 +1,44 @@+package ingest++import (+ "encoding/json"+ "os"+ "path/filepath"+)++type Store struct {+ Path string+}++func (s Store) Load() State {+ data, err := os.ReadFile(s.Path)+ if err != nil {+ return emptyState()+ }+ var state State+ if err := json.Unmarshal(data, &state); err != nil {+ return emptyState()+ }+ if state.FileOffsets == nil {+ state.FileOffsets = map[string]int64{}+ }+ return state+}++func (s Store) Save(state State) error {+ if state.FileOffsets == nil {+ state.FileOffsets = map[string]int64{}+ }+ if err := os.MkdirAll(filepath.Dir(s.Path), 0700); err != nil {+ return err+ }+ data, err := json.MarshalIndent(state, "", " ")+ if err != nil {+ return err+ }+ return os.WriteFile(s.Path, data, 0600)+}++func emptyState() State {+ return State{FileOffsets: map[string]int64{}}+}diff --git a/cli/beacon/internal/ingest/types.go b/cli/beacon/internal/ingest/types.gonew file mode 100644index 00000000..43bc4cae--- /dev/null+++ b/cli/beacon/internal/ingest/types.go@@ -0,0 +1,100 @@+package ingest++import (+ "encoding/json"+ "net/http"++ beaconauth "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/auth"+)++const (+ DefaultBatchEvents = 500+ DefaultBatchBytes = 1024 * 1024+)++type Settings struct {+ Enabled bool+ Managed bool+ IngestURL string+ SourceID string+ ContentRetention string+}++type State struct {+ Enabled bool `json:"enabled"`+ Managed bool `json:"managed"`+ LoggedIn bool `json:"logged_in"`+ UserEmail string `json:"user_email,omitempty"`+ OrgName string `json:"org_name,omitempty"`+ SourceID string `json:"source_id,omitempty"`+ LastUploadAt string `json:"last_upload_at,omitempty"`+ LastEventAt string `json:"last_event_at,omitempty"`+ LastCursor Cursor `json:"last_cursor,omitempty"`+ AcceptedCount int `json:"accepted_count"`+ RejectedCount int `json:"rejected_count"`+ LastError string `json:"last_error,omitempty"`+ FileOffsets map[string]int64 `json:"file_offsets,omitempty"`+ UpdatedAt string `json:"updated_at,omitempty"`+}++type Cursor struct {+ LogPath string `json:"log_path,omitempty"`+ Offset int64 `json:"offset,omitempty"`+ Line int `json:"line,omitempty"`+ Archive string `json:"archive,omitempty"`+}++type SourceMetadata struct {+ SourceID string+ Hostname string+ EndpointMode string+ LogPath string+ ContentRetention string+ ManagedMode bool+}++type Batch struct {+ Cursor Cursor+ Events []json.RawMessage+ Rejected int+}++type Source interface {+ Metadata() SourceMetadata+ Batches(state State, maxEvents int, maxBytes int) ([]Batch, error)+}++type Options struct {+ Settings Settings+ Creds *beaconauth.Credentials+ Store Store+ Client Client+ Source Source+ HTTPClient *http.Client+}++type Result struct {+ State State+ Uploaded bool+}++type uploadRequest struct {+ UploadID string `json:"upload_id"`+ Source uploadSource `json:"source"`+ Cursor Cursor `json:"cursor"`+ Events []json.RawMessage `json:"events"`+}++type uploadSource struct {+ SourceID string `json:"source_id,omitempty"`+ Hostname string `json:"hostname,omitempty"`+ EndpointMode string `json:"endpoint_mode"`+ LogPath string `json:"log_path,omitempty"`+ ContentRetention string `json:"content_retention"`+ ManagedMode bool `json:"managed_mode"`+}++type uploadResponse struct {+ Accepted int `json:"accepted"`+ Rejected int `json:"rejected"`+}diff --git a/cli/beacon/internal/ingest/upload.go b/cli/beacon/internal/ingest/upload.gonew file mode 100644index 00000000..eca76816--- /dev/null+++ b/cli/beacon/internal/ingest/upload.go@@ -0,0 +1,89 @@+package ingest++import (+ "context"+ "crypto/sha256"+ "encoding/hex"+ "fmt"+ "time"+)++func Upload(ctx context.Context, opts Options) Result {+ state := opts.Store.Load()+ applyRuntimeState(&state, opts.Settings, opts.Creds)++ if !state.Enabled || !state.Managed {+ state.LastError = ""+ _ = opts.Store.Save(state)+ return Result{State: state}+ }+ if opts.Creds == nil || opts.Creds.Token == "" || opts.Creds.IsExpired() {+ state.LastError = "run beacon login before endpoint ingest upload"+ _ = opts.Store.Save(state)+ return Result{State: state}+ }+ if opts.Source == nil {+ state.LastError = "endpoint ingest source is not configured"+ _ = opts.Store.Save(state)+ return Result{State: state}+ }++ client := opts.Client+ if client.URL == "" || client.HTTPClient == nil {+ client = NewClient(opts.Settings.IngestURL, opts.HTTPClient)+ }++ batches, err := opts.Source.Batches(state, DefaultBatchEvents, DefaultBatchBytes)+ if err != nil {+ state.LastError = err.Error()+ _ = opts.Store.Save(state)+ return Result{State: state}+ }++ uploaded := false+ for _, batch := range batches {+ state.RejectedCount += batch.Rejected+ if len(batch.Events) == 0 {+ state.FileOffsets[batch.Cursor.LogPath] = batch.Cursor.Offset+ continue+ }+ response, err := client.UploadBatch(ctx, opts.Creds.Token, uploadRequest{+ UploadID: uploadID(state.SourceID, batch.Cursor.LogPath, batch.Cursor.Offset),+ Source: uploadSourceFromMetadata(opts.Source.Metadata()),+ Cursor: batch.Cursor,+ Events: batch.Events,+ })+ if err != nil {+ state.LastError = err.Error()+ break+ }+ now := time.Now().UTC().Format(time.RFC3339)+ state.AcceptedCount += response.Accepted+ state.RejectedCount += response.Rejected+ state.LastUploadAt = now+ state.LastEventAt = now+ state.LastCursor = batch.Cursor+ state.FileOffsets[batch.Cursor.LogPath] = batch.Cursor.Offset+ state.LastError = ""+ uploaded = true+ }+ state.UpdatedAt = time.Now().UTC().Format(time.RFC3339)+ _ = opts.Store.Save(state)+ return Result{State: state, Uploaded: uploaded}+}++func uploadSourceFromMetadata(metadata SourceMetadata) uploadSource {+ return uploadSource{+ SourceID: metadata.SourceID,+ Hostname: metadata.Hostname,+ EndpointMode: metadata.EndpointMode,+ LogPath: metadata.LogPath,+ ContentRetention: metadata.ContentRetention,+ ManagedMode: metadata.ManagedMode,+ }+}++func uploadID(sourceID, path string, offset int64) string {+ sum := sha256.Sum256([]byte(fmt.Sprintf("%s|%s|%d", sourceID, path, offset)))+ return hex.EncodeToString(sum[:])+}diff --git a/cli/beacon/internal/ingest/upload_test.go b/cli/beacon/internal/ingest/upload_test.gonew file mode 100644index 00000000..7cca85eb--- /dev/null+++ b/cli/beacon/internal/ingest/upload_test.go@@ -0,0 +1,147 @@+package ingest++import (+ "context"+ "encoding/json"+ "net/http"+ "net/http/httptest"+ "os"+ "path/filepath"+ "testing"++ beaconauth "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/auth"+)++type fakeSource struct {+ metadata SourceMetadata+ batches []Batch+ called bool+}++func (s *fakeSource) Metadata() SourceMetadata {+ return s.metadata+}++func (s *fakeSource) Batches(state State, maxEvents int, maxBytes int) ([]Batch, error) {+ s.called = true+ return s.batches, nil+}++func TestStatusIsReadOnly(t *testing.T) {+ statePath := filepath.Join(t.TempDir(), "upload-state.json")+ state := Status(Settings{Enabled: true, Managed: true, SourceID: "source-1"}, Store{Path: statePath}, &beaconauth.Credentials{+ Token: "token",+ UserID: "user",+ Email: "[email protected]",+ })++ if !state.Enabled || !state.Managed || !state.LoggedIn || state.SourceID != "source-1" {+ t.Fatalf("unexpected status: %#v", state)+ }+ if _, err := os.Stat(statePath); !os.IsNotExist(err) {+ t.Fatalf("Status wrote state file, err=%v", err)+ }+}++func TestUploadIsInertUnlessManagedEnabled(t *testing.T) {+ source := &fakeSource{}+ res := Upload(context.Background(), Options{+ Settings: Settings{Enabled: false, Managed: false},+ Store: Store{Path: filepath.Join(t.TempDir(), "upload-state.json")},+ Creds: &beaconauth.Credentials{Token: "token", UserID: "user"},+ Source: source,+ })++ if source.called {+ t.Fatal("disabled upload should not read source batches")+ }+ if res.Uploaded || res.State.AcceptedCount != 0 || res.State.LastUploadAt != "" {+ t.Fatalf("disabled upload should not send events: %#v", res.State)+ }+}++func TestUploadRequiresLogin(t *testing.T) {+ source := &fakeSource{}+ res := Upload(context.Background(), Options{+ Settings: Settings{Enabled: true, Managed: true, SourceID: "source-1"},+ Store: Store{Path: filepath.Join(t.TempDir(), "upload-state.json")},+ Source: source,+ })++ if source.called {+ t.Fatal("upload without login should not read source batches")+ }+ if res.State.LastError == "" {+ t.Fatalf("expected login error: %#v", res.State)+ }+}++func TestUploadPostsBatchAndPersistsCursor(t *testing.T) {+ var received uploadRequest+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ if got := r.Header.Get("Authorization"); got != "Bearer token" {+ t.Fatalf("Authorization = %q, want bearer token", got)+ }+ if err := json.NewDecoder(r.Body).Decode(&received); err != nil {+ t.Fatal(err)+ }+ _, _ = w.Write([]byte(`{"accepted":1,"rejected":0}`))+ }))+ defer server.Close()++ statePath := filepath.Join(t.TempDir(), "upload-state.json")+ source := &fakeSource{+ metadata: SourceMetadata{SourceID: "source-1", EndpointMode: "user", LogPath: "runtime.jsonl", ContentRetention: "redacted", ManagedMode: true},+ batches: []Batch{{+ Cursor: Cursor{LogPath: "runtime.jsonl", Offset: 128, Line: 1},+ Events: []json.RawMessage{json.RawMessage(`{"event":"ok"}`)},+ }},+ }+ res := Upload(context.Background(), Options{+ Settings: Settings{Enabled: true, Managed: true, SourceID: "source-1", IngestURL: server.URL},+ Store: Store{Path: statePath},+ Creds: &beaconauth.Credentials{Token: "token", UserID: "user"},+ Source: source,+ })++ if res.State.LastError != "" {+ t.Fatalf("LastError = %q", res.State.LastError)+ }+ if !res.Uploaded || res.State.AcceptedCount != 1 || res.State.FileOffsets["runtime.jsonl"] != 128 {+ t.Fatalf("unexpected upload state: %#v", res.State)+ }+ if received.Source.SourceID != "source-1" || !received.Source.ManagedMode || len(received.Events) != 1 {+ t.Fatalf("unexpected request: %#v", received)+ }+ if _, err := os.Stat(statePath); err != nil {+ t.Fatalf("state file not written: %v", err)+ }+}++func TestUploadDoesNotAdvanceCursorOnFailedPost(t *testing.T) {+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ http.Error(w, "nope", http.StatusInternalServerError)+ }))+ defer server.Close()++ source := &fakeSource{+ metadata: SourceMetadata{SourceID: "source-1", EndpointMode: "user", LogPath: "runtime.jsonl", ContentRetention: "redacted", ManagedMode: true},+ batches: []Batch{{+ Cursor: Cursor{LogPath: "runtime.jsonl", Offset: 128, Line: 1},+ Events: []json.RawMessage{json.RawMessage(`{"event":"ok"}`)},+ }},+ }+ res := Upload(context.Background(), Options{+ Settings: Settings{Enabled: true, Managed: true, SourceID: "source-1", IngestURL: server.URL},+ Store: Store{Path: filepath.Join(t.TempDir(), "upload-state.json")},+ Creds: &beaconauth.Credentials{Token: "token", UserID: "user"},+ Source: source,+ })++ if res.State.LastError == "" {+ t.Fatal("expected upload error")+ }+ if res.State.FileOffsets["runtime.jsonl"] != 0 || res.State.LastCursor.Offset != 0 {+ t.Fatalf("cursor advanced on failed post: %#v", res.State)+ }+}