The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu
Go agent-ai-cliagent-harnessesagent-securitybrowser-extensionciclaude-codecloudcodexcursordetection-engineeringjevknowledge-basememorymobile-device-managementobservabilitysecuritysecurity-information-and-event-managementsecurity-toolstelemetrytraces
package ci
import (
"context"
"crypto/rand"
"encoding/hex"
"errors"
"fmt"
"os"
"os/exec"
"path"
"path/filepath"
"strings"
"time"
"github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/schema"
"github.com/asymptote-labs/agent-beacon/pkg/asymptoteobserve/objectkey"
)
const (
UploadS3 = "s3"
UploadGCS = "gcs"
)
const (
EnvS3Bucket = "BEACON_CI_S3_BUCKET"
EnvS3Prefix = "BEACON_CI_S3_PREFIX"
EnvGCSBucket = "BEACON_CI_GCS_BUCKET"
EnvGCSPrefix = "BEACON_CI_GCS_PREFIX"
)
var uploadCredentialEnv = []string{
"AWS_ACCESS_KEY_ID",
"AWS_SECRET_ACCESS_KEY",
"AWS_SESSION_TOKEN",
"AWS_PROFILE",
"AWS_WEB_IDENTITY_TOKEN_FILE",
"GOOGLE_APPLICATION_CREDENTIALS",
"CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE",
}
var (
uploadLookPath = exec.LookPath
runUploadCommand = func(ctx context.Context, name string, args ...string) ([]byte, error) {
return exec.CommandContext(ctx, name, args...).CombinedOutput()
}
sessionClock = time.Now
sessionRandom = rand.Read
)
type UploadDestination struct {
Provider string `json:"provider"`
Bucket string `json:"bucket"`
Key string `json:"key"`
URI string `json:"uri"`
}
type UploadResult struct {
Provider string `json:"provider"`
Target string `json:"target"`
Status string `json:"status"`
Message string `json:"message,omitempty"`
}
// newUploadSession names the object-store folder of one CI session (Session.UploadSession): its
// start time in Unix nanoseconds and 8 random hex characters. It is chosen once by Provision and
// saved in the state file, so beacon ci finish uploads under the key beacon ci start chose.
// Matrix legs and repeated sessions in one job share every GitHub run field, and runners started
// together can read the same clock, so the random half keeps one upload from replacing another.
func newUploadSession(startedAt time.Time) (string, error) {
var suffix [4]byte
if _, err := sessionRandom(suffix[:]); err != nil {
return "", fmt.Errorf("ci upload session: %w", err)
}
return fmt.Sprintf("%d-%s", startedAt.UnixNano(), hex.EncodeToString(suffix[:])), nil
}
func resolveUploadDestinations(providers []string, run *schema.RunInfo, logPath, session string) ([]UploadDestination, error) {
normalized, err := normalizeUploads(providers)
if err != nil {
return nil, err
}
if len(normalized) == 0 {
return nil, nil
}
destinations := make([]UploadDestination, 0, len(normalized))
for _, provider := range normalized {
switch provider {
case UploadS3:
dest, err := uploadDestination(provider, os.Getenv(EnvS3Bucket), os.Getenv(EnvS3Prefix), run, logPath, session)
if err != nil {
return nil, err
}
destinations = append(destinations, dest)
case UploadGCS:
dest, err := uploadDestination(provider, os.Getenv(EnvGCSBucket), os.Getenv(EnvGCSPrefix), run, logPath, session)
if err != nil {
return nil, err
}
destinations = append(destinations, dest)
}
}
return destinations, nil
}
func ResolveUploadDestinationsForSession(providers []string, session *Session) ([]UploadDestination, error) {
if session == nil {
return nil, fmt.Errorf("ci session is nil")
}
return resolveUploadDestinations(providers, session.Run, session.LogPath, session.UploadSession)
}
func normalizeUploads(values []string) ([]string, error) {
seen := map[string]struct{}{}
var providers []string
for _, value := range values {
for _, part := range strings.Split(value, ",") {
provider := strings.ToLower(strings.TrimSpace(part))
if provider == "" {
continue
}
switch provider {
case UploadS3, UploadGCS:
if _, ok := seen[provider]; ok {
continue
}
seen[provider] = struct{}{}
providers = append(providers, provider)
default:
return nil, fmt.Errorf("unsupported --upload %q; supported values are %s and %s", part, UploadS3, UploadGCS)
}
}
}
return providers, nil
}
func uploadDestination(provider, bucket, prefix string, run *schema.RunInfo, logPath, session string) (UploadDestination, error) {
bucket = strings.TrimSpace(bucket)
if bucket == "" {
return UploadDestination{}, fmt.Errorf("--upload %s requires a bucket; set %s", provider, uploadBucketEnv(provider))
}
session = objectkey.CleanPart(session)
if session == "" {
return UploadDestination{}, errors.New("ci session state has no upload_session; run beacon ci start and beacon ci finish with the same Beacon version")
}
key := uploadObjectKey(prefix, run, logPath, session)
uri := fmt.Sprintf("%s://%s/%s", uploadScheme(provider), bucket, key)
return UploadDestination{Provider: provider, Bucket: bucket, Key: key, URI: uri}, nil
}
func uploadObjectKey(prefix string, run *schema.RunInfo, logPath, session string) string {
name := filepath.Base(logPath)
if strings.TrimSpace(name) == "" || name == "." || name == string(filepath.Separator) {
name = "runtime.jsonl"
}
parts := objectkey.CleanParts(prefix)
if run != nil && strings.TrimSpace(run.Repository) != "" && strings.TrimSpace(run.RunID) != "" {
parts = append(parts, objectkey.CleanParts(run.Repository)...)
parts = append(parts, objectkey.CleanPart(run.RunID), objectkey.CleanPart(run.RunAttempt), objectkey.CleanPart(run.Job))
} else {
parts = append(parts, "ci")
}
parts = append(parts, session, objectkey.CleanPart(name))
return path.Join(parts...)
}
func uploadBucketEnv(provider string) string {
if provider == UploadGCS {
return EnvGCSBucket
}
return EnvS3Bucket
}
func uploadScheme(provider string) string {
if provider == UploadGCS {
return "gs"
}
return "s3"
}
func (s *Session) UploadArtifacts(ctx context.Context) ([]UploadResult, error) {
if s == nil || len(s.Uploads) == 0 {
return nil, nil
}
results := make([]UploadResult, 0, len(s.Uploads))
for _, dest := range s.Uploads {
result, err := uploadArtifact(ctx, s.LogPath, dest)
results = append(results, result)
if err != nil {
return results, err
}
}
return results, nil
}
func uploadArtifact(ctx context.Context, logPath string, dest UploadDestination) (UploadResult, error) {
var name string
var args []string
switch dest.Provider {
case UploadS3:
if _, err := uploadLookPath("aws"); err != nil {
return uploadFailure(dest, "aws CLI not found in PATH"), fmt.Errorf("--upload s3 requires the aws CLI in PATH")
}
name = "aws"
args = []string{"s3", "cp", logPath, dest.URI}
case UploadGCS:
switch {
case commandAvailable("gcloud"):
name = "gcloud"
args = []string{"storage", "cp", logPath, dest.URI}
case commandAvailable("gsutil"):
name = "gsutil"
args = []string{"cp", logPath, dest.URI}
default:
return uploadFailure(dest, "gcloud or gsutil not found in PATH"), fmt.Errorf("--upload gcs requires gcloud or gsutil in PATH")
}
default:
return uploadFailure(dest, "unsupported provider"), fmt.Errorf("unsupported upload provider %q", dest.Provider)
}
output, err := runUploadCommand(ctx, name, args...)
if err != nil {
message := strings.TrimSpace(string(output))
if message == "" {
message = err.Error()
}
return uploadFailure(dest, message), fmt.Errorf("upload %s to %s failed: %w", dest.Provider, dest.URI, err)
}
return UploadResult{Provider: dest.Provider, Target: dest.URI, Status: "ok"}, nil
}
func uploadFailure(dest UploadDestination, message string) UploadResult {
return UploadResult{Provider: dest.Provider, Target: dest.URI, Status: "fail", Message: message}
}
func commandAvailable(name string) bool {
_, err := uploadLookPath(name)
return err == nil
}