Skip to content

The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu

fix

Justin D'Souza0ef8938cb265
diff --git a/cli/beacon-hooks/internal/cloudshuttle/shuttle.go b/cli/beacon-hooks/internal/cloudshuttle/shuttle.go
index 18abb22f..4bfcb837 100644
--- a/cli/beacon-hooks/internal/cloudshuttle/shuttle.go
+++ b/cli/beacon-hooks/internal/cloudshuttle/shuttle.go
@@ -2,6 +2,7 @@ package cloudshuttle
import (
"bytes"
+ "compress/gzip"
"context"
"crypto"
"crypto/hmac"
@@ -33,12 +34,14 @@ const (
defaultTokenURI = "https://oauth2.googleapis.com/token"
defaultGCSEndpoint = "https://storage.googleapis.com"
defaultS3Region = "us-east-1"
- contentTypeJSONL = "text/plain; charset=utf-8"
+ contentTypeJSONL = "application/x-ndjson"
+ contentEncoding = "gzip"
uploadGCS = "gcs"
uploadS3 = "s3"
)
var httpClient = http.DefaultClient
+var nowUTC = func() time.Time { return time.Now().UTC() }
type Config struct {
Upload string
@@ -167,16 +170,15 @@ func Upload(ctx context.Context, cfg Config, force bool) error {
}
func ObjectName(cfg Config) string {
+ now := nowUTC()
parts := cleanKeyParts(cfg.Prefix)
- parts = append(parts, "provider="+cleanKeyPart(defaultString(cfg.Provider, "unknown")))
- if cfg.UserID != "" {
- parts = append(parts, "user_id="+cleanKeyPart(cfg.UserID))
- }
- if cfg.Repository != "" {
- parts = append(parts, cleanKeyParts("repo="+cfg.Repository)...)
- }
- parts = append(parts, "run_id="+cleanKeyPart(defaultString(cfg.RunID, "unknown")))
- parts = append(parts, "runtime.jsonl")
+ parts = append(parts, "runtime", "date="+now.Format("2006-01-02"))
+ filename := strings.Join([]string{
+ fmt.Sprintf("%d", now.Unix()),
+ cleanKeyPart(defaultString(cfg.Provider, "unknown")),
+ cleanKeyPart(defaultString(cfg.RunID, "unknown")),
+ }, "-") + ".jsonl.gz"
+ parts = append(parts, filename)
return path.Join(parts...)
}
@@ -325,16 +327,52 @@ func parseRSAPrivateKey(raw string) (*rsa.PrivateKey, error) {
}
func uploadSnapshot(ctx context.Context, cfg Config, objectName, filePath string) error {
+ compressedPath, cleanup, err := gzipSnapshot(filePath)
+ if err != nil {
+ return err
+ }
+ defer cleanup()
switch cfg.Upload {
case uploadS3:
- return uploadS3Object(ctx, cfg, objectName, filePath)
+ return uploadS3Object(ctx, cfg, objectName, compressedPath)
default:
token, err := accessToken(ctx, cfg.CredentialsB64)
if err != nil {
return err
}
- return uploadGCSObject(ctx, cfg.GCSEndpoint, cfg.Bucket, objectName, filePath, token)
+ return uploadGCSObject(ctx, cfg.GCSEndpoint, cfg.Bucket, objectName, compressedPath, token)
+ }
+}
+
+func gzipSnapshot(filePath string) (string, func(), error) {
+ source, err := os.Open(filePath)
+ if err != nil {
+ return "", func() {}, err
+ }
+ defer source.Close()
+ temp, err := os.CreateTemp(filepath.Dir(filePath), "beacon-cloud-*.jsonl.gz")
+ if err != nil {
+ return "", func() {}, err
}
+ tempPath := temp.Name()
+ cleanup := func() { _ = os.Remove(tempPath) }
+ gz := gzip.NewWriter(temp)
+ if _, err := io.Copy(gz, source); err != nil {
+ _ = gz.Close()
+ _ = temp.Close()
+ cleanup()
+ return "", func() {}, err
+ }
+ if err := gz.Close(); err != nil {
+ _ = temp.Close()
+ cleanup()
+ return "", func() {}, err
+ }
+ if err := temp.Close(); err != nil {
+ cleanup()
+ return "", func() {}, err
+ }
+ return tempPath, cleanup, nil
}
func uploadGCSObject(ctx context.Context, endpoint, bucket, objectName, filePath, token string) error {
@@ -353,6 +391,7 @@ func uploadGCSObject(ctx context.Context, endpoint, bucket, objectName, filePath
}
req.Header.Set("Authorization", "Bearer "+token)
req.Header.Set("Content-Type", contentTypeJSONL)
+ req.Header.Set("Content-Encoding", contentEncoding)
resp, err := httpClient.Do(req)
if err != nil {
return err
@@ -391,6 +430,7 @@ func uploadS3Object(ctx context.Context, cfg Config, objectName, filePath string
req.ContentLength = info.Size()
}
req.Header.Set("Content-Type", contentTypeJSONL)
+ req.Header.Set("Content-Encoding", contentEncoding)
req.Header.Set("X-Amz-Content-Sha256", payloadHash)
req.Header.Set("X-Amz-Date", time.Now().UTC().Format("20060102T150405Z"))
if cfg.S3SessionToken != "" {
@@ -414,6 +454,7 @@ func signS3Request(req *http.Request, cfg Config, payloadHash string) {
dateStamp := amzDate[:8]
credentialScope := dateStamp + "/" + cfg.S3Region + "/s3/aws4_request"
headers := map[string]string{
+ "content-encoding": req.Header.Get("Content-Encoding"),
"content-type": req.Header.Get("Content-Type"),
"host": req.URL.Host,
"x-amz-content-sha256": payloadHash,
diff --git a/cli/beacon-hooks/internal/cloudshuttle/shuttle_test.go b/cli/beacon-hooks/internal/cloudshuttle/shuttle_test.go
index d7064994..ba061004 100644
--- a/cli/beacon-hooks/internal/cloudshuttle/shuttle_test.go
+++ b/cli/beacon-hooks/internal/cloudshuttle/shuttle_test.go
@@ -1,6 +1,8 @@
package cloudshuttle
import (
+ "bytes"
+ "compress/gzip"
"context"
"crypto/rand"
"crypto/rsa"
@@ -17,9 +19,12 @@ import (
"path/filepath"
"strings"
"testing"
+ "time"
)
-func TestObjectNamePartitionsByProviderUserRepoAndRun(t *testing.T) {
+func TestObjectNameUsesDatePartitionedGzipLayout(t *testing.T) {
+ restore := stubNow(t, time.Date(2026, 7, 12, 20, 17, 3, 0, time.UTC))
+ defer restore()
got := ObjectName(Config{
Prefix: "agent-traces/customer=test",
Provider: "claude_code_web",
@@ -27,20 +32,22 @@ func TestObjectNamePartitionsByProviderUserRepoAndRun(t *testing.T) {
Repository: "asymptote-labs/agent-beacon",
RunID: "cse_123",
})
- want := "agent-traces/customer=test/provider=claude_code_web/user_id=user-1/repo=asymptote-labs/agent-beacon/run_id=cse_123/runtime.jsonl"
+ want := "agent-traces/customer=test/runtime/date=2026-07-12/1783887423-claude_code_web-cse_123.jsonl.gz"
if got != want {
t.Fatalf("ObjectName = %q, want %q", got, want)
}
}
func TestObjectNameUsesCursorCloudProvider(t *testing.T) {
+ restore := stubNow(t, time.Date(2026, 7, 12, 20, 17, 3, 0, time.UTC))
+ defer restore()
got := ObjectName(Config{
Prefix: "agent-traces",
Provider: "cursor_cloud",
UserID: "user-1",
RunID: "manual-123",
})
- want := "agent-traces/provider=cursor_cloud/user_id=user-1/run_id=manual-123/runtime.jsonl"
+ want := "agent-traces/runtime/date=2026-07-12/1783887423-cursor_cloud-manual-123.jsonl.gz"
if got != want {
t.Fatalf("ObjectName = %q, want %q", got, want)
}
@@ -80,8 +87,11 @@ func TestResetFromEnvRemovesCloudRuntimeFiles(t *testing.T) {
}
func TestUploadSendsJSONLToGCS(t *testing.T) {
+ restore := stubNow(t, time.Date(2026, 7, 12, 20, 17, 3, 0, time.UTC))
+ defer restore()
key := mustRSAKey(t)
- var uploadedPath, uploadedAuth, uploadedType, uploadedBody string
+ var uploadedPath, uploadedAuth, uploadedType, uploadedEncoding string
+ var uploadedBody []byte
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case "/token":
@@ -96,8 +106,9 @@ func TestUploadSendsJSONLToGCS(t *testing.T) {
uploadedPath = r.URL.EscapedPath()
uploadedAuth = r.Header.Get("Authorization")
uploadedType = r.Header.Get("Content-Type")
+ uploadedEncoding = r.Header.Get("Content-Encoding")
data, _ := io.ReadAll(r.Body)
- uploadedBody = string(data)
+ uploadedBody = data
w.WriteHeader(http.StatusOK)
}
}))
@@ -136,11 +147,14 @@ func TestUploadSendsJSONLToGCS(t *testing.T) {
if uploadedType != contentTypeJSONL {
t.Fatalf("Content-Type = %q, want %q", uploadedType, contentTypeJSONL)
}
- if !strings.Contains(uploadedPath, "/bucket/prefix/provider=claude_code_web/user_id=user/run_id=run/runtime.jsonl") {
+ if uploadedEncoding != contentEncoding {
+ t.Fatalf("Content-Encoding = %q, want %q", uploadedEncoding, contentEncoding)
+ }
+ if !strings.Contains(uploadedPath, "/bucket/prefix/runtime/date=2026-07-12/1783887423-claude_code_web-run.jsonl.gz") {
t.Fatalf("upload path = %q", uploadedPath)
}
- if uploadedBody != "{\"event\":\"ok\"}\n" {
- t.Fatalf("uploaded body = %q", uploadedBody)
+ if got := gunzipString(t, uploadedBody); got != "{\"event\":\"ok\"}\n" {
+ t.Fatalf("uploaded body = %q", got)
}
}
@@ -183,7 +197,10 @@ func TestConfigFromEnvDefaultsToGCSWhenS3BucketIsAlsoSet(t *testing.T) {
}
func TestUploadSendsJSONLToS3(t *testing.T) {
- var uploadedPath, uploadedAuth, uploadedDate, uploadedHash, uploadedToken, uploadedType, uploadedBody string
+ restore := stubNow(t, time.Date(2026, 7, 12, 20, 17, 3, 0, time.UTC))
+ defer restore()
+ var uploadedPath, uploadedAuth, uploadedDate, uploadedHash, uploadedToken, uploadedType, uploadedEncoding string
+ var uploadedBody []byte
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
uploadedPath = r.URL.EscapedPath()
uploadedAuth = r.Header.Get("Authorization")
@@ -191,8 +208,9 @@ func TestUploadSendsJSONLToS3(t *testing.T) {
uploadedHash = r.Header.Get("X-Amz-Content-Sha256")
uploadedToken = r.Header.Get("X-Amz-Security-Token")
uploadedType = r.Header.Get("Content-Type")
+ uploadedEncoding = r.Header.Get("Content-Encoding")
data, _ := io.ReadAll(r.Body)
- uploadedBody = string(data)
+ uploadedBody = data
w.WriteHeader(http.StatusOK)
}))
defer server.Close()
@@ -220,13 +238,13 @@ func TestUploadSendsJSONLToS3(t *testing.T) {
if err := Upload(context.Background(), cfg, true); err != nil {
t.Fatalf("Upload returned error: %v", err)
}
- if !strings.Contains(uploadedPath, "/bucket/prefix/provider=claude_code_web/user_id=user/run_id=run/runtime.jsonl") {
+ if !strings.Contains(uploadedPath, "/bucket/prefix/runtime/date=2026-07-12/1783887423-claude_code_web-run.jsonl.gz") {
t.Fatalf("upload path = %q", uploadedPath)
}
if !strings.HasPrefix(uploadedAuth, "AWS4-HMAC-SHA256 Credential=AKIATEST/") {
t.Fatalf("Authorization = %q", uploadedAuth)
}
- for _, want := range []string{"us-west-2/s3/aws4_request", "SignedHeaders=content-type;host;x-amz-content-sha256;x-amz-date;x-amz-security-token"} {
+ for _, want := range []string{"us-west-2/s3/aws4_request", "SignedHeaders=content-encoding;content-type;host;x-amz-content-sha256;x-amz-date;x-amz-security-token"} {
if !strings.Contains(uploadedAuth, want) {
t.Fatalf("Authorization missing %q: %q", want, uploadedAuth)
}
@@ -234,7 +252,7 @@ func TestUploadSendsJSONLToS3(t *testing.T) {
if uploadedDate == "" {
t.Fatal("X-Amz-Date is empty")
}
- sum := sha256.Sum256([]byte(body))
+ sum := sha256.Sum256(uploadedBody)
if uploadedHash != hex.EncodeToString(sum[:]) {
t.Fatalf("X-Amz-Content-Sha256 = %q", uploadedHash)
}
@@ -244,8 +262,11 @@ func TestUploadSendsJSONLToS3(t *testing.T) {
if uploadedType != contentTypeJSONL {
t.Fatalf("Content-Type = %q, want %q", uploadedType, contentTypeJSONL)
}
- if uploadedBody != body {
- t.Fatalf("uploaded body = %q", uploadedBody)
+ if uploadedEncoding != contentEncoding {
+ t.Fatalf("Content-Encoding = %q, want %q", uploadedEncoding, contentEncoding)
+ }
+ if got := gunzipString(t, uploadedBody); got != body {
+ t.Fatalf("uploaded body = %q", got)
}
}
@@ -280,6 +301,27 @@ func mustRSAKey(t *testing.T) *rsa.PrivateKey {
return key
}
+func stubNow(t *testing.T, value time.Time) func() {
+ t.Helper()
+ old := nowUTC
+ nowUTC = func() time.Time { return value.UTC() }
+ return func() { nowUTC = old }
+}
+
+func gunzipString(t *testing.T, data []byte) string {
+ t.Helper()
+ reader, err := gzip.NewReader(bytes.NewReader(data))
+ if err != nil {
+ t.Fatalf("gzip reader: %v", err)
+ }
+ defer reader.Close()
+ out, err := io.ReadAll(reader)
+ if err != nil {
+ t.Fatalf("gzip read: %v", err)
+ }
+ return string(out)
+}
+
func pemKey(t *testing.T, key *rsa.PrivateKey) string {
t.Helper()
data, err := x509.MarshalPKCS8PrivateKey(key)
diff --git a/cli/beacon/README.md b/cli/beacon/README.md
index 7fca825c..87234ce9 100644
--- a/cli/beacon/README.md
+++ b/cli/beacon/README.md
@@ -39,7 +39,7 @@ configured.
Use `beacon cloud` helpers to configure provider-managed cloud agent sandboxes.
Claude Code on the web and Cursor cloud agents can forward Beacon JSONL to
-customer-managed GCS:
+customer-managed GCS or S3:
```bash
./beacon cloud gcs setup \
@@ -53,7 +53,12 @@ customer-managed GCS:
Copy the printed `BEACON_CLOUD_GCS_BUCKET`,
`BEACON_CLOUD_GCS_PREFIX`, and `BEACON_CLOUD_GCS_CREDENTIALS_B64`
-values into the cloud agent environment. Also set:
+values into the cloud agent environment.
+
+For S3, use `beacon cloud s3 setup --apply --print-env` and copy the printed
+`BEACON_CLOUD_S3_*` and AWS credential values instead.
+
+For both destinations, also set:
```bash
BEACON_ORIGIN=cloud
@@ -84,14 +89,14 @@ paste it into the provider's cloud setup field:
```
The setup script installs `beacon-hooks` into the cloud sandbox and uploads a
-browser-viewable `/tmp/beacon/runtime.jsonl` snapshot to GCS. Claude web writes
+gzip-compressed `/tmp/beacon/runtime.jsonl` snapshot to object storage. Claude web writes
`.claude/settings.local.json` inside the sandbox clone. Cursor cloud uses
`.cursor/environment.json` to install Beacon and generate project-level
`.cursor/hooks.json` because Cursor cloud only runs repository project hooks.
```text
-<prefix>/provider=claude_code_web/user_id=<id>/run_id=<claude-session-id>/runtime.jsonl
-<prefix>/provider=cursor_cloud/user_id=<id>/run_id=<generated-or-explicit-run-id>/runtime.jsonl
+<prefix>/runtime/date=YYYY-MM-DD/<timestamp>-claude_code_web-<claude-session-id>.jsonl.gz
+<prefix>/runtime/date=YYYY-MM-DD/<timestamp>-cursor_cloud-<generated-or-explicit-run-id>.jsonl.gz
```
Cloud network access must allow `oauth2.googleapis.com` and
diff --git a/docs/runtimes/claude-code-cloud-agents.mdx b/docs/runtimes/claude-code-cloud-agents.mdx
index 58955b53..44efeb0f 100644
--- a/docs/runtimes/claude-code-cloud-agents.mdx
+++ b/docs/runtimes/claude-code-cloud-agents.mdx
@@ -53,13 +53,13 @@ The setup has three parts:
Each Claude Code cloud agent session writes one readable JSONL object:
```text
-gs://<bucket>/<prefix>/provider=claude_code_web/user_id=<user_id>/run_id=<claude_session_id>/runtime.jsonl
+gs://<bucket>/<prefix>/runtime/date=YYYY-MM-DD/<timestamp>-claude_code_web-<claude_session_id>.jsonl.gz
```
or:
```text
-s3://<bucket>/<prefix>/provider=claude_code_web/user_id=<user_id>/run_id=<claude_session_id>/runtime.jsonl
+s3://<bucket>/<prefix>/runtime/date=YYYY-MM-DD/<timestamp>-claude_code_web-<claude_session_id>.jsonl.gz
```
## Prerequisites
@@ -295,7 +295,7 @@ aws s3 ls --recursive "s3://${BEACON_TEST_BUCKET}/${BEACON_CLOUD_S3_PREFIX}/"
You should see a path like:
```text
-provider=claude_code_web/user_id=<user_id>/run_id=cse_.../runtime.jsonl
+runtime/date=YYYY-MM-DD/<timestamp>-claude_code_web-cse_....jsonl.gz
```
<Frame caption="Beacon uploads one readable runtime.jsonl object per Claude Code cloud agent session.">
@@ -305,13 +305,13 @@ provider=claude_code_web/user_id=<user_id>/run_id=cse_.../runtime.jsonl
Inspect the log:
```bash
-gcloud storage cat "gs://${BEACON_TEST_BUCKET}/${BEACON_CLOUD_GCS_PREFIX}/provider=claude_code_web/user_id=<user_id>/run_id=<run_id>/runtime.jsonl" | head
+gcloud storage cat "gs://${BEACON_TEST_BUCKET}/${BEACON_CLOUD_GCS_PREFIX}/runtime/date=<date>/<object>.jsonl.gz" | gzip -dc | head
```
For S3:
```bash
-aws s3 cp "s3://${BEACON_TEST_BUCKET}/${BEACON_CLOUD_S3_PREFIX}/provider=claude_code_web/user_id=<user_id>/run_id=<run_id>/runtime.jsonl" - | head
+aws s3 cp "s3://${BEACON_TEST_BUCKET}/${BEACON_CLOUD_S3_PREFIX}/runtime/date=<date>/<object>.jsonl.gz" - | gzip -dc | head
```
Expected fields include:
diff --git a/docs/runtimes/cursor-cloud-agents.mdx b/docs/runtimes/cursor-cloud-agents.mdx
index 47472c1d..6cf88d19 100644
--- a/docs/runtimes/cursor-cloud-agents.mdx
+++ b/docs/runtimes/cursor-cloud-agents.mdx
@@ -47,7 +47,7 @@ The setup has four parts:
Each Cursor cloud agent conversation writes one readable JSONL object:
```text
-gs://<bucket>/<prefix>/provider=cursor_cloud/user_id=<user_id>/run_id=<cursor_conversation_id>/runtime.jsonl
+gs://<bucket>/<prefix>/runtime/date=YYYY-MM-DD/<timestamp>-cursor_cloud-<cursor_conversation_id>.jsonl.gz
```
Beacon uses Cursor's `conversation_id` hook field as the default `run_id` for
@@ -242,7 +242,7 @@ gcloud storage ls --recursive "gs://${BEACON_TEST_BUCKET}/${BEACON_CLOUD_GCS_PRE
You should see a path like:
```text
-provider=cursor_cloud/user_id=<user_id>/run_id=<run_id>/runtime.jsonl
+runtime/date=YYYY-MM-DD/<timestamp>-cursor_cloud-<run_id>.jsonl.gz
```
<Frame caption="Beacon uploads one readable runtime.jsonl object per Cursor cloud agent session.">
@@ -252,7 +252,7 @@ provider=cursor_cloud/user_id=<user_id>/run_id=<run_id>/runtime.jsonl
Inspect the log:
```bash
-gcloud storage cat "gs://${BEACON_TEST_BUCKET}/${BEACON_CLOUD_GCS_PREFIX}/provider=cursor_cloud/user_id=<user_id>/run_id=<run_id>/runtime.jsonl" | head
+gcloud storage cat "gs://${BEACON_TEST_BUCKET}/${BEACON_CLOUD_GCS_PREFIX}/runtime/date=<date>/<object>.jsonl.gz" | gzip -dc | head
```
Expected fields include: