The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu
Go agent-ai-cliagent-harnessesagent-securitybrowser-extensionciclaude-codecloudcodexcursordetection-engineeringjevknowledge-basememorymobile-device-managementobservabilitysecuritysecurity-information-and-event-managementsecurity-toolstelemetrytraces
fix: serialize cloud uploads and preserve unacknowledged bytes (#990)
Divyam Talwar42eccaa072d0
diff --git a/cli/beacon-hooks/internal/cloudshuttle/reset_transaction_test.go b/cli/beacon-hooks/internal/cloudshuttle/reset_transaction_test.gonew file mode 100644index 00000000..ff608a49--- /dev/null+++ b/cli/beacon-hooks/internal/cloudshuttle/reset_transaction_test.go@@ -0,0 +1,124 @@+package cloudshuttle++import (+ "context"+ "io"+ "net/http"+ "os"+ "sync"+ "sync/atomic"+ "testing"+ "time"+)++func TestPreserveSharesUploadTransaction(t *testing.T) {+ cfg := transactionConfig(t)+ transactionSeed(t, cfg)+ transactionWrite(t, cfg.LogPath, transactionFirst)+ first, second, release := make(chan struct{}), make(chan struct{}), make(chan struct{})+ var once sync.Once+ defer once.Do(func() { close(release) })+ var calls atomic.Int32+ transactionClient(t, func(r *http.Request) (*http.Response, error) {+ if _, err := transactionBody(r); err != nil {+ return nil, err+ }+ if calls.Add(1) == 1 {+ close(first)+ select {+ case <-release:+ case <-r.Context().Done():+ return nil, r.Context().Err()+ }+ } else {+ close(second)+ }+ return transactionOK(), nil+ })+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)+ defer cancel()+ uploaded := make(chan error, 1)+ go func() { uploaded <- Upload(ctx, cfg, true) }()+ select {+ case <-first:+ case <-ctx.Done():+ t.Fatal(ctx.Err())+ }+ preserved := make(chan error, 1)+ go func() { preserved <- preserveExistingLog(cfg) }()+ select {+ case <-second:+ t.Error("preservation contacted provider while upload transaction still held")+ case <-time.After(150 * time.Millisecond):+ }+ once.Do(func() { close(release) })+ if err := transactionAwait(t, uploaded); err != nil {+ t.Error(err)+ }+ if err := transactionAwait(t, preserved); err != nil {+ t.Error(err)+ }+ if _, err := os.Stat(cfg.LogPath); !os.IsNotExist(err) {+ t.Fatalf("unchanged preserved source: %v", err)+ }+}++func TestManagedConcurrentCursorDoesNotRewind(t *testing.T) {+ cfg := transactionConfig(t)+ cfg.Upload, cfg.IngestURL, cfg.DeviceKey = uploadAsymptote, "https://managed.invalid", "bcn_device_synthetic"+ transactionWrite(t, cfg.LogPath, transactionFirst)+ entered, newer, release := make(chan struct{}), make(chan struct{}), make(chan struct{})+ var once sync.Once+ defer once.Do(func() { close(release) })+ var calls atomic.Int32+ transactionClient(t, func(r *http.Request) (*http.Response, error) {+ if _, err := transactionBody(r); err != nil {+ return nil, err+ }+ if calls.Add(1) == 1 {+ close(entered)+ select {+ case <-release:+ case <-r.Context().Done():+ return nil, r.Context().Err()+ }+ } else {+ close(newer)+ }+ return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(http.NoBody)}, nil+ })+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)+ defer cancel()+ a := make(chan error, 1)+ go func() { a <- Upload(ctx, cfg, true) }()+ select {+ case <-entered:+ case <-ctx.Done():+ t.Fatal(ctx.Err())+ }+ transactionWrite(t, cfg.LogPath, transactionBoth)+ b := make(chan error, 1)+ go func() { b <- Upload(ctx, cfg, true) }()+ finishedB := false+ select {+ case <-newer:+ if err := transactionAwait(t, b); err != nil {+ t.Error(err)+ }+ finishedB = true+ case <-time.After(150 * time.Millisecond):+ }+ once.Do(func() { close(release) })+ if err := transactionAwait(t, a); err != nil {+ t.Error(err)+ }+ if !finishedB {+ if err := transactionAwait(t, b); err != nil {+ t.Error(err)+ }+ }+ st, err := readState(cfg.StatePath)+ if err != nil || st.LastSize != int64(len(transactionBoth)) {+ t.Fatalf("managed cursor rewound: %+v %v", st, err)+ }+}diff --git a/cli/beacon-hooks/internal/cloudshuttle/shuttle.go b/cli/beacon-hooks/internal/cloudshuttle/shuttle.goindex 58983848..ed3eea5a 100644--- a/cli/beacon-hooks/internal/cloudshuttle/shuttle.go+++ b/cli/beacon-hooks/internal/cloudshuttle/shuttle.go@@ -152,56 +152,56 @@ func ResetFromEnv() error {if strings.TrimSpace(os.Getenv("BEACON_ORIGIN")) != "cloud" {return nil}- if err := preserveExistingLog(cfg); err != nil {+ ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)+ defer cancel()+ release, err := lockUpload(ctx, cfg.StatePath)+ if err != nil {return err}- for _, path := range []string{cfg.LogPath + ".lock", cfg.StatePath} {- if path == "" {- continue- }- if err := os.Remove(path); err != nil && !os.IsNotExist(err) {- return err- }+ defer release()+ if err := preserveExistingLogLocked(ctx, cfg); err != nil {+ return err+ }+ // Lock files are stable protocol identities: unlinking one lets another process+ // lock a new inode while an existing holder still owns the old inode.+ if err := os.Remove(cfg.StatePath); err != nil && !os.IsNotExist(err) {+ return err}return nil}func Upload(ctx context.Context, cfg Config, force bool) error {cfg = normalizeConfig(cfg)- if !uploadConfigured(cfg) {+ if !uploadConfigured(cfg) || strings.TrimSpace(cfg.RunID) == "" || !force {return nil}- if strings.TrimSpace(cfg.RunID) == "" {- return nil+ release, err := lockUpload(ctx, cfg.StatePath)+ if err != nil {+ return err}- info, err := os.Stat(cfg.LogPath)+ defer release()++ snapshot, cleanup, err := snapshotLog(ctx, cfg.LogPath)if err != nil {if os.IsNotExist(err) {return nil}return err}- if info.Size() == 0 {- return nil- }- if !force {+ defer cleanup()+ if snapshot.info.Size() == 0 {return nil}- snapshot, cleanup, err := snapshotLog(cfg.LogPath)- if err != nil {- return err- }- defer cleanup()if cfg.Upload == uploadAsymptote {- return uploadAsymptoteIncremental(ctx, cfg, snapshot, info.Size())+ return uploadAsymptoteIncremental(ctx, cfg, snapshot.path, snapshot.info.Size())}objectName := uploadObjectName(cfg)- if err := uploadSnapshot(ctx, cfg, objectName, snapshot); err != nil {+ if err := uploadSnapshot(ctx, cfg, objectName, snapshot.path); err != nil {return err}return writeState(cfg.StatePath, state{LastUpload: time.Now().UTC().Format(time.RFC3339),- LastSize: info.Size(),+ LastSize: snapshot.info.Size(),LastObject: objectName,Provider: cfg.Provider,RunID: cfg.RunID,@@ -242,50 +242,53 @@ func ObjectName(cfg Config) string {return path.Join(parts...)}-func snapshotLog(logPath string) (string, func(), error) {+type logSnapshot struct {+ path string+ info os.FileInfo+}++func snapshotLog(ctx context.Context, logPath string) (logSnapshot, func(), error) {lock, err := os.OpenFile(logPath+".lock", os.O_CREATE|os.O_RDWR, 0644)if err != nil {- return "", func() {}, err+ return logSnapshot{}, func() {}, err}- held, err := filelock.Shared(lock)+ defer lock.Close()+ held, err := filelock.SharedContext(ctx, lock)if err != nil {- _ = lock.Close()- return "", func() {}, err+ return logSnapshot{}, func() {}, err}- // Registered close-first so LIFO runs it last: the lock is released, then the handle closed.- // The previous order relied on closing the descriptor to drop the lock implicitly, which left- // the explicit unlock failing on a closed handle with its error discarded.- defer func() {- if err := lock.Close(); err != nil {- fmt.Fprintf(os.Stderr, "cloudshuttle: failed to close lock file %s.lock: %v\n", logPath, err)- }- }()defer held.Release()source, err := os.Open(logPath)if err != nil {- return "", func() {}, err+ return logSnapshot{}, func() {}, err}defer source.Close()- if err := os.MkdirAll(filepath.Dir(logPath), 0755); err != nil {- return "", func() {}, err+ // Identity and length belong to the same opened source under the writer lock.+ // A separate pre-lock Stat can describe a different generation or older size.+ info, err := source.Stat()+ if err != nil {+ return logSnapshot{}, func() {}, err}temp, err := os.CreateTemp(filepath.Dir(logPath), "beacon-cloud-*.jsonl")if err != nil {- return "", func() {}, err+ return logSnapshot{}, func() {}, err}- tempPath := temp.Name()- cleanup := func() { _ = os.Remove(tempPath) }- if _, err := io.Copy(temp, source); err != nil {+ cleanup := func() { _ = os.Remove(temp.Name()) }+ if _, err := io.CopyN(temp, source, info.Size()); err != nil {_ = temp.Close()cleanup()- return "", func() {}, err+ return logSnapshot{}, func() {}, err}if err := temp.Close(); err != nil {cleanup()- return "", func() {}, err+ return logSnapshot{}, func() {}, err}- return tempPath, cleanup, nil+ if err := ctx.Err(); err != nil {+ cleanup()+ return logSnapshot{}, func() {}, err+ }+ return logSnapshot{path: temp.Name(), info: info}, cleanup, nil}func accessToken(ctx context.Context, credentialsB64 string) (string, error) {@@ -581,44 +584,46 @@ func writeState(path string, st state) error {func preserveExistingLog(cfg Config) error {cfg = normalizeConfig(cfg)- info, err := os.Stat(cfg.LogPath)+ ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)+ defer cancel()+ release, err := lockUpload(ctx, cfg.StatePath)+ if err != nil {+ return err+ }+ defer release()+ return preserveExistingLogLocked(ctx, cfg)+}++// The caller owns the upload transaction; this must never acquire that lock again.+func preserveExistingLogLocked(ctx context.Context, cfg Config) error {+ snapshot, cleanup, err := snapshotLog(ctx, cfg.LogPath)if err != nil {if os.IsNotExist(err) {return nil}return err}- if info.Size() == 0 {- return os.Remove(cfg.LogPath)- }+ defer cleanup()+ uploaded := snapshot.info.Size() == 0st, stateErr := readState(cfg.StatePath)hasState := stateErr == nil && st.LastObject != ""- // Bucket targets can only re-upload under the object name a previous run already used, so- // they need state. Managed ingest is addressed by the lines' own session ids, so a log from- // a run that never got a successful batch out (no state) is shipped whole rather than- // renamed aside, where an ephemeral sandbox would lose it.- if uploadConfigured(cfg) && (hasState || cfg.Upload == uploadAsymptote) {- snapshot, cleanup, err := snapshotLog(cfg.LogPath)- if err == nil {- defer cleanup()- ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)- defer cancel()- var uploadErr error- if cfg.Upload == uploadAsymptote {- // Ship whatever the previous run's last hook did not get to, under that run's- // identity so the stored offset applies and nothing is sent twice.- previous := cfg- if hasState {- previous.Provider, previous.RunID = st.Provider, st.RunID- }- uploadErr = uploadAsymptoteIncremental(ctx, previous, snapshot, info.Size())- } else {- uploadErr = uploadSnapshot(ctx, cfg, st.LastObject, snapshot)- }- if uploadErr == nil {- return os.Remove(cfg.LogPath)+ // Bucket targets retain their previous run object. Managed ingest can ship a+ // never-uploaded run because each line carries its own session identity.+ if !uploaded && uploadConfigured(cfg) && (hasState || cfg.Upload == uploadAsymptote) {+ var uploadErr error+ if cfg.Upload == uploadAsymptote {+ previous := cfg+ if hasState {+ previous.Provider, previous.RunID = st.Provider, st.RunID}+ uploadErr = uploadAsymptoteIncremental(ctx, previous, snapshot.path, snapshot.info.Size())+ } else {+ uploadErr = uploadSnapshot(ctx, cfg, st.LastObject, snapshot.path)}+ uploaded = uploadErr == nil+ }+ if uploaded {+ return os.Remove(cfg.LogPath)}preservedPath := fmt.Sprintf("%s.previous-%d", cfg.LogPath, time.Now().UTC().UnixNano())if err := os.Rename(cfg.LogPath, preservedPath); err != nil && !os.IsNotExist(err) {diff --git a/cli/beacon-hooks/internal/cloudshuttle/shuttle_test.go b/cli/beacon-hooks/internal/cloudshuttle/shuttle_test.goindex 6fc69c48..0d9183bb 100644--- a/cli/beacon-hooks/internal/cloudshuttle/shuttle_test.go+++ b/cli/beacon-hooks/internal/cloudshuttle/shuttle_test.go@@ -70,7 +70,7 @@ func useSandbox(t *testing.T, dir string) {t.Cleanup(func() { SandboxDir = prev })}-func TestResetFromEnvRemovesCloudRuntimeFiles(t *testing.T) {+func TestResetFromEnvPreservesLockFiles(t *testing.T) {dir := t.TempDir()useSandbox(t, dir)logPath := filepath.Join(dir, "runtime.jsonl")@@ -87,11 +87,14 @@ func TestResetFromEnvRemovesCloudRuntimeFiles(t *testing.T) {if err := ResetFromEnv(); err != nil {t.Fatalf("ResetFromEnv returned error: %v", err)}- for _, path := range []string{logPath, logPath + ".lock", statePath} {+ for _, path := range []string{logPath, statePath} {if _, err := os.Stat(path); !os.IsNotExist(err) {t.Fatalf("%s still exists, stat err=%v", path, err)}}+ if _, err := os.Stat(logPath + ".lock"); err != nil {+ t.Fatalf("runtime lock must retain its stable identity: %v", err)+ }}// A target the environment supplies cannot select a log outside the sandbox directory, so adiff --git a/cli/beacon-hooks/internal/cloudshuttle/transaction.go b/cli/beacon-hooks/internal/cloudshuttle/transaction.gonew file mode 100644index 00000000..c7bb7b9b--- /dev/null+++ b/cli/beacon-hooks/internal/cloudshuttle/transaction.go@@ -0,0 +1,31 @@+package cloudshuttle++import (+ "context"+ "os"+ "path/filepath"++ "github.com/asymptote-labs/agent-beacon/pkg/asymptoteobserve/filelock"+)++// lockUpload serializes state reads, snapshots, provider acknowledgment and checkpoint+// replacement across hook processes. The file remains in place even when state is reset.+// Lock order is transaction -> runtime log; no network work holds the runtime-log lock.+func lockUpload(ctx context.Context, statePath string) (func(), error) {+ if err := os.MkdirAll(filepath.Dir(statePath), 0755); err != nil {+ return nil, err+ }+ f, err := os.OpenFile(statePath+".lock", os.O_CREATE|os.O_RDWR, 0600)+ if err != nil {+ return nil, err+ }+ held, err := filelock.ExclusiveContext(ctx, f)+ if err != nil {+ _ = f.Close()+ return nil, err+ }+ return func() {+ _ = held.Release()+ _ = f.Close()+ }, nil+}diff --git a/cli/beacon-hooks/internal/cloudshuttle/transaction_edges_test.go b/cli/beacon-hooks/internal/cloudshuttle/transaction_edges_test.gonew file mode 100644index 00000000..59bddbd6--- /dev/null+++ b/cli/beacon-hooks/internal/cloudshuttle/transaction_edges_test.go@@ -0,0 +1,78 @@+package cloudshuttle++import (+ "context"+ "errors"+ "net/http"+ "os"+ "path/filepath"+ "sync/atomic"+ "testing"+ "time"++ "github.com/asymptote-labs/agent-beacon/pkg/asymptoteobserve/filelock"+)++func TestUploadCancellationWhileWaitingForWriterLock(t *testing.T) {+ cfg := transactionConfig(t)+ transactionWrite(t, cfg.LogPath, transactionFirst)+ f, err := os.OpenFile(cfg.LogPath+".lock", os.O_CREATE|os.O_RDWR, 0600)+ if err != nil {+ t.Fatal(err)+ }+ defer f.Close()+ held, err := filelock.Exclusive(f)+ if err != nil {+ t.Fatal(err)+ }+ defer held.Release()+ var calls atomic.Int32+ transactionClient(t, func(r *http.Request) (*http.Response, error) { calls.Add(1); return transactionOK(), nil })+ ctx, cancel := context.WithTimeout(context.Background(), 80*time.Millisecond)+ defer cancel()+ done := make(chan error, 1)+ go func() { done <- Upload(ctx, cfg, true) }()+ select {+ case err := <-done:+ if !errors.Is(err, context.DeadlineExceeded) {+ t.Errorf("writer contention = %v", err)+ }+ case <-time.After(time.Second):+ t.Error("writer contention ignored context deadline")+ if err := held.Release(); err != nil {+ t.Fatal(err)+ }+ transactionAwait(t, done)+ }+ if calls.Load() != 0 {+ t.Fatal("cancelled snapshot touched provider")+ }+}++func TestUploadTransactionLockErrorsFailClosed(t *testing.T) {+ cfg := transactionConfig(t)+ transactionWrite(t, cfg.LogPath, transactionFirst)+ if err := os.Mkdir(cfg.StatePath+".lock", 0700); err != nil {+ t.Fatal(err)+ }+ var calls atomic.Int32+ transactionClient(t, func(r *http.Request) (*http.Response, error) { calls.Add(1); return transactionOK(), nil })+ ctx, cancel := context.WithTimeout(context.Background(), time.Second)+ defer cancel()+ err := Upload(ctx, cfg, true)+ if err == nil || errors.Is(err, context.DeadlineExceeded) {+ t.Fatalf("real lock-open error ignored/retried: %v", err)+ }+ if calls.Load() != 0 {+ t.Fatal("lock-open error touched provider")+ }+ if err := os.Remove(cfg.StatePath + ".lock"); err != nil {+ t.Fatal(err)+ }+ if err := Upload(ctx, cfg, true); err != nil {+ t.Fatalf("retry after lock-open repair: %v", err)+ }+ if _, err := os.Stat(filepath.Join(filepath.Dir(cfg.StatePath), "state.json")); err != nil {+ t.Fatal(err)+ }+}diff --git a/cli/beacon-hooks/internal/cloudshuttle/upload_transaction_test.go b/cli/beacon-hooks/internal/cloudshuttle/upload_transaction_test.gonew file mode 100644index 00000000..fc293d59--- /dev/null+++ b/cli/beacon-hooks/internal/cloudshuttle/upload_transaction_test.go@@ -0,0 +1,326 @@+package cloudshuttle++import (+ "compress/gzip"+ "context"+ "encoding/json"+ "errors"+ "fmt"+ "io"+ "net/http"+ "net/http/httptest"+ "os"+ "os/exec"+ "path/filepath"+ "strings"+ "sync"+ "sync/atomic"+ "testing"+ "time"++ "github.com/asymptote-labs/agent-beacon/pkg/asymptoteobserve/filelock"+)++const transactionFirst = "{\"n\":1}\n"+const transactionBoth = transactionFirst + "{\"n\":2}\n"++type transactionTransport func(*http.Request) (*http.Response, error)++func (f transactionTransport) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) }++func transactionConfig(t *testing.T) Config {+ t.Helper()+ dir := t.TempDir()+ return Config{Upload: uploadS3, LogPath: filepath.Join(dir, "runtime.jsonl"),+ StatePath: filepath.Join(dir, "state.json"), Bucket: "synthetic-bucket", Provider: "claude_code_web",+ RunID: "same-run", S3Region: "us-east-1", S3AccessKeyID: "synthetic", S3SecretKey: "synthetic",+ S3Endpoint: "http://127.0.0.1:1"}+}++func transactionWrite(t *testing.T, path, contents string) {+ t.Helper()+ if err := os.WriteFile(path, []byte(contents), 0600); err != nil {+ t.Fatal(err)+ }+}++func transactionSeed(t *testing.T, cfg Config) {+ t.Helper()+ if err := writeState(cfg.StatePath, state{Provider: cfg.Provider, RunID: cfg.RunID,+ LastObject: "beacon/runtime/existing-run.jsonl.gz"}); err != nil {+ t.Fatal(err)+ }+}++func transactionBody(r *http.Request) (string, error) {+ gz, err := gzip.NewReader(r.Body)+ if err != nil {+ return "", err+ }+ defer gz.Close()+ b, err := io.ReadAll(gz)+ return string(b), err+}++func transactionOK() *http.Response {+ return &http.Response{StatusCode: 200, Body: io.NopCloser(strings.NewReader("{}")), Header: make(http.Header)}+}++func transactionClient(t *testing.T, transport transactionTransport) {+ t.Helper()+ old := httpClient+ httpClient = &http.Client{Transport: transport}+ t.Cleanup(func() { httpClient = old })+}++func transactionAwait(t *testing.T, ch <-chan error) error {+ t.Helper()+ select {+ case err := <-ch:+ return err+ case <-time.After(10 * time.Second):+ t.Fatal("upload did not finish")+ return nil+ }+}++// Both successful uploads must commit in snapshot order, even across hook processes.+// The seed fixes the remote key independently of the wall clock.+func TestUploadSerializesSnapshotsAndCheckpoint(t *testing.T) {+ for _, processes := range []bool{false, true} {+ t.Run(fmt.Sprintf("processes=%t", processes), func(t *testing.T) {+ cfg := transactionConfig(t)+ transactionSeed(t, cfg)+ transactionWrite(t, cfg.LogPath, transactionFirst)+ first, newer, release := make(chan struct{}), make(chan struct{}), make(chan struct{})+ var once sync.Once+ defer once.Do(func() { close(release) })+ var mu sync.Mutex+ var remote string+ var requests int+ handler := func(r *http.Request) (*http.Response, error) {+ body, err := transactionBody(r)+ if err != nil {+ return nil, err+ }+ mu.Lock()+ requests+++ n := requests+ mu.Unlock()+ if n == 1 {+ close(first)+ select {+ case <-release:+ case <-r.Context().Done():+ return nil, r.Context().Err()+ }+ } else {+ close(newer)+ }+ mu.Lock()+ remote = body+ mu.Unlock()+ return transactionOK(), nil+ }+ if processes {+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ resp, err := handler(r)+ if err != nil {+ http.Error(w, err.Error(), 500)+ return+ }+ w.WriteHeader(resp.StatusCode)+ }))+ defer server.Close()+ cfg.S3Endpoint = server.URL+ } else {+ transactionClient(t, handler)+ }+ start := func() <-chan error {+ done := make(chan error, 1)+ go func() {+ if processes {+ done <- transactionChild(t, cfg, "success")+ } else {+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)+ defer cancel()+ done <- Upload(ctx, cfg, true)+ }+ }()+ return done+ }+ a := start()+ select {+ case <-first:+ case <-time.After(5 * time.Second):+ t.Fatal("first upload never reached provider")+ }+ transactionWrite(t, cfg.LogPath, transactionBoth)+ b := start()+ // A broken transaction reaches/finishes the second PUT first; the correct+ // implementation waits until A releases its transaction ownership.+ select {+ case <-newer:+ if err := transactionAwait(t, b); err != nil {+ t.Error(err)+ }+ once.Do(func() { close(release) })+ case <-time.After(400 * time.Millisecond):+ once.Do(func() { close(release) })+ if err := transactionAwait(t, b); err != nil {+ t.Error(err)+ }+ }+ if err := transactionAwait(t, a); err != nil {+ t.Error(err)+ }+ mu.Lock()+ got := remote+ mu.Unlock()+ st, err := readState(cfg.StatePath)+ if err != nil {+ t.Fatal(err)+ }+ if got != transactionBoth || st.LastSize != int64(len(transactionBoth)) {+ t.Fatalf("acknowledged newer snapshot lost: remote=%q LastSize=%d; want %q/%d", got, st.LastSize, transactionBoth, len(transactionBoth))+ }+ if st.LastObject != "beacon/runtime/existing-run.jsonl.gz" {+ t.Fatalf("existing bucket key changed: %+v", st)+ }+ })+ }+}++func transactionChild(t *testing.T, cfg Config, expectation string) error {+ t.Helper()+ data, err := json.Marshal(cfg)+ if err != nil {+ return err+ }+ executable, err := os.Executable()+ if err != nil {+ return err+ }+ cmd := exec.Command(executable, "-test.run=^TestUploadTransactionChild$", "-test.timeout=10s")+ cmd.Env = append(os.Environ(), "BEACON_TRANSACTION_CHILD="+string(data), "BEACON_TRANSACTION_EXPECT="+expectation)+ output, err := cmd.CombinedOutput()+ if err != nil {+ return fmt.Errorf("child: %w\n%s", err, output)+ }+ return nil+}++func TestUploadTransactionChild(t *testing.T) {+ raw := os.Getenv("BEACON_TRANSACTION_CHILD")+ if raw == "" {+ return+ }+ var cfg Config+ if err := json.Unmarshal([]byte(raw), &cfg); err != nil {+ t.Fatal(err)+ }+ duration := 5 * time.Second+ if os.Getenv("BEACON_TRANSACTION_EXPECT") == "deadline" {+ duration = 150 * time.Millisecond+ }+ ctx, cancel := context.WithTimeout(context.Background(), duration)+ defer cancel()+ err := Upload(ctx, cfg, true)+ if os.Getenv("BEACON_TRANSACTION_EXPECT") == "deadline" {+ if !errors.Is(err, context.DeadlineExceeded) {+ t.Fatalf("contended upload = %v; want deadline", err)+ }+ } else if err != nil {+ t.Fatal(err)+ }+}++// Holding the protocol lock from another process is a causal exclusion proof:+// the child must exhaust its deadline without touching the fake provider.+func TestUploadTransactionCrossProcessContentionAndIndependentTargets(t *testing.T) {+ cfg := transactionConfig(t)+ transactionWrite(t, cfg.LogPath, transactionFirst)+ var calls atomic.Int32+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { calls.Add(1); w.WriteHeader(200) }))+ defer server.Close()+ cfg.S3Endpoint = server.URL+ f, err := os.OpenFile(cfg.StatePath+".lock", os.O_CREATE|os.O_RDWR, 0600)+ if err != nil {+ t.Fatal(err)+ }+ defer f.Close()+ held, err := filelock.Exclusive(f)+ if err != nil {+ t.Fatal(err)+ }+ defer held.Release()+ if err := transactionChild(t, cfg, "deadline"); err != nil {+ t.Error(err)+ }+ if got := calls.Load(); got != 0 {+ t.Errorf("contended child touched provider %d times", got)+ }+ other := cfg+ other.Bucket = "independent-bucket"+ other.StatePath = filepath.Join(filepath.Dir(cfg.StatePath), "independent.json")+ if err := transactionChild(t, other, "success"); err != nil {+ t.Fatal(err)+ }+ if err := held.Release(); err != nil {+ t.Fatal(err)+ }+ if err := transactionChild(t, cfg, "success"); err != nil {+ t.Fatal(err)+ }+}++func TestUploadReleasesTransactionAfterProviderAndCheckpointFailures(t *testing.T) {+ for _, fault := range []string{"rejection", "timeout", "checkpoint"} {+ t.Run(fault, func(t *testing.T) {+ cfg := transactionConfig(t)+ transactionWrite(t, cfg.LogPath, transactionFirst)+ if fault == "checkpoint" {+ if err := os.Mkdir(cfg.StatePath, 0700); err != nil {+ t.Fatal(err)+ }+ }+ fail := true+ transactionClient(t, func(r *http.Request) (*http.Response, error) {+ if fail && fault == "timeout" {+ <-r.Context().Done()+ return nil, r.Context().Err()+ }+ resp := transactionOK()+ if fail && fault == "rejection" {+ resp.StatusCode = 503+ }+ return resp, nil+ })+ ctx, cancel := context.WithTimeout(context.Background(), 80*time.Millisecond)+ err := Upload(ctx, cfg, true)+ cancel()+ if err == nil {+ t.Fatal("failed provider/checkpoint reported success")+ }+ if fault == "checkpoint" {+ if err := os.Remove(cfg.StatePath); err != nil {+ t.Fatal(err)+ }+ }+ fail = false+ ctx, cancel = context.WithTimeout(context.Background(), time.Second)+ defer cancel()+ if err := Upload(ctx, cfg, true); err != nil {+ t.Fatalf("retry blocked or failed after %s: %v", fault, err)+ }+ st, err := readState(cfg.StatePath)+ if err != nil || st.LastSize != int64(len(transactionFirst)) {+ t.Fatalf("retry checkpoint: %+v %v", st, err)+ }+ matches, _ := filepath.Glob(filepath.Join(filepath.Dir(cfg.StatePath), ".shuttle-state-*"))+ if len(matches) != 0 {+ t.Fatalf("checkpoint temporary files leaked: %v", matches)+ }+ })+ }+}diff --git a/pkg/asymptoteobserve/filelock/filelock.go b/pkg/asymptoteobserve/filelock/filelock.goindex 9b8f35a1..900c6c6c 100644--- a/pkg/asymptoteobserve/filelock/filelock.go+++ b/pkg/asymptoteobserve/filelock/filelock.go@@ -13,7 +13,11 @@// has to get right anyway.package filelock-import "os"+import (+ "context"+ "os"+ "time"+)// Lock is a held lock. Release unlocks; closing the file is the caller's business, since the// caller opened it.@@ -59,12 +63,52 @@ func TryExclusive(f *os.File) (*Lock, error) {if f == nil {return nil, os.ErrInvalid}- if err := tryLockFile(f); err != nil {+ if err := tryLockFile(f, true); err != nil {return nil, err}return &Lock{file: f}, nil}+// ExclusiveContext waits for an exclusive lock until ctx is cancelled. Only actual lock+// contention is retried; invalid handles and other operating-system errors are returned.+func ExclusiveContext(ctx context.Context, f *os.File) (*Lock, error) {+ return lockContext(ctx, f, true)+}++// SharedContext is Shared with a cancellable contention wait.+func SharedContext(ctx context.Context, f *os.File) (*Lock, error) {+ return lockContext(ctx, f, false)+}++func lockContext(ctx context.Context, f *os.File, exclusive bool) (*Lock, error) {+ if f == nil {+ return nil, os.ErrInvalid+ }+ for {+ if err := ctx.Err(); err != nil {+ return nil, err+ }+ err := tryLockFile(f, exclusive)+ if err == nil {+ if err := ctx.Err(); err != nil {+ _ = unlockFile(f)+ return nil, err+ }+ return &Lock{file: f}, nil+ }+ if !isContention(err) {+ return nil, err+ }+ timer := time.NewTimer(10 * time.Millisecond)+ select {+ case <-ctx.Done():+ timer.Stop()+ return nil, ctx.Err()+ case <-timer.C:+ }+ }+}+// Release unlocks. Safe to call more than once, so `defer l.Release()` alongside an explicit// release on the success path cannot double-unlock -- on Windows that would return an error for// a lock the process no longer holds, which reads as a failure where there is none.diff --git a/pkg/asymptoteobserve/filelock/filelock_context_test.go b/pkg/asymptoteobserve/filelock/filelock_context_test.gonew file mode 100644index 00000000..bcbbd6a0--- /dev/null+++ b/pkg/asymptoteobserve/filelock/filelock_context_test.go@@ -0,0 +1,105 @@+package filelock++import (+ "context"+ "errors"+ "os"+ "path/filepath"+ "testing"+ "time"+)++func TestContextLocksCancelContentionAndRelease(t *testing.T) {+ for _, shared := range []bool{false, true} {+ name := "exclusive"+ if shared {+ name = "shared"+ }+ t.Run(name, func(t *testing.T) {+ path := filepath.Join(t.TempDir(), "transaction.lock")+ first, err := Exclusive(openLockFile(t, path))+ if err != nil {+ t.Fatal(err)+ }+ defer first.Release()+ second := openLockFile(t, path)+ acquire := ExclusiveContext+ if shared {+ acquire = SharedContext+ }+ ctx, cancel := context.WithTimeout(context.Background(), 60*time.Millisecond)+ held, err := acquire(ctx, second)+ cancel()+ if !errors.Is(err, context.DeadlineExceeded) || held != nil {+ t.Fatalf("contended lock = %v, %v; want deadline without ownership", held, err)+ }+ if err := first.Release(); err != nil {+ t.Fatal(err)+ }+ ctx, cancel = context.WithTimeout(context.Background(), time.Second)+ defer cancel()+ held, err = acquire(ctx, second)+ if err != nil {+ t.Fatalf("lock unavailable after cancellation: %v", err)+ }+ if err := held.Release(); err != nil {+ t.Fatal(err)+ }+ })+ }+}++func TestContextLocksFailClosedOnRealErrors(t *testing.T) {+ for name, acquire := range map[string]func(context.Context, *os.File) (*Lock, error){+ "exclusive": ExclusiveContext, "shared": SharedContext,+ } {+ t.Run(name, func(t *testing.T) {+ if _, err := acquire(context.Background(), nil); !errors.Is(err, os.ErrInvalid) {+ t.Fatalf("nil file: %v", err)+ }+ f := openLockFile(t, filepath.Join(t.TempDir(), "closed.lock"))+ if err := f.Close(); err != nil {+ t.Fatal(err)+ }+ ctx, cancel := context.WithTimeout(context.Background(), time.Second)+ defer cancel()+ _, err := acquire(ctx, f)+ if err == nil || errors.Is(err, context.DeadlineExceeded) {+ t.Fatalf("real lock error retried/ignored: %v", err)+ }+ })+ }+}++func TestContextLocksAlreadyCancelledDoNotAcquire(t *testing.T) {+ ctx, cancel := context.WithCancel(context.Background())+ cancel()+ f := openLockFile(t, filepath.Join(t.TempDir(), "free.lock"))+ for _, acquire := range []func(context.Context, *os.File) (*Lock, error){ExclusiveContext, SharedContext} {+ held, err := acquire(ctx, f)+ if !errors.Is(err, context.Canceled) || held != nil {+ t.Fatalf("cancelled lock = %v, %v", held, err)+ }+ }+ other, err := TryExclusive(openLockFile(t, f.Name()))+ if err != nil {+ t.Fatalf("cancelled acquisition leaked ownership: %v", err)+ }+ defer other.Release()+}++func TestSharedContextLocksCoexist(t *testing.T) {+ path := filepath.Join(t.TempDir(), "shared.lock")+ first, err := SharedContext(context.Background(), openLockFile(t, path))+ if err != nil {+ t.Fatal(err)+ }+ defer first.Release()+ ctx, cancel := context.WithTimeout(context.Background(), time.Second)+ defer cancel()+ second, err := SharedContext(ctx, openLockFile(t, path))+ if err != nil {+ t.Fatal(err)+ }+ defer second.Release()+}diff --git a/pkg/asymptoteobserve/filelock/filelock_unix.go b/pkg/asymptoteobserve/filelock/filelock_unix.goindex 2449776b..a871d4f4 100644--- a/pkg/asymptoteobserve/filelock/filelock_unix.go+++ b/pkg/asymptoteobserve/filelock/filelock_unix.go@@ -3,6 +3,7 @@package filelockimport (+ "errors""os""syscall")@@ -16,10 +17,18 @@ func lockFile(f *os.File, exclusive bool) error {return syscall.Flock(int(f.Fd()), how)}-// tryLockFile takes an exclusive lock without waiting, reporting EWOULDBLOCK when another holder+// tryLockFile takes a lock without waiting, reporting EWOULDBLOCK when another holder// has it.-func tryLockFile(f *os.File) error {- return syscall.Flock(int(f.Fd()), syscall.LOCK_EX|syscall.LOCK_NB)+func tryLockFile(f *os.File, exclusive bool) error {+ how := syscall.LOCK_SH+ if exclusive {+ how = syscall.LOCK_EX+ }+ return syscall.Flock(int(f.Fd()), how|syscall.LOCK_NB)+}++func isContention(err error) bool {+ return errors.Is(err, syscall.EWOULDBLOCK) || errors.Is(err, syscall.EAGAIN)}func unlockFile(f *os.File) error {diff --git a/pkg/asymptoteobserve/filelock/filelock_windows.go b/pkg/asymptoteobserve/filelock/filelock_windows.goindex ea3f6121..0c534840 100644--- a/pkg/asymptoteobserve/filelock/filelock_windows.go+++ b/pkg/asymptoteobserve/filelock/filelock_windows.go@@ -3,6 +3,7 @@package filelockimport (+ "errors""os""golang.org/x/sys/windows"@@ -36,15 +37,23 @@ func lockFile(f *os.File, exclusive bool) error {0, lockRangeLow, lockRangeHigh, &overlapped)}-// tryLockFile takes an exclusive lock without waiting, reporting ERROR_LOCK_VIOLATION when another+// tryLockFile takes a lock without waiting, reporting ERROR_LOCK_VIOLATION when another// holder has it.-func tryLockFile(f *os.File) error {+func tryLockFile(f *os.File, exclusive bool) error {+ flags := uint32(windows.LOCKFILE_FAIL_IMMEDIATELY)+ if exclusive {+ flags |= windows.LOCKFILE_EXCLUSIVE_LOCK+ }var overlapped windows.Overlappedreturn windows.LockFileEx(windows.Handle(f.Fd()),- windows.LOCKFILE_EXCLUSIVE_LOCK|windows.LOCKFILE_FAIL_IMMEDIATELY,+ flags,0, lockRangeLow, lockRangeHigh, &overlapped)}+func isContention(err error) bool {+ return errors.Is(err, windows.ERROR_LOCK_VIOLATION)+}+func unlockFile(f *os.File) error {var overlapped windows.Overlappedreturn windows.UnlockFileEx(windows.Handle(f.Fd()),