Skip to content

The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu

fix: serialize cloud uploads and preserve unacknowledged bytes (#990)

Divyam Talwar42eccaa072d0
diff --git a/cli/beacon-hooks/internal/cloudshuttle/reset_transaction_test.go b/cli/beacon-hooks/internal/cloudshuttle/reset_transaction_test.go
new file mode 100644
index 00000000..ff608a49
--- /dev/null
+++ b/cli/beacon-hooks/internal/cloudshuttle/reset_transaction_test.go
@@ -0,0 +1,124 @@
+package cloudshuttle
+
+import (
+ "context"
+ "io"
+ "net/http"
+ "os"
+ "sync"
+ "sync/atomic"
+ "testing"
+ "time"
+)
+
+func TestPreserveSharesUploadTransaction(t *testing.T) {
+ cfg := transactionConfig(t)
+ transactionSeed(t, cfg)
+ transactionWrite(t, cfg.LogPath, transactionFirst)
+ first, second, release := make(chan struct{}), make(chan struct{}), make(chan struct{})
+ var once sync.Once
+ defer once.Do(func() { close(release) })
+ var calls atomic.Int32
+ transactionClient(t, func(r *http.Request) (*http.Response, error) {
+ if _, err := transactionBody(r); err != nil {
+ return nil, err
+ }
+ if calls.Add(1) == 1 {
+ close(first)
+ select {
+ case <-release:
+ case <-r.Context().Done():
+ return nil, r.Context().Err()
+ }
+ } else {
+ close(second)
+ }
+ return transactionOK(), nil
+ })
+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+ defer cancel()
+ uploaded := make(chan error, 1)
+ go func() { uploaded <- Upload(ctx, cfg, true) }()
+ select {
+ case <-first:
+ case <-ctx.Done():
+ t.Fatal(ctx.Err())
+ }
+ preserved := make(chan error, 1)
+ go func() { preserved <- preserveExistingLog(cfg) }()
+ select {
+ case <-second:
+ t.Error("preservation contacted provider while upload transaction still held")
+ case <-time.After(150 * time.Millisecond):
+ }
+ once.Do(func() { close(release) })
+ if err := transactionAwait(t, uploaded); err != nil {
+ t.Error(err)
+ }
+ if err := transactionAwait(t, preserved); err != nil {
+ t.Error(err)
+ }
+ if _, err := os.Stat(cfg.LogPath); !os.IsNotExist(err) {
+ t.Fatalf("unchanged preserved source: %v", err)
+ }
+}
+
+func TestManagedConcurrentCursorDoesNotRewind(t *testing.T) {
+ cfg := transactionConfig(t)
+ cfg.Upload, cfg.IngestURL, cfg.DeviceKey = uploadAsymptote, "https://managed.invalid", "bcn_device_synthetic"
+ transactionWrite(t, cfg.LogPath, transactionFirst)
+ entered, newer, release := make(chan struct{}), make(chan struct{}), make(chan struct{})
+ var once sync.Once
+ defer once.Do(func() { close(release) })
+ var calls atomic.Int32
+ transactionClient(t, func(r *http.Request) (*http.Response, error) {
+ if _, err := transactionBody(r); err != nil {
+ return nil, err
+ }
+ if calls.Add(1) == 1 {
+ close(entered)
+ select {
+ case <-release:
+ case <-r.Context().Done():
+ return nil, r.Context().Err()
+ }
+ } else {
+ close(newer)
+ }
+ return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(http.NoBody)}, nil
+ })
+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+ defer cancel()
+ a := make(chan error, 1)
+ go func() { a <- Upload(ctx, cfg, true) }()
+ select {
+ case <-entered:
+ case <-ctx.Done():
+ t.Fatal(ctx.Err())
+ }
+ transactionWrite(t, cfg.LogPath, transactionBoth)
+ b := make(chan error, 1)
+ go func() { b <- Upload(ctx, cfg, true) }()
+ finishedB := false
+ select {
+ case <-newer:
+ if err := transactionAwait(t, b); err != nil {
+ t.Error(err)
+ }
+ finishedB = true
+ case <-time.After(150 * time.Millisecond):
+ }
+ once.Do(func() { close(release) })
+ if err := transactionAwait(t, a); err != nil {
+ t.Error(err)
+ }
+ if !finishedB {
+ if err := transactionAwait(t, b); err != nil {
+ t.Error(err)
+ }
+ }
+ st, err := readState(cfg.StatePath)
+ if err != nil || st.LastSize != int64(len(transactionBoth)) {
+ t.Fatalf("managed cursor rewound: %+v %v", st, err)
+ }
+}
diff --git a/cli/beacon-hooks/internal/cloudshuttle/shuttle.go b/cli/beacon-hooks/internal/cloudshuttle/shuttle.go
index 58983848..ed3eea5a 100644
--- a/cli/beacon-hooks/internal/cloudshuttle/shuttle.go
+++ b/cli/beacon-hooks/internal/cloudshuttle/shuttle.go
@@ -152,56 +152,56 @@ func ResetFromEnv() error {
if strings.TrimSpace(os.Getenv("BEACON_ORIGIN")) != "cloud" {
return nil
}
- if err := preserveExistingLog(cfg); err != nil {
+ ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+ defer cancel()
+ release, err := lockUpload(ctx, cfg.StatePath)
+ if err != nil {
return err
}
- for _, path := range []string{cfg.LogPath + ".lock", cfg.StatePath} {
- if path == "" {
- continue
- }
- if err := os.Remove(path); err != nil && !os.IsNotExist(err) {
- return err
- }
+ defer release()
+ if err := preserveExistingLogLocked(ctx, cfg); err != nil {
+ return err
+ }
+ // Lock files are stable protocol identities: unlinking one lets another process
+ // lock a new inode while an existing holder still owns the old inode.
+ if err := os.Remove(cfg.StatePath); err != nil && !os.IsNotExist(err) {
+ return err
}
return nil
}
func Upload(ctx context.Context, cfg Config, force bool) error {
cfg = normalizeConfig(cfg)
- if !uploadConfigured(cfg) {
+ if !uploadConfigured(cfg) || strings.TrimSpace(cfg.RunID) == "" || !force {
return nil
}
- if strings.TrimSpace(cfg.RunID) == "" {
- return nil
+ release, err := lockUpload(ctx, cfg.StatePath)
+ if err != nil {
+ return err
}
- info, err := os.Stat(cfg.LogPath)
+ defer release()
+
+ snapshot, cleanup, err := snapshotLog(ctx, cfg.LogPath)
if err != nil {
if os.IsNotExist(err) {
return nil
}
return err
}
- if info.Size() == 0 {
- return nil
- }
- if !force {
+ defer cleanup()
+ if snapshot.info.Size() == 0 {
return nil
}
- snapshot, cleanup, err := snapshotLog(cfg.LogPath)
- if err != nil {
- return err
- }
- defer cleanup()
if cfg.Upload == uploadAsymptote {
- return uploadAsymptoteIncremental(ctx, cfg, snapshot, info.Size())
+ return uploadAsymptoteIncremental(ctx, cfg, snapshot.path, snapshot.info.Size())
}
objectName := uploadObjectName(cfg)
- if err := uploadSnapshot(ctx, cfg, objectName, snapshot); err != nil {
+ if err := uploadSnapshot(ctx, cfg, objectName, snapshot.path); err != nil {
return err
}
return writeState(cfg.StatePath, state{
LastUpload: time.Now().UTC().Format(time.RFC3339),
- LastSize: info.Size(),
+ LastSize: snapshot.info.Size(),
LastObject: objectName,
Provider: cfg.Provider,
RunID: cfg.RunID,
@@ -242,50 +242,53 @@ func ObjectName(cfg Config) string {
return path.Join(parts...)
}
-func snapshotLog(logPath string) (string, func(), error) {
+type logSnapshot struct {
+ path string
+ info os.FileInfo
+}
+
+func snapshotLog(ctx context.Context, logPath string) (logSnapshot, func(), error) {
lock, err := os.OpenFile(logPath+".lock", os.O_CREATE|os.O_RDWR, 0644)
if err != nil {
- return "", func() {}, err
+ return logSnapshot{}, func() {}, err
}
- held, err := filelock.Shared(lock)
+ defer lock.Close()
+ held, err := filelock.SharedContext(ctx, lock)
if err != nil {
- _ = lock.Close()
- return "", func() {}, err
+ return logSnapshot{}, func() {}, err
}
- // Registered close-first so LIFO runs it last: the lock is released, then the handle closed.
- // The previous order relied on closing the descriptor to drop the lock implicitly, which left
- // the explicit unlock failing on a closed handle with its error discarded.
- defer func() {
- if err := lock.Close(); err != nil {
- fmt.Fprintf(os.Stderr, "cloudshuttle: failed to close lock file %s.lock: %v\n", logPath, err)
- }
- }()
defer held.Release()
source, err := os.Open(logPath)
if err != nil {
- return "", func() {}, err
+ return logSnapshot{}, func() {}, err
}
defer source.Close()
- if err := os.MkdirAll(filepath.Dir(logPath), 0755); err != nil {
- return "", func() {}, err
+ // Identity and length belong to the same opened source under the writer lock.
+ // A separate pre-lock Stat can describe a different generation or older size.
+ info, err := source.Stat()
+ if err != nil {
+ return logSnapshot{}, func() {}, err
}
temp, err := os.CreateTemp(filepath.Dir(logPath), "beacon-cloud-*.jsonl")
if err != nil {
- return "", func() {}, err
+ return logSnapshot{}, func() {}, err
}
- tempPath := temp.Name()
- cleanup := func() { _ = os.Remove(tempPath) }
- if _, err := io.Copy(temp, source); err != nil {
+ cleanup := func() { _ = os.Remove(temp.Name()) }
+ if _, err := io.CopyN(temp, source, info.Size()); err != nil {
_ = temp.Close()
cleanup()
- return "", func() {}, err
+ return logSnapshot{}, func() {}, err
}
if err := temp.Close(); err != nil {
cleanup()
- return "", func() {}, err
+ return logSnapshot{}, func() {}, err
}
- return tempPath, cleanup, nil
+ if err := ctx.Err(); err != nil {
+ cleanup()
+ return logSnapshot{}, func() {}, err
+ }
+ return logSnapshot{path: temp.Name(), info: info}, cleanup, nil
}
func accessToken(ctx context.Context, credentialsB64 string) (string, error) {
@@ -581,44 +584,46 @@ func writeState(path string, st state) error {
func preserveExistingLog(cfg Config) error {
cfg = normalizeConfig(cfg)
- info, err := os.Stat(cfg.LogPath)
+ ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+ defer cancel()
+ release, err := lockUpload(ctx, cfg.StatePath)
+ if err != nil {
+ return err
+ }
+ defer release()
+ return preserveExistingLogLocked(ctx, cfg)
+}
+
+// The caller owns the upload transaction; this must never acquire that lock again.
+func preserveExistingLogLocked(ctx context.Context, cfg Config) error {
+ snapshot, cleanup, err := snapshotLog(ctx, cfg.LogPath)
if err != nil {
if os.IsNotExist(err) {
return nil
}
return err
}
- if info.Size() == 0 {
- return os.Remove(cfg.LogPath)
- }
+ defer cleanup()
+ uploaded := snapshot.info.Size() == 0
st, stateErr := readState(cfg.StatePath)
hasState := stateErr == nil && st.LastObject != ""
- // Bucket targets can only re-upload under the object name a previous run already used, so
- // they need state. Managed ingest is addressed by the lines' own session ids, so a log from
- // a run that never got a successful batch out (no state) is shipped whole rather than
- // renamed aside, where an ephemeral sandbox would lose it.
- if uploadConfigured(cfg) && (hasState || cfg.Upload == uploadAsymptote) {
- snapshot, cleanup, err := snapshotLog(cfg.LogPath)
- if err == nil {
- defer cleanup()
- ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
- defer cancel()
- var uploadErr error
- if cfg.Upload == uploadAsymptote {
- // Ship whatever the previous run's last hook did not get to, under that run's
- // identity so the stored offset applies and nothing is sent twice.
- previous := cfg
- if hasState {
- previous.Provider, previous.RunID = st.Provider, st.RunID
- }
- uploadErr = uploadAsymptoteIncremental(ctx, previous, snapshot, info.Size())
- } else {
- uploadErr = uploadSnapshot(ctx, cfg, st.LastObject, snapshot)
- }
- if uploadErr == nil {
- return os.Remove(cfg.LogPath)
+ // Bucket targets retain their previous run object. Managed ingest can ship a
+ // never-uploaded run because each line carries its own session identity.
+ if !uploaded && uploadConfigured(cfg) && (hasState || cfg.Upload == uploadAsymptote) {
+ var uploadErr error
+ if cfg.Upload == uploadAsymptote {
+ previous := cfg
+ if hasState {
+ previous.Provider, previous.RunID = st.Provider, st.RunID
}
+ uploadErr = uploadAsymptoteIncremental(ctx, previous, snapshot.path, snapshot.info.Size())
+ } else {
+ uploadErr = uploadSnapshot(ctx, cfg, st.LastObject, snapshot.path)
}
+ uploaded = uploadErr == nil
+ }
+ if uploaded {
+ return os.Remove(cfg.LogPath)
}
preservedPath := fmt.Sprintf("%s.previous-%d", cfg.LogPath, time.Now().UTC().UnixNano())
if err := os.Rename(cfg.LogPath, preservedPath); err != nil && !os.IsNotExist(err) {
diff --git a/cli/beacon-hooks/internal/cloudshuttle/shuttle_test.go b/cli/beacon-hooks/internal/cloudshuttle/shuttle_test.go
index 6fc69c48..0d9183bb 100644
--- a/cli/beacon-hooks/internal/cloudshuttle/shuttle_test.go
+++ b/cli/beacon-hooks/internal/cloudshuttle/shuttle_test.go
@@ -70,7 +70,7 @@ func useSandbox(t *testing.T, dir string) {
t.Cleanup(func() { SandboxDir = prev })
}
-func TestResetFromEnvRemovesCloudRuntimeFiles(t *testing.T) {
+func TestResetFromEnvPreservesLockFiles(t *testing.T) {
dir := t.TempDir()
useSandbox(t, dir)
logPath := filepath.Join(dir, "runtime.jsonl")
@@ -87,11 +87,14 @@ func TestResetFromEnvRemovesCloudRuntimeFiles(t *testing.T) {
if err := ResetFromEnv(); err != nil {
t.Fatalf("ResetFromEnv returned error: %v", err)
}
- for _, path := range []string{logPath, logPath + ".lock", statePath} {
+ for _, path := range []string{logPath, statePath} {
if _, err := os.Stat(path); !os.IsNotExist(err) {
t.Fatalf("%s still exists, stat err=%v", path, err)
}
}
+ if _, err := os.Stat(logPath + ".lock"); err != nil {
+ t.Fatalf("runtime lock must retain its stable identity: %v", err)
+ }
}
// A target the environment supplies cannot select a log outside the sandbox directory, so a
diff --git a/cli/beacon-hooks/internal/cloudshuttle/transaction.go b/cli/beacon-hooks/internal/cloudshuttle/transaction.go
new file mode 100644
index 00000000..c7bb7b9b
--- /dev/null
+++ b/cli/beacon-hooks/internal/cloudshuttle/transaction.go
@@ -0,0 +1,31 @@
+package cloudshuttle
+
+import (
+ "context"
+ "os"
+ "path/filepath"
+
+ "github.com/asymptote-labs/agent-beacon/pkg/asymptoteobserve/filelock"
+)
+
+// lockUpload serializes state reads, snapshots, provider acknowledgment and checkpoint
+// replacement across hook processes. The file remains in place even when state is reset.
+// Lock order is transaction -> runtime log; no network work holds the runtime-log lock.
+func lockUpload(ctx context.Context, statePath string) (func(), error) {
+ if err := os.MkdirAll(filepath.Dir(statePath), 0755); err != nil {
+ return nil, err
+ }
+ f, err := os.OpenFile(statePath+".lock", os.O_CREATE|os.O_RDWR, 0600)
+ if err != nil {
+ return nil, err
+ }
+ held, err := filelock.ExclusiveContext(ctx, f)
+ if err != nil {
+ _ = f.Close()
+ return nil, err
+ }
+ return func() {
+ _ = held.Release()
+ _ = f.Close()
+ }, nil
+}
diff --git a/cli/beacon-hooks/internal/cloudshuttle/transaction_edges_test.go b/cli/beacon-hooks/internal/cloudshuttle/transaction_edges_test.go
new file mode 100644
index 00000000..59bddbd6
--- /dev/null
+++ b/cli/beacon-hooks/internal/cloudshuttle/transaction_edges_test.go
@@ -0,0 +1,78 @@
+package cloudshuttle
+
+import (
+ "context"
+ "errors"
+ "net/http"
+ "os"
+ "path/filepath"
+ "sync/atomic"
+ "testing"
+ "time"
+
+ "github.com/asymptote-labs/agent-beacon/pkg/asymptoteobserve/filelock"
+)
+
+func TestUploadCancellationWhileWaitingForWriterLock(t *testing.T) {
+ cfg := transactionConfig(t)
+ transactionWrite(t, cfg.LogPath, transactionFirst)
+ f, err := os.OpenFile(cfg.LogPath+".lock", os.O_CREATE|os.O_RDWR, 0600)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer f.Close()
+ held, err := filelock.Exclusive(f)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer held.Release()
+ var calls atomic.Int32
+ transactionClient(t, func(r *http.Request) (*http.Response, error) { calls.Add(1); return transactionOK(), nil })
+ ctx, cancel := context.WithTimeout(context.Background(), 80*time.Millisecond)
+ defer cancel()
+ done := make(chan error, 1)
+ go func() { done <- Upload(ctx, cfg, true) }()
+ select {
+ case err := <-done:
+ if !errors.Is(err, context.DeadlineExceeded) {
+ t.Errorf("writer contention = %v", err)
+ }
+ case <-time.After(time.Second):
+ t.Error("writer contention ignored context deadline")
+ if err := held.Release(); err != nil {
+ t.Fatal(err)
+ }
+ transactionAwait(t, done)
+ }
+ if calls.Load() != 0 {
+ t.Fatal("cancelled snapshot touched provider")
+ }
+}
+
+func TestUploadTransactionLockErrorsFailClosed(t *testing.T) {
+ cfg := transactionConfig(t)
+ transactionWrite(t, cfg.LogPath, transactionFirst)
+ if err := os.Mkdir(cfg.StatePath+".lock", 0700); err != nil {
+ t.Fatal(err)
+ }
+ var calls atomic.Int32
+ transactionClient(t, func(r *http.Request) (*http.Response, error) { calls.Add(1); return transactionOK(), nil })
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+ defer cancel()
+ err := Upload(ctx, cfg, true)
+ if err == nil || errors.Is(err, context.DeadlineExceeded) {
+ t.Fatalf("real lock-open error ignored/retried: %v", err)
+ }
+ if calls.Load() != 0 {
+ t.Fatal("lock-open error touched provider")
+ }
+ if err := os.Remove(cfg.StatePath + ".lock"); err != nil {
+ t.Fatal(err)
+ }
+ if err := Upload(ctx, cfg, true); err != nil {
+ t.Fatalf("retry after lock-open repair: %v", err)
+ }
+ if _, err := os.Stat(filepath.Join(filepath.Dir(cfg.StatePath), "state.json")); err != nil {
+ t.Fatal(err)
+ }
+}
diff --git a/cli/beacon-hooks/internal/cloudshuttle/upload_transaction_test.go b/cli/beacon-hooks/internal/cloudshuttle/upload_transaction_test.go
new file mode 100644
index 00000000..fc293d59
--- /dev/null
+++ b/cli/beacon-hooks/internal/cloudshuttle/upload_transaction_test.go
@@ -0,0 +1,326 @@
+package cloudshuttle
+
+import (
+ "compress/gzip"
+ "context"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io"
+ "net/http"
+ "net/http/httptest"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "strings"
+ "sync"
+ "sync/atomic"
+ "testing"
+ "time"
+
+ "github.com/asymptote-labs/agent-beacon/pkg/asymptoteobserve/filelock"
+)
+
+const transactionFirst = "{\"n\":1}\n"
+const transactionBoth = transactionFirst + "{\"n\":2}\n"
+
+type transactionTransport func(*http.Request) (*http.Response, error)
+
+func (f transactionTransport) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) }
+
+func transactionConfig(t *testing.T) Config {
+ t.Helper()
+ dir := t.TempDir()
+ return Config{Upload: uploadS3, LogPath: filepath.Join(dir, "runtime.jsonl"),
+ StatePath: filepath.Join(dir, "state.json"), Bucket: "synthetic-bucket", Provider: "claude_code_web",
+ RunID: "same-run", S3Region: "us-east-1", S3AccessKeyID: "synthetic", S3SecretKey: "synthetic",
+ S3Endpoint: "http://127.0.0.1:1"}
+}
+
+func transactionWrite(t *testing.T, path, contents string) {
+ t.Helper()
+ if err := os.WriteFile(path, []byte(contents), 0600); err != nil {
+ t.Fatal(err)
+ }
+}
+
+func transactionSeed(t *testing.T, cfg Config) {
+ t.Helper()
+ if err := writeState(cfg.StatePath, state{Provider: cfg.Provider, RunID: cfg.RunID,
+ LastObject: "beacon/runtime/existing-run.jsonl.gz"}); err != nil {
+ t.Fatal(err)
+ }
+}
+
+func transactionBody(r *http.Request) (string, error) {
+ gz, err := gzip.NewReader(r.Body)
+ if err != nil {
+ return "", err
+ }
+ defer gz.Close()
+ b, err := io.ReadAll(gz)
+ return string(b), err
+}
+
+func transactionOK() *http.Response {
+ return &http.Response{StatusCode: 200, Body: io.NopCloser(strings.NewReader("{}")), Header: make(http.Header)}
+}
+
+func transactionClient(t *testing.T, transport transactionTransport) {
+ t.Helper()
+ old := httpClient
+ httpClient = &http.Client{Transport: transport}
+ t.Cleanup(func() { httpClient = old })
+}
+
+func transactionAwait(t *testing.T, ch <-chan error) error {
+ t.Helper()
+ select {
+ case err := <-ch:
+ return err
+ case <-time.After(10 * time.Second):
+ t.Fatal("upload did not finish")
+ return nil
+ }
+}
+
+// Both successful uploads must commit in snapshot order, even across hook processes.
+// The seed fixes the remote key independently of the wall clock.
+func TestUploadSerializesSnapshotsAndCheckpoint(t *testing.T) {
+ for _, processes := range []bool{false, true} {
+ t.Run(fmt.Sprintf("processes=%t", processes), func(t *testing.T) {
+ cfg := transactionConfig(t)
+ transactionSeed(t, cfg)
+ transactionWrite(t, cfg.LogPath, transactionFirst)
+ first, newer, release := make(chan struct{}), make(chan struct{}), make(chan struct{})
+ var once sync.Once
+ defer once.Do(func() { close(release) })
+ var mu sync.Mutex
+ var remote string
+ var requests int
+ handler := func(r *http.Request) (*http.Response, error) {
+ body, err := transactionBody(r)
+ if err != nil {
+ return nil, err
+ }
+ mu.Lock()
+ requests++
+ n := requests
+ mu.Unlock()
+ if n == 1 {
+ close(first)
+ select {
+ case <-release:
+ case <-r.Context().Done():
+ return nil, r.Context().Err()
+ }
+ } else {
+ close(newer)
+ }
+ mu.Lock()
+ remote = body
+ mu.Unlock()
+ return transactionOK(), nil
+ }
+ if processes {
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ resp, err := handler(r)
+ if err != nil {
+ http.Error(w, err.Error(), 500)
+ return
+ }
+ w.WriteHeader(resp.StatusCode)
+ }))
+ defer server.Close()
+ cfg.S3Endpoint = server.URL
+ } else {
+ transactionClient(t, handler)
+ }
+ start := func() <-chan error {
+ done := make(chan error, 1)
+ go func() {
+ if processes {
+ done <- transactionChild(t, cfg, "success")
+ } else {
+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+ defer cancel()
+ done <- Upload(ctx, cfg, true)
+ }
+ }()
+ return done
+ }
+ a := start()
+ select {
+ case <-first:
+ case <-time.After(5 * time.Second):
+ t.Fatal("first upload never reached provider")
+ }
+ transactionWrite(t, cfg.LogPath, transactionBoth)
+ b := start()
+ // A broken transaction reaches/finishes the second PUT first; the correct
+ // implementation waits until A releases its transaction ownership.
+ select {
+ case <-newer:
+ if err := transactionAwait(t, b); err != nil {
+ t.Error(err)
+ }
+ once.Do(func() { close(release) })
+ case <-time.After(400 * time.Millisecond):
+ once.Do(func() { close(release) })
+ if err := transactionAwait(t, b); err != nil {
+ t.Error(err)
+ }
+ }
+ if err := transactionAwait(t, a); err != nil {
+ t.Error(err)
+ }
+ mu.Lock()
+ got := remote
+ mu.Unlock()
+ st, err := readState(cfg.StatePath)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if got != transactionBoth || st.LastSize != int64(len(transactionBoth)) {
+ t.Fatalf("acknowledged newer snapshot lost: remote=%q LastSize=%d; want %q/%d", got, st.LastSize, transactionBoth, len(transactionBoth))
+ }
+ if st.LastObject != "beacon/runtime/existing-run.jsonl.gz" {
+ t.Fatalf("existing bucket key changed: %+v", st)
+ }
+ })
+ }
+}
+
+func transactionChild(t *testing.T, cfg Config, expectation string) error {
+ t.Helper()
+ data, err := json.Marshal(cfg)
+ if err != nil {
+ return err
+ }
+ executable, err := os.Executable()
+ if err != nil {
+ return err
+ }
+ cmd := exec.Command(executable, "-test.run=^TestUploadTransactionChild$", "-test.timeout=10s")
+ cmd.Env = append(os.Environ(), "BEACON_TRANSACTION_CHILD="+string(data), "BEACON_TRANSACTION_EXPECT="+expectation)
+ output, err := cmd.CombinedOutput()
+ if err != nil {
+ return fmt.Errorf("child: %w\n%s", err, output)
+ }
+ return nil
+}
+
+func TestUploadTransactionChild(t *testing.T) {
+ raw := os.Getenv("BEACON_TRANSACTION_CHILD")
+ if raw == "" {
+ return
+ }
+ var cfg Config
+ if err := json.Unmarshal([]byte(raw), &cfg); err != nil {
+ t.Fatal(err)
+ }
+ duration := 5 * time.Second
+ if os.Getenv("BEACON_TRANSACTION_EXPECT") == "deadline" {
+ duration = 150 * time.Millisecond
+ }
+ ctx, cancel := context.WithTimeout(context.Background(), duration)
+ defer cancel()
+ err := Upload(ctx, cfg, true)
+ if os.Getenv("BEACON_TRANSACTION_EXPECT") == "deadline" {
+ if !errors.Is(err, context.DeadlineExceeded) {
+ t.Fatalf("contended upload = %v; want deadline", err)
+ }
+ } else if err != nil {
+ t.Fatal(err)
+ }
+}
+
+// Holding the protocol lock from another process is a causal exclusion proof:
+// the child must exhaust its deadline without touching the fake provider.
+func TestUploadTransactionCrossProcessContentionAndIndependentTargets(t *testing.T) {
+ cfg := transactionConfig(t)
+ transactionWrite(t, cfg.LogPath, transactionFirst)
+ var calls atomic.Int32
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { calls.Add(1); w.WriteHeader(200) }))
+ defer server.Close()
+ cfg.S3Endpoint = server.URL
+ f, err := os.OpenFile(cfg.StatePath+".lock", os.O_CREATE|os.O_RDWR, 0600)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer f.Close()
+ held, err := filelock.Exclusive(f)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer held.Release()
+ if err := transactionChild(t, cfg, "deadline"); err != nil {
+ t.Error(err)
+ }
+ if got := calls.Load(); got != 0 {
+ t.Errorf("contended child touched provider %d times", got)
+ }
+ other := cfg
+ other.Bucket = "independent-bucket"
+ other.StatePath = filepath.Join(filepath.Dir(cfg.StatePath), "independent.json")
+ if err := transactionChild(t, other, "success"); err != nil {
+ t.Fatal(err)
+ }
+ if err := held.Release(); err != nil {
+ t.Fatal(err)
+ }
+ if err := transactionChild(t, cfg, "success"); err != nil {
+ t.Fatal(err)
+ }
+}
+
+func TestUploadReleasesTransactionAfterProviderAndCheckpointFailures(t *testing.T) {
+ for _, fault := range []string{"rejection", "timeout", "checkpoint"} {
+ t.Run(fault, func(t *testing.T) {
+ cfg := transactionConfig(t)
+ transactionWrite(t, cfg.LogPath, transactionFirst)
+ if fault == "checkpoint" {
+ if err := os.Mkdir(cfg.StatePath, 0700); err != nil {
+ t.Fatal(err)
+ }
+ }
+ fail := true
+ transactionClient(t, func(r *http.Request) (*http.Response, error) {
+ if fail && fault == "timeout" {
+ <-r.Context().Done()
+ return nil, r.Context().Err()
+ }
+ resp := transactionOK()
+ if fail && fault == "rejection" {
+ resp.StatusCode = 503
+ }
+ return resp, nil
+ })
+ ctx, cancel := context.WithTimeout(context.Background(), 80*time.Millisecond)
+ err := Upload(ctx, cfg, true)
+ cancel()
+ if err == nil {
+ t.Fatal("failed provider/checkpoint reported success")
+ }
+ if fault == "checkpoint" {
+ if err := os.Remove(cfg.StatePath); err != nil {
+ t.Fatal(err)
+ }
+ }
+ fail = false
+ ctx, cancel = context.WithTimeout(context.Background(), time.Second)
+ defer cancel()
+ if err := Upload(ctx, cfg, true); err != nil {
+ t.Fatalf("retry blocked or failed after %s: %v", fault, err)
+ }
+ st, err := readState(cfg.StatePath)
+ if err != nil || st.LastSize != int64(len(transactionFirst)) {
+ t.Fatalf("retry checkpoint: %+v %v", st, err)
+ }
+ matches, _ := filepath.Glob(filepath.Join(filepath.Dir(cfg.StatePath), ".shuttle-state-*"))
+ if len(matches) != 0 {
+ t.Fatalf("checkpoint temporary files leaked: %v", matches)
+ }
+ })
+ }
+}
diff --git a/pkg/asymptoteobserve/filelock/filelock.go b/pkg/asymptoteobserve/filelock/filelock.go
index 9b8f35a1..900c6c6c 100644
--- a/pkg/asymptoteobserve/filelock/filelock.go
+++ b/pkg/asymptoteobserve/filelock/filelock.go
@@ -13,7 +13,11 @@
// has to get right anyway.
package filelock
-import "os"
+import (
+ "context"
+ "os"
+ "time"
+)
// Lock is a held lock. Release unlocks; closing the file is the caller's business, since the
// caller opened it.
@@ -59,12 +63,52 @@ func TryExclusive(f *os.File) (*Lock, error) {
if f == nil {
return nil, os.ErrInvalid
}
- if err := tryLockFile(f); err != nil {
+ if err := tryLockFile(f, true); err != nil {
return nil, err
}
return &Lock{file: f}, nil
}
+// ExclusiveContext waits for an exclusive lock until ctx is cancelled. Only actual lock
+// contention is retried; invalid handles and other operating-system errors are returned.
+func ExclusiveContext(ctx context.Context, f *os.File) (*Lock, error) {
+ return lockContext(ctx, f, true)
+}
+
+// SharedContext is Shared with a cancellable contention wait.
+func SharedContext(ctx context.Context, f *os.File) (*Lock, error) {
+ return lockContext(ctx, f, false)
+}
+
+func lockContext(ctx context.Context, f *os.File, exclusive bool) (*Lock, error) {
+ if f == nil {
+ return nil, os.ErrInvalid
+ }
+ for {
+ if err := ctx.Err(); err != nil {
+ return nil, err
+ }
+ err := tryLockFile(f, exclusive)
+ if err == nil {
+ if err := ctx.Err(); err != nil {
+ _ = unlockFile(f)
+ return nil, err
+ }
+ return &Lock{file: f}, nil
+ }
+ if !isContention(err) {
+ return nil, err
+ }
+ timer := time.NewTimer(10 * time.Millisecond)
+ select {
+ case <-ctx.Done():
+ timer.Stop()
+ return nil, ctx.Err()
+ case <-timer.C:
+ }
+ }
+}
+
// Release unlocks. Safe to call more than once, so `defer l.Release()` alongside an explicit
// release on the success path cannot double-unlock -- on Windows that would return an error for
// a lock the process no longer holds, which reads as a failure where there is none.
diff --git a/pkg/asymptoteobserve/filelock/filelock_context_test.go b/pkg/asymptoteobserve/filelock/filelock_context_test.go
new file mode 100644
index 00000000..bcbbd6a0
--- /dev/null
+++ b/pkg/asymptoteobserve/filelock/filelock_context_test.go
@@ -0,0 +1,105 @@
+package filelock
+
+import (
+ "context"
+ "errors"
+ "os"
+ "path/filepath"
+ "testing"
+ "time"
+)
+
+func TestContextLocksCancelContentionAndRelease(t *testing.T) {
+ for _, shared := range []bool{false, true} {
+ name := "exclusive"
+ if shared {
+ name = "shared"
+ }
+ t.Run(name, func(t *testing.T) {
+ path := filepath.Join(t.TempDir(), "transaction.lock")
+ first, err := Exclusive(openLockFile(t, path))
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer first.Release()
+ second := openLockFile(t, path)
+ acquire := ExclusiveContext
+ if shared {
+ acquire = SharedContext
+ }
+ ctx, cancel := context.WithTimeout(context.Background(), 60*time.Millisecond)
+ held, err := acquire(ctx, second)
+ cancel()
+ if !errors.Is(err, context.DeadlineExceeded) || held != nil {
+ t.Fatalf("contended lock = %v, %v; want deadline without ownership", held, err)
+ }
+ if err := first.Release(); err != nil {
+ t.Fatal(err)
+ }
+ ctx, cancel = context.WithTimeout(context.Background(), time.Second)
+ defer cancel()
+ held, err = acquire(ctx, second)
+ if err != nil {
+ t.Fatalf("lock unavailable after cancellation: %v", err)
+ }
+ if err := held.Release(); err != nil {
+ t.Fatal(err)
+ }
+ })
+ }
+}
+
+func TestContextLocksFailClosedOnRealErrors(t *testing.T) {
+ for name, acquire := range map[string]func(context.Context, *os.File) (*Lock, error){
+ "exclusive": ExclusiveContext, "shared": SharedContext,
+ } {
+ t.Run(name, func(t *testing.T) {
+ if _, err := acquire(context.Background(), nil); !errors.Is(err, os.ErrInvalid) {
+ t.Fatalf("nil file: %v", err)
+ }
+ f := openLockFile(t, filepath.Join(t.TempDir(), "closed.lock"))
+ if err := f.Close(); err != nil {
+ t.Fatal(err)
+ }
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+ defer cancel()
+ _, err := acquire(ctx, f)
+ if err == nil || errors.Is(err, context.DeadlineExceeded) {
+ t.Fatalf("real lock error retried/ignored: %v", err)
+ }
+ })
+ }
+}
+
+func TestContextLocksAlreadyCancelledDoNotAcquire(t *testing.T) {
+ ctx, cancel := context.WithCancel(context.Background())
+ cancel()
+ f := openLockFile(t, filepath.Join(t.TempDir(), "free.lock"))
+ for _, acquire := range []func(context.Context, *os.File) (*Lock, error){ExclusiveContext, SharedContext} {
+ held, err := acquire(ctx, f)
+ if !errors.Is(err, context.Canceled) || held != nil {
+ t.Fatalf("cancelled lock = %v, %v", held, err)
+ }
+ }
+ other, err := TryExclusive(openLockFile(t, f.Name()))
+ if err != nil {
+ t.Fatalf("cancelled acquisition leaked ownership: %v", err)
+ }
+ defer other.Release()
+}
+
+func TestSharedContextLocksCoexist(t *testing.T) {
+ path := filepath.Join(t.TempDir(), "shared.lock")
+ first, err := SharedContext(context.Background(), openLockFile(t, path))
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer first.Release()
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+ defer cancel()
+ second, err := SharedContext(ctx, openLockFile(t, path))
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer second.Release()
+}
diff --git a/pkg/asymptoteobserve/filelock/filelock_unix.go b/pkg/asymptoteobserve/filelock/filelock_unix.go
index 2449776b..a871d4f4 100644
--- a/pkg/asymptoteobserve/filelock/filelock_unix.go
+++ b/pkg/asymptoteobserve/filelock/filelock_unix.go
@@ -3,6 +3,7 @@
package filelock
import (
+ "errors"
"os"
"syscall"
)
@@ -16,10 +17,18 @@ func lockFile(f *os.File, exclusive bool) error {
return syscall.Flock(int(f.Fd()), how)
}
-// tryLockFile takes an exclusive lock without waiting, reporting EWOULDBLOCK when another holder
+// tryLockFile takes a lock without waiting, reporting EWOULDBLOCK when another holder
// has it.
-func tryLockFile(f *os.File) error {
- return syscall.Flock(int(f.Fd()), syscall.LOCK_EX|syscall.LOCK_NB)
+func tryLockFile(f *os.File, exclusive bool) error {
+ how := syscall.LOCK_SH
+ if exclusive {
+ how = syscall.LOCK_EX
+ }
+ return syscall.Flock(int(f.Fd()), how|syscall.LOCK_NB)
+}
+
+func isContention(err error) bool {
+ return errors.Is(err, syscall.EWOULDBLOCK) || errors.Is(err, syscall.EAGAIN)
}
func unlockFile(f *os.File) error {
diff --git a/pkg/asymptoteobserve/filelock/filelock_windows.go b/pkg/asymptoteobserve/filelock/filelock_windows.go
index ea3f6121..0c534840 100644
--- a/pkg/asymptoteobserve/filelock/filelock_windows.go
+++ b/pkg/asymptoteobserve/filelock/filelock_windows.go
@@ -3,6 +3,7 @@
package filelock
import (
+ "errors"
"os"
"golang.org/x/sys/windows"
@@ -36,15 +37,23 @@ func lockFile(f *os.File, exclusive bool) error {
0, lockRangeLow, lockRangeHigh, &overlapped)
}
-// tryLockFile takes an exclusive lock without waiting, reporting ERROR_LOCK_VIOLATION when another
+// tryLockFile takes a lock without waiting, reporting ERROR_LOCK_VIOLATION when another
// holder has it.
-func tryLockFile(f *os.File) error {
+func tryLockFile(f *os.File, exclusive bool) error {
+ flags := uint32(windows.LOCKFILE_FAIL_IMMEDIATELY)
+ if exclusive {
+ flags |= windows.LOCKFILE_EXCLUSIVE_LOCK
+ }
var overlapped windows.Overlapped
return windows.LockFileEx(windows.Handle(f.Fd()),
- windows.LOCKFILE_EXCLUSIVE_LOCK|windows.LOCKFILE_FAIL_IMMEDIATELY,
+ flags,
0, lockRangeLow, lockRangeHigh, &overlapped)
}
+func isContention(err error) bool {
+ return errors.Is(err, windows.ERROR_LOCK_VIOLATION)
+}
+
func unlockFile(f *os.File) error {
var overlapped windows.Overlapped
return windows.UnlockFileEx(windows.Handle(f.Fd()),