Skip to content

The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu

fix(memory): keep review state transitions atomic (#985)

Divyam Talwar20e82bfed460
diff --git a/cli/beacon/internal/learning/candidate.go b/cli/beacon/internal/learning/candidate.go
index 3c5a04e6..66f3d3bb 100644
--- a/cli/beacon/internal/learning/candidate.go
+++ b/cli/beacon/internal/learning/candidate.go
@@ -133,7 +133,17 @@ func ApproveCandidateWithEdits(store *Store, id, reason string, edits ApprovalEd
if kind != "" && !ValidMemoryKind(kind) {
return asymptoteobserve.LearningCandidateV1{}, asymptoteobserve.LearningMemoryV1{}, fmt.Errorf("unknown memory kind %q (want workflow, correction, debugging_pattern, gotcha, or convention)", kind)
}
- candidate, ok, err := store.GetCandidate(id)
+ db, err := store.db()
+ if err != nil {
+ return asymptoteobserve.LearningCandidateV1{}, asymptoteobserve.LearningMemoryV1{}, err
+ }
+ defer db.Close()
+ tx, err := db.Begin()
+ if err != nil {
+ return asymptoteobserve.LearningCandidateV1{}, asymptoteobserve.LearningMemoryV1{}, err
+ }
+ defer tx.Rollback()
+ candidate, ok, err := getCandidate(tx, id)
if err != nil {
return asymptoteobserve.LearningCandidateV1{}, asymptoteobserve.LearningMemoryV1{}, err
}
@@ -162,10 +172,13 @@ func ApproveCandidateWithEdits(store *Store, id, reason string, edits ApprovalEd
candidate.MemoryID = memory.ID
candidate.ApprovedAt = now
candidate.ReviewReason = strings.TrimSpace(reason)
- if err := store.PutMemory(memory); err != nil {
+ if err := putMemory(tx, memory); err != nil {
return asymptoteobserve.LearningCandidateV1{}, asymptoteobserve.LearningMemoryV1{}, err
}
- if err := store.PutCandidate(candidate); err != nil {
+ if err := putCandidate(tx, candidate); err != nil {
+ return asymptoteobserve.LearningCandidateV1{}, asymptoteobserve.LearningMemoryV1{}, err
+ }
+ if err := tx.Commit(); err != nil {
return asymptoteobserve.LearningCandidateV1{}, asymptoteobserve.LearningMemoryV1{}, err
}
return candidate, memory, nil
@@ -191,7 +204,17 @@ func approvedTags(edited, original []string) []string {
}
func RejectCandidate(store *Store, id, reason string) (asymptoteobserve.LearningCandidateV1, error) {
- candidate, ok, err := store.GetCandidate(id)
+ db, err := store.db()
+ if err != nil {
+ return asymptoteobserve.LearningCandidateV1{}, err
+ }
+ defer db.Close()
+ tx, err := db.Begin()
+ if err != nil {
+ return asymptoteobserve.LearningCandidateV1{}, err
+ }
+ defer tx.Rollback()
+ candidate, ok, err := getCandidate(tx, id)
if err != nil {
return asymptoteobserve.LearningCandidateV1{}, err
}
@@ -204,7 +227,10 @@ func RejectCandidate(store *Store, id, reason string) (asymptoteobserve.Learning
candidate.State = asymptoteobserve.LearningCandidateStateRejected
candidate.RejectedAt = nowString()
candidate.ReviewReason = strings.TrimSpace(reason)
- if err := store.PutCandidate(candidate); err != nil {
+ if err := putCandidate(tx, candidate); err != nil {
+ return asymptoteobserve.LearningCandidateV1{}, err
+ }
+ if err := tx.Commit(); err != nil {
return asymptoteobserve.LearningCandidateV1{}, err
}
return candidate, nil
@@ -214,14 +240,24 @@ func SupersedeCandidate(store *Store, id, replacementMemoryID, reason string) (a
if strings.TrimSpace(replacementMemoryID) == "" {
return asymptoteobserve.LearningCandidateV1{}, fmt.Errorf("replacement memory id is required")
}
- replacement, ok, err := store.GetMemory(replacementMemoryID)
+ db, err := store.db()
+ if err != nil {
+ return asymptoteobserve.LearningCandidateV1{}, err
+ }
+ defer db.Close()
+ tx, err := db.Begin()
+ if err != nil {
+ return asymptoteobserve.LearningCandidateV1{}, err
+ }
+ defer tx.Rollback()
+ replacement, ok, err := getMemory(tx, replacementMemoryID)
if err != nil {
return asymptoteobserve.LearningCandidateV1{}, err
}
if !ok {
return asymptoteobserve.LearningCandidateV1{}, fmt.Errorf("replacement memory not found: %s", replacementMemoryID)
}
- candidate, ok, err := store.GetCandidate(id)
+ candidate, ok, err := getCandidate(tx, id)
if err != nil {
return asymptoteobserve.LearningCandidateV1{}, err
}
@@ -236,22 +272,25 @@ func SupersedeCandidate(store *Store, id, replacementMemoryID, reason string) (a
candidate.SupersededAt = now
candidate.SupersededBy = replacement.ID
candidate.ReviewReason = strings.TrimSpace(reason)
- if err := store.PutCandidate(candidate); err != nil {
+ if err := putCandidate(tx, candidate); err != nil {
return asymptoteobserve.LearningCandidateV1{}, err
}
if candidate.MemoryID != "" {
- memory, ok, err := store.GetMemory(candidate.MemoryID)
+ memory, ok, err := getMemory(tx, candidate.MemoryID)
if err != nil {
return asymptoteobserve.LearningCandidateV1{}, err
}
if ok {
memory.SupersededBy = replacement.ID
memory.UpdatedAt = now
- if err := store.PutMemory(memory); err != nil {
+ if err := putMemory(tx, memory); err != nil {
return asymptoteobserve.LearningCandidateV1{}, err
}
}
}
+ if err := tx.Commit(); err != nil {
+ return asymptoteobserve.LearningCandidateV1{}, err
+ }
return candidate, nil
}
diff --git a/cli/beacon/internal/learning/candidate_insert_test.go b/cli/beacon/internal/learning/candidate_insert_test.go
index 9350ef8d..bc0df516 100644
--- a/cli/beacon/internal/learning/candidate_insert_test.go
+++ b/cli/beacon/internal/learning/candidate_insert_test.go
@@ -1,7 +1,6 @@
package learning
import (
- "net/url"
"path/filepath"
"reflect"
"strings"
@@ -12,7 +11,6 @@ import (
"modernc.org/sqlite"
- "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/sqliteuri"
"github.com/asymptote-labs/agent-beacon/pkg/asymptoteobserve"
)
@@ -124,7 +122,7 @@ func TestInsertCandidateDoesNotOverwriteConcurrentReview(t *testing.T) {
t.Fatal(err)
}
gate := &candidateGenerationGate{paused: make(chan struct{}), resume: make(chan struct{})}
- dsn := sqliteuri.File(store.Path(), url.Values{"_pragma": {"busy_timeout(5000)"}})
+ dsn := store.dsn()
candidateGenerationGates.Store(dsn, gate)
var release sync.Once
finished := make(chan struct{})
diff --git a/cli/beacon/internal/learning/candidate_transaction_test.go b/cli/beacon/internal/learning/candidate_transaction_test.go
new file mode 100644
index 00000000..50a9594a
--- /dev/null
+++ b/cli/beacon/internal/learning/candidate_transaction_test.go
@@ -0,0 +1,237 @@
+package learning
+
+import (
+ "fmt"
+ "path/filepath"
+ "reflect"
+ "strings"
+ "testing"
+
+ "github.com/asymptote-labs/agent-beacon/pkg/asymptoteobserve"
+)
+
+func TestReviewApprovalRollsBackAndRetries(t *testing.T) {
+ for _, boundary := range []struct{ name, trigger string }{
+ {"memory-insert", "BEFORE INSERT ON memories"},
+ {"candidate-update", "BEFORE UPDATE ON candidates"},
+ } {
+ t.Run(boundary.name, func(t *testing.T) {
+ store := Open(filepath.Join(t.TempDir(), "memory.db"))
+ candidate := putReviewCandidate(t, store, "approval")
+ before := snapshotReview(t, store)
+ setReviewFault(t, store, boundary.trigger)
+ if _, _, err := ApproveCandidate(store, candidate.ID, "first review"); err == nil || !strings.Contains(err.Error(), "review write fault") {
+ t.Fatalf("approval error = %v, want injected write failure", err)
+ }
+ if after := snapshotReview(t, store); !reflect.DeepEqual(after, before) {
+ t.Errorf("failed approval changed persisted records or recall:\nbefore=%#v\nafter=%#v", before, after)
+ }
+ clearReviewFault(t, store)
+ approved, memory, err := ApproveCandidateWithEdits(store, candidate.ID, "retry review", ApprovalEdits{
+ Title: "Edited retry title", Body: " Reviewer-authored lesson. ", Kind: asymptoteobserve.LearningMemoryKindWorkflow,
+ })
+ if err != nil {
+ t.Fatalf("retry with reviewer edits: %v", err)
+ }
+ if approved.State != asymptoteobserve.LearningCandidateStateApproved || approved.MemoryID != memory.ID || memory.Title != "Edited retry title" || memory.Body != "Reviewer-authored lesson." {
+ t.Fatalf("retry result = %#v / %#v", approved, memory)
+ }
+ if got := snapshotReview(t, store); len(got.memories) != 1 || !reflect.DeepEqual(got.recalled, []string{memory.ID}) {
+ t.Fatalf("retry left orphan or missing memory: %#v", got)
+ }
+ })
+ }
+}
+
+func TestReviewSupersessionRollsBackAndRetries(t *testing.T) {
+ for _, boundary := range []struct{ name, trigger string }{
+ {"candidate-update", "BEFORE UPDATE ON candidates"},
+ {"memory-update", "BEFORE UPDATE ON memories"},
+ } {
+ t.Run(boundary.name, func(t *testing.T) {
+ store := Open(filepath.Join(t.TempDir(), "memory.db"))
+ candidate := putReviewCandidate(t, store, "original")
+ _, original, err := ApproveCandidate(store, candidate.ID, "original review")
+ if err != nil {
+ t.Fatal(err)
+ }
+ replacementCandidate := putReviewCandidate(t, store, "replacement")
+ _, replacement, err := ApproveCandidate(store, replacementCandidate.ID, "replacement review")
+ if err != nil {
+ t.Fatal(err)
+ }
+ before := snapshotReview(t, store)
+ setReviewFault(t, store, boundary.trigger)
+ if _, err := SupersedeCandidate(store, candidate.ID, replacement.ID, "first supersession"); err == nil || !strings.Contains(err.Error(), "review write fault") {
+ t.Fatalf("supersession error = %v, want injected write failure", err)
+ }
+ if after := snapshotReview(t, store); !reflect.DeepEqual(after, before) {
+ t.Errorf("failed supersession changed persisted records or recall:\nbefore=%#v\nafter=%#v", before, after)
+ }
+ clearReviewFault(t, store)
+ superseded, err := SupersedeCandidate(store, candidate.ID, replacement.ID, "retry supersession")
+ if err != nil {
+ t.Fatalf("retry supersession: %v", err)
+ }
+ stored, ok, err := store.GetMemory(original.ID)
+ if err != nil || !ok || stored.SupersededBy != replacement.ID || superseded.SupersededBy != replacement.ID || superseded.State != asymptoteobserve.LearningCandidateStateSuperseded {
+ t.Fatalf("retry records = %#v / %#v, found=%v err=%v", superseded, stored, ok, err)
+ }
+ if got := snapshotReview(t, store); !reflect.DeepEqual(got.recalled, []string{replacement.ID}) {
+ t.Fatalf("superseded memory remains recallable: %#v", got)
+ }
+ })
+ }
+}
+
+func TestReviewRejectRollsBackAndRetries(t *testing.T) {
+ store := Open(filepath.Join(t.TempDir(), "memory.db"))
+ candidate := putReviewCandidate(t, store, "reject")
+ before := snapshotReview(t, store)
+ setReviewFault(t, store, "BEFORE UPDATE ON candidates")
+ if _, err := RejectCandidate(store, candidate.ID, "failed review"); err == nil || !strings.Contains(err.Error(), "review write fault") {
+ t.Fatalf("reject error = %v, want injected write failure", err)
+ }
+ if after := snapshotReview(t, store); !reflect.DeepEqual(after, before) {
+ t.Fatalf("failed rejection changed records: before=%#v after=%#v", before, after)
+ }
+ clearReviewFault(t, store)
+ rejected, err := RejectCandidate(store, candidate.ID, " retry rejection ")
+ if err != nil || rejected.State != asymptoteobserve.LearningCandidateStateRejected || rejected.ReviewReason != "retry rejection" {
+ t.Fatalf("retry rejection = %#v, err=%v", rejected, err)
+ }
+ if got := snapshotReview(t, store); len(got.memories) != 0 || len(got.recalled) != 0 {
+ t.Fatalf("rejection created a memory: %#v", got)
+ }
+}
+
+func TestReviewConcurrentApprovalAndRejectionStayConsistent(t *testing.T) {
+ store := Open(filepath.Join(t.TempDir(), "memory.db"))
+ for iteration := 0; iteration < 12; iteration++ {
+ candidate := putReviewCandidate(t, store, fmt.Sprintf("concurrent-%d", iteration))
+ start := make(chan struct{})
+ results := make(chan error, 2)
+ go func() {
+ <-start
+ _, _, err := ApproveCandidate(store, candidate.ID, "concurrent approval")
+ results <- err
+ }()
+ go func() {
+ <-start
+ _, err := RejectCandidate(store, candidate.ID, "concurrent rejection")
+ results <- err
+ }()
+ close(start)
+ first, second := <-results, <-results
+ if (first == nil) == (second == nil) {
+ t.Errorf("iteration %d: want one review winner, errors = %v / %v", iteration, first, second)
+ }
+ stored, ok, err := store.GetCandidate(candidate.ID)
+ if err != nil || !ok {
+ t.Fatalf("read reviewed candidate: found=%v err=%v", ok, err)
+ }
+ memories, err := store.ListMemories(Query{ProjectID: candidate.Project.ID, Limit: 500})
+ if err != nil {
+ t.Fatal(err)
+ }
+ var linked []asymptoteobserve.LearningMemoryV1
+ for _, memory := range memories {
+ if memory.CandidateID == candidate.ID {
+ linked = append(linked, memory)
+ }
+ }
+ switch stored.State {
+ case asymptoteobserve.LearningCandidateStateApproved:
+ if len(linked) != 1 || linked[0].ID != stored.MemoryID {
+ t.Errorf("iteration %d: approved candidate lacks its sole memory: %#v / %#v", iteration, stored, linked)
+ }
+ case asymptoteobserve.LearningCandidateStateRejected:
+ if len(linked) != 0 || stored.MemoryID != "" {
+ t.Errorf("iteration %d: rejected candidate has recallable memory: %#v / %#v", iteration, stored, linked)
+ }
+ default:
+ t.Errorf("iteration %d: review state = %q", iteration, stored.State)
+ }
+ }
+}
+
+func putReviewCandidate(t *testing.T, store *Store, suffix string) asymptoteobserve.LearningCandidateV1 {
+ t.Helper()
+ candidate, ok := CandidateFromEvaluation(testLearningEvaluationWithID("eval-"+suffix, "trace-"+suffix))
+ if !ok {
+ t.Fatal("candidate not created")
+ }
+ if err := store.PutCandidate(candidate); err != nil {
+ t.Fatal(err)
+ }
+ return candidate
+}
+
+func setReviewFault(t *testing.T, store *Store, boundary string) {
+ t.Helper()
+ execReviewSQL(t, store, "CREATE TRIGGER fail_review "+boundary+" BEGIN SELECT RAISE(ABORT, 'review write fault'); END")
+}
+
+func clearReviewFault(t *testing.T, store *Store) {
+ t.Helper()
+ execReviewSQL(t, store, "DROP TRIGGER fail_review")
+}
+
+func execReviewSQL(t *testing.T, store *Store, statement string) {
+ t.Helper()
+ db, err := store.db()
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer db.Close()
+ if _, err := db.Exec(statement); err != nil {
+ t.Fatal(err)
+ }
+}
+
+type reviewSnapshot struct {
+ candidates []string
+ memories []string
+ recalled []string
+}
+
+func snapshotReview(t *testing.T, store *Store) reviewSnapshot {
+ t.Helper()
+ db, err := store.db()
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer db.Close()
+ var snapshot reviewSnapshot
+ for _, table := range []struct {
+ query string
+ dest *[]string
+ }{
+ {"SELECT candidate_json FROM candidates ORDER BY id", &snapshot.candidates},
+ {"SELECT memory_json FROM memories ORDER BY id", &snapshot.memories},
+ } {
+ rows, err := db.Query(table.query)
+ if err != nil {
+ t.Fatal(err)
+ }
+ for rows.Next() {
+ var value string
+ if err := rows.Scan(&value); err != nil {
+ t.Fatal(err)
+ }
+ *table.dest = append(*table.dest, value)
+ }
+ if err := rows.Err(); err != nil {
+ t.Fatal(err)
+ }
+ rows.Close()
+ }
+ memories, err := store.ListMemories(Query{Limit: 500})
+ if err != nil {
+ t.Fatal(err)
+ }
+ for _, memory := range memories {
+ snapshot.recalled = append(snapshot.recalled, memory.ID)
+ }
+ return snapshot
+}
diff --git a/cli/beacon/internal/learning/store.go b/cli/beacon/internal/learning/store.go
index c270364c..45e27212 100644
--- a/cli/beacon/internal/learning/store.go
+++ b/cli/beacon/internal/learning/store.go
@@ -27,6 +27,11 @@ type Store struct {
dbPath string
}
+type recordDB interface {
+ Exec(query string, args ...any) (sql.Result, error)
+ QueryRow(query string, args ...any) *sql.Row
+}
+
// Query selects stored learning records.
//
// ProjectPath is the trust boundary of this package. The CLI resolves an
@@ -64,6 +69,12 @@ func (s *Store) Path() string {
return s.dbPath
}
+// dsn opens the store with a busy timeout, and with review transactions taking the write lock at
+// BEGIN, so a competing review waits instead of failing with SQLITE_BUSY when it upgrades.
+func (s *Store) dsn() string {
+ return sqliteuri.File(s.dbPath, url.Values{"_pragma": {"busy_timeout(5000)"}, "_txlock": {"immediate"}})
+}
+
func (s *Store) db() (*sql.DB, error) {
if s.dbPath == "" {
s.dbPath = storeFile
@@ -71,7 +82,7 @@ func (s *Store) db() (*sql.DB, error) {
if err := os.MkdirAll(filepath.Dir(s.dbPath), 0700); err != nil && filepath.Dir(s.dbPath) != "." {
return nil, err
}
- db, err := sql.Open("sqlite", sqliteuri.File(s.dbPath, url.Values{"_pragma": {"busy_timeout(5000)"}}))
+ db, err := sql.Open("sqlite", s.dsn())
if err != nil {
return nil, err
}
@@ -465,15 +476,19 @@ func (s *Store) GetEvaluation(id string) (asymptoteobserve.LearningEvaluationV1,
}
func (s *Store) PutCandidate(c asymptoteobserve.LearningCandidateV1) error {
- c, data, err := prepareCandidate(c)
+ db, err := s.db()
if err != nil {
return err
}
- db, err := s.db()
+ defer db.Close()
+ return putCandidate(db, c)
+}
+
+func putCandidate(db recordDB, c asymptoteobserve.LearningCandidateV1) error {
+ c, data, err := prepareCandidate(c)
if err != nil {
return err
}
- defer db.Close()
_, err = db.Exec(`INSERT INTO candidates (id, project_id, state, kind, title, source_evaluation_id, memory_id, created_at, updated_at, candidate_json)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(id) DO UPDATE SET state = excluded.state, kind = excluded.kind, title = excluded.title, memory_id = excluded.memory_id, updated_at = excluded.updated_at, candidate_json = excluded.candidate_json`,
@@ -533,8 +548,17 @@ func (s *Store) ListCandidates(query Query) ([]asymptoteobserve.LearningCandidat
}
func (s *Store) GetCandidate(id string) (asymptoteobserve.LearningCandidateV1, bool, error) {
+ db, err := s.db()
+ if err != nil {
+ return asymptoteobserve.LearningCandidateV1{}, false, err
+ }
+ defer db.Close()
+ return getCandidate(db, id)
+}
+
+func getCandidate(db recordDB, id string) (asymptoteobserve.LearningCandidateV1, bool, error) {
var raw string
- ok, err := s.getJSON(`SELECT candidate_json FROM candidates WHERE id = ?`, id, &raw)
+ ok, err := getJSON(db, `SELECT candidate_json FROM candidates WHERE id = ?`, id, &raw)
if err != nil || !ok {
return asymptoteobserve.LearningCandidateV1{}, ok, err
}
@@ -543,6 +567,15 @@ func (s *Store) GetCandidate(id string) (asymptoteobserve.LearningCandidateV1, b
}
func (s *Store) PutMemory(m asymptoteobserve.LearningMemoryV1) error {
+ db, err := s.db()
+ if err != nil {
+ return err
+ }
+ defer db.Close()
+ return putMemory(db, m)
+}
+
+func putMemory(db recordDB, m asymptoteobserve.LearningMemoryV1) error {
if m.SchemaVersion == "" {
m.SchemaVersion = asymptoteobserve.LearningSchemaVersion
}
@@ -558,11 +591,6 @@ func (s *Store) PutMemory(m asymptoteobserve.LearningMemoryV1) error {
if err != nil {
return err
}
- db, err := s.db()
- if err != nil {
- return err
- }
- defer db.Close()
_, err = db.Exec(`INSERT INTO memories (id, candidate_id, project_id, kind, title, created_at, updated_at, superseded_by, memory_json)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(id) DO UPDATE SET title = excluded.title, updated_at = excluded.updated_at, superseded_by = excluded.superseded_by, memory_json = excluded.memory_json`,
@@ -639,8 +667,17 @@ func listJSON[T any](s *Store, query Query, selectSQL string, args []interface{}
}
func (s *Store) GetMemory(id string) (asymptoteobserve.LearningMemoryV1, bool, error) {
+ db, err := s.db()
+ if err != nil {
+ return asymptoteobserve.LearningMemoryV1{}, false, err
+ }
+ defer db.Close()
+ return getMemory(db, id)
+}
+
+func getMemory(db recordDB, id string) (asymptoteobserve.LearningMemoryV1, bool, error) {
var raw string
- ok, err := s.getJSON(`SELECT memory_json FROM memories WHERE id = ?`, id, &raw)
+ ok, err := getJSON(db, `SELECT memory_json FROM memories WHERE id = ?`, id, &raw)
if err != nil || !ok {
return asymptoteobserve.LearningMemoryV1{}, ok, err
}
@@ -654,7 +691,11 @@ func (s *Store) getJSON(query, id string, dest *string) (bool, error) {
return false, err
}
defer db.Close()
- err = db.QueryRow(query, id).Scan(dest)
+ return getJSON(db, query, id, dest)
+}
+
+func getJSON(db recordDB, query, id string, dest *string) (bool, error) {
+ err := db.QueryRow(query, id).Scan(dest)
if err == sql.ErrNoRows {
return false, nil
}