The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu
Go agent-ai-cliagent-harnessesagent-securitybrowser-extensionciclaude-codecloudcodexcursordetection-engineeringjevknowledge-basememorymobile-device-managementobservabilitysecuritysecurity-information-and-event-managementsecurity-toolstelemetrytraces
package devincloud
import (
"context"
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"testing"
)
type fakeUploader struct {
objects map[string][]byte
calls int
}
func (f *fakeUploader) Upload(_ context.Context, objectName string, data []byte) error {
if f.objects == nil {
f.objects = map[string][]byte{}
}
f.objects[objectName] = data
f.calls++
return nil
}
func TestUploadEnabledLaterStillUploads(t *testing.T) {
dir := t.TempDir()
logPath := filepath.Join(dir, "runtime.jsonl")
statePath := filepath.Join(dir, "state.json")
client := pullTestServer(t)
// First sweep with upload disabled: local sync only.
if _, err := PullOnce(context.Background(), client, PullOptions{Write: true, LogPath: logPath, StatePath: statePath}); err != nil {
t.Fatalf("first (no upload): %v", err)
}
// Second sweep, session unchanged, upload now enabled: must NOT be skipped.
up := &fakeUploader{}
if _, err := PullOnce(context.Background(), client, PullOptions{
Write: true, LogPath: logPath, StatePath: statePath, Upload: up, UploadPrefix: "agent-traces",
}); err != nil {
t.Fatalf("second (upload on): %v", err)
}
if up.calls != 1 {
t.Fatalf("expected unchanged session to upload once upload was enabled, got %d uploads", up.calls)
}
// Third sweep, still unchanged and already uploaded: now skipped (no re-upload).
up2 := &fakeUploader{}
if _, err := PullOnce(context.Background(), client, PullOptions{
Write: true, LogPath: logPath, StatePath: statePath, Upload: up2, UploadPrefix: "agent-traces",
}); err != nil {
t.Fatalf("third: %v", err)
}
if up2.calls != 0 {
t.Fatalf("expected no re-upload once UploadedAt matches updated_at, got %d", up2.calls)
}
}
type flakyUploader struct {
failFirst int
calls int
}
func (f *flakyUploader) Upload(_ context.Context, _ string, _ []byte) error {
f.calls++
if f.calls <= f.failFirst {
return fmt.Errorf("transient upload failure")
}
return nil
}
func TestPartialFailureRetriedThenSkipped(t *testing.T) {
dir := t.TempDir()
logPath := filepath.Join(dir, "runtime.jsonl")
statePath := filepath.Join(dir, "state.json")
client := pullTestServer(t) // one suspended session, one user message
up := &flakyUploader{failFirst: 1}
opts := PullOptions{Write: true, LogPath: logPath, StatePath: statePath, Upload: up, UploadPrefix: "agent-traces"}
// Sweep 1: local emit succeeds, upload fails -> session left unsynced.
if _, err := PullOnce(context.Background(), client, opts); err == nil {
t.Fatal("expected upload error on first sweep")
}
lines1 := len(nonEmptyLines(t, logPath))
// Sweep 2: not skipped (SyncedAt never advanced); reprocess + upload succeeds,
// with no duplicate local lines.
sum2, err := PullOnce(context.Background(), client, opts)
if err != nil {
t.Fatalf("second sweep: %v", err)
}
if sum2.SessionsChanged != 1 || sum2.Uploaded != 1 {
t.Fatalf("partial-failed session should retry+upload, got %+v", sum2)
}
if got := len(nonEmptyLines(t, logPath)); got != lines1 {
t.Fatalf("retry duplicated local lines: %d -> %d", lines1, got)
}
// Sweep 3: now fully synced and uploaded -> skipped.
sum3, err := PullOnce(context.Background(), client, opts)
if err != nil {
t.Fatalf("third sweep: %v", err)
}
if sum3.SessionsChanged != 0 {
t.Fatalf("fully synced session should be skipped, got %+v", sum3)
}
}
func pullTestServer(t *testing.T) *Client {
srv := fakeDevinAPI(t, `{"items":[
{"session_id":"s1","status":"suspended","user_id":"user-1","created_at":1000,"updated_at":1600,"acus_consumed":1}
],"has_next_page":false,"total":1}`, map[string]string{
"s1": `{"items":[
{"event_id":"e1","source":"user","message":"prompt one","created_at":1000},
{"event_id":"e2","source":"devin","message":"reply one","created_at":1100}
],"has_next_page":false}`,
})
t.Cleanup(srv.Close)
return testClient(t, srv)
}
func TestPullOnceWritesAndUploads(t *testing.T) {
dir := t.TempDir()
logPath := filepath.Join(dir, "runtime.jsonl")
statePath := filepath.Join(dir, "state.json")
up := &fakeUploader{}
client := pullTestServer(t)
sum, err := PullOnce(context.Background(), client, PullOptions{
Write: true,
LogPath: logPath,
StatePath: statePath,
Upload: up,
UploadPrefix: "agent-traces",
})
if err != nil {
t.Fatalf("PullOnce: %v", err)
}
// started + prompt + agent + ended = 4
if sum.EventsEmitted != 4 || sum.SessionsChanged != 1 || sum.Uploaded != 1 {
t.Fatalf("summary = %+v, want 4 events / 1 changed / 1 uploaded", sum)
}
lines := nonEmptyLines(t, logPath)
if len(lines) != 4 {
t.Fatalf("log has %d lines, want 4", len(lines))
}
// The messages API's own field names reach the log: source decides prompt vs reply, and the
// text lands on each. A drifted field tag still yields four lines, all of them agent.message.
got := map[string]string{}
for _, line := range lines {
var ev struct {
Event struct{ Action string } `json:"event"`
Message string `json:"message"`
Prompt *struct{ Text string } `json:"prompt"`
}
if err := json.Unmarshal([]byte(line), &ev); err != nil {
t.Fatalf("log line is not an event: %v", err)
}
text := ev.Message
if ev.Prompt != nil {
text = ev.Prompt.Text
}
got[ev.Event.Action] = text
}
if got["prompt.submitted"] != "prompt one" || got["agent.message"] != "reply one" {
t.Fatalf("log actions = %q, want the user message as a prompt and Devin's as a reply", got)
}
wantObj := "agent-traces/provider=devin_cloud/user_id=user-1/run_id=s1/runtime.jsonl"
body, ok := up.objects[wantObj]
if !ok {
t.Fatalf("uploaded object %q missing; have %v", wantObj, keys(up.objects))
}
if got := len(nonEmptyLinesBytes(body)); got != 4 {
t.Fatalf("uploaded object has %d lines, want 4", got)
}
}
func TestUploadSnapshotIsRedacted(t *testing.T) {
dir := t.TempDir()
up := &fakeUploader{}
srv := fakeDevinAPI(t, `{"items":[
{"session_id":"s1","status":"suspended","user_id":"user-1","created_at":1000,"updated_at":1600}
],"has_next_page":false,"total":1}`, map[string]string{
"s1": `{"items":[
{"event_id":"e1","source":"user","message":"deploy with token=supersecretvalue123456","created_at":1000}
],"has_next_page":false}`,
})
defer srv.Close()
_, err := PullOnce(context.Background(), testClient(t, srv), PullOptions{
Write: true,
LogPath: filepath.Join(dir, "runtime.jsonl"),
StatePath: filepath.Join(dir, "state.json"),
Upload: up,
UploadPrefix: "agent-traces",
})
if err != nil {
t.Fatalf("PullOnce: %v", err)
}
for obj, body := range up.objects {
if strings.Contains(string(body), "supersecretvalue123456") {
t.Fatalf("uploaded object %q leaked an unredacted secret:\n%s", obj, body)
}
}
}
// A key the API refuses must fail the pull. The 403 body is JSON, so a client that decoded it as
// a page of sessions would report an org with no sessions and exit 0 on a bad credential.
func TestPullOnceWithARejectedKeyFailsAndWritesNothing(t *testing.T) {
srv := fakeDevinAPI(t, `{"items":[{"session_id":"s1","status":"finished","created_at":1,"updated_at":2}],"has_next_page":false}`, nil)
defer srv.Close()
dir := t.TempDir()
logPath := filepath.Join(dir, "runtime.jsonl")
statePath := filepath.Join(dir, "state.json")
client := New(testOrg, "cog_wrong", WithBaseURL(srv.URL), WithRetry(0, 0))
_, err := PullOnce(context.Background(), client, PullOptions{Write: true, LogPath: logPath, StatePath: statePath})
if err == nil || !strings.Contains(err.Error(), "403") {
t.Fatalf("PullOnce with a rejected key = %v, want the 403", err)
}
if _, statErr := os.Stat(logPath); !os.IsNotExist(statErr) {
t.Fatalf("a refused pull wrote the runtime log: %v", statErr)
}
}
func TestPullOncePersistsStateOnError(t *testing.T) {
dir := t.TempDir()
logPath := filepath.Join(dir, "runtime.jsonl")
statePath := filepath.Join(dir, "state.json")
var s2Calls int
mux := http.NewServeMux()
mux.HandleFunc("/v3/organizations/"+testOrg+"/sessions", func(w http.ResponseWriter, _ *http.Request) {
_, _ = w.Write([]byte(`{"items":[
{"session_id":"s1","status":"suspended","user_id":"u","created_at":10,"updated_at":20},
{"session_id":"s2","status":"suspended","user_id":"u","created_at":10,"updated_at":20}
],"has_next_page":false,"total":2}`))
})
mux.HandleFunc("/v3/organizations/"+testOrg+"/sessions/s1/messages", func(w http.ResponseWriter, _ *http.Request) {
_, _ = w.Write([]byte(`{"items":[{"event_id":"e1","source":"user","message":"hi","created_at":10}],"has_next_page":false}`))
})
mux.HandleFunc("/v3/organizations/"+testOrg+"/sessions/s2/messages", func(w http.ResponseWriter, _ *http.Request) {
s2Calls++
if s2Calls == 1 {
http.Error(w, `{"detail":"boom"}`, http.StatusInternalServerError)
return
}
_, _ = w.Write([]byte(`{"items":[{"event_id":"e2","source":"user","message":"yo","created_at":10}],"has_next_page":false}`))
})
srv := httptest.NewServer(mux)
defer srv.Close()
client := New(testOrg, "cog_test", WithBaseURL(srv.URL), WithRetry(0, 0))
opts := PullOptions{Write: true, LogPath: logPath, StatePath: statePath}
// First sweep: s1 succeeds and is written; s2 errors -> PullOnce returns error.
if _, err := PullOnce(context.Background(), client, opts); err == nil {
t.Fatal("expected error from failing session")
}
afterFirst := len(nonEmptyLines(t, logPath)) // s1: started + prompt + ended = 3
// Second sweep: s1 must be skipped (state persisted), s2 now succeeds.
if _, err := PullOnce(context.Background(), client, opts); err != nil {
t.Fatalf("second sweep: %v", err)
}
total := len(nonEmptyLines(t, logPath))
added := total - afterFirst
if added != 3 { // only s2's started + prompt + ended; s1 not re-emitted
t.Fatalf("second sweep added %d lines, want 3 (s1 not duplicated)", added)
}
}
func TestOneSessionErrorDoesNotBlockOthers(t *testing.T) {
dir := t.TempDir()
logPath := filepath.Join(dir, "runtime.jsonl")
mux := http.NewServeMux()
mux.HandleFunc("/v3/organizations/"+testOrg+"/sessions", func(w http.ResponseWriter, _ *http.Request) {
// "bad" sorts before "good"; bad fails, good must still be processed.
_, _ = w.Write([]byte(`{"items":[
{"session_id":"bad","status":"suspended","user_id":"u","created_at":10,"updated_at":20},
{"session_id":"good","status":"suspended","user_id":"u","created_at":10,"updated_at":20}
],"has_next_page":false,"total":2}`))
})
mux.HandleFunc("/v3/organizations/"+testOrg+"/sessions/bad/messages", func(w http.ResponseWriter, _ *http.Request) {
http.Error(w, `{"detail":"always fails"}`, http.StatusInternalServerError)
})
mux.HandleFunc("/v3/organizations/"+testOrg+"/sessions/good/messages", func(w http.ResponseWriter, _ *http.Request) {
_, _ = w.Write([]byte(`{"items":[{"event_id":"g1","source":"user","message":"hi","created_at":10}],"has_next_page":false}`))
})
srv := httptest.NewServer(mux)
defer srv.Close()
client := New(testOrg, "cog_test", WithBaseURL(srv.URL), WithRetry(0, 0))
sum, err := PullOnce(context.Background(), client, PullOptions{
Write: true, LogPath: logPath, StatePath: filepath.Join(dir, "state.json"),
})
if err == nil {
t.Fatal("expected an aggregated error from the failing session")
}
if sum.Errors != 1 {
t.Fatalf("Errors = %d, want 1", sum.Errors)
}
// The healthy session must still have been ingested despite the bad one.
lines := nonEmptyLines(t, logPath)
if len(lines) != 3 { // good: started + prompt + ended
t.Fatalf("healthy session not ingested: %d lines, want 3", len(lines))
}
for _, l := range lines {
if !strings.Contains(l, `"run_id":"good"`) {
t.Fatalf("unexpected event from non-good session: %s", l)
}
}
}
func TestForceRefreshReFetchesWithoutDuplicating(t *testing.T) {
dir := t.TempDir()
logPath := filepath.Join(dir, "runtime.jsonl")
statePath := filepath.Join(dir, "state.json")
var msgCalls int
mux := http.NewServeMux()
mux.HandleFunc("/v3/organizations/"+testOrg+"/sessions", func(w http.ResponseWriter, _ *http.Request) {
_, _ = w.Write([]byte(`{"items":[{"session_id":"s1","status":"suspended","user_id":"u","created_at":10,"updated_at":20}],"has_next_page":false,"total":1}`))
})
mux.HandleFunc("/v3/organizations/"+testOrg+"/sessions/s1/messages", func(w http.ResponseWriter, _ *http.Request) {
msgCalls++
_, _ = w.Write([]byte(`{"items":[{"event_id":"e1","source":"user","message":"hi","created_at":10}],"has_next_page":false}`))
})
srv := httptest.NewServer(mux)
defer srv.Close()
client := New(testOrg, "cog_test", WithBaseURL(srv.URL), WithRetry(0, 0))
base := PullOptions{Write: true, LogPath: logPath, StatePath: statePath}
if _, err := PullOnce(context.Background(), client, base); err != nil {
t.Fatalf("first: %v", err)
}
// Without force, an unchanged session is skipped (no second message fetch).
if _, err := PullOnce(context.Background(), client, base); err != nil {
t.Fatalf("second: %v", err)
}
if msgCalls != 1 {
t.Fatalf("expected 1 message fetch (skip on unchanged), got %d", msgCalls)
}
// With force, the unchanged session is re-fetched, but dedup prevents new log lines.
before := len(nonEmptyLines(t, logPath))
forced := base
forced.ForceRefresh = true
if _, err := PullOnce(context.Background(), client, forced); err != nil {
t.Fatalf("forced: %v", err)
}
if msgCalls != 2 {
t.Fatalf("expected re-fetch with ForceRefresh, msgCalls=%d", msgCalls)
}
if got := len(nonEmptyLines(t, logPath)); got != before {
t.Fatalf("force-resync duplicated log lines: %d -> %d", before, got)
}
}
func TestEndedEmittedOncePerObservedEpisode(t *testing.T) {
dir := t.TempDir()
logPath := filepath.Join(dir, "runtime.jsonl")
current := ""
mux := http.NewServeMux()
mux.HandleFunc("/v3/organizations/"+testOrg+"/sessions", func(w http.ResponseWriter, _ *http.Request) {
_, _ = w.Write([]byte(current))
})
mux.HandleFunc("/v3/organizations/"+testOrg+"/sessions/s1/messages", func(w http.ResponseWriter, _ *http.Request) {
_, _ = w.Write([]byte(`{"items":[{"event_id":"e1","source":"user","message":"hi","created_at":10}],"has_next_page":false}`))
})
srv := httptest.NewServer(mux)
defer srv.Close()
client := New(testOrg, "cog_test", WithBaseURL(srv.URL), WithRetry(0, 0))
opts := PullOptions{Write: true, LogPath: logPath, StatePath: filepath.Join(dir, "state.json")}
sess := func(status string, updated int) string {
return fmt.Sprintf(`{"items":[{"session_id":"s1","status":%q,"user_id":"u","created_at":10,"updated_at":%d}],"has_next_page":false,"total":1}`, status, updated)
}
sweep := func() {
if _, err := PullOnce(context.Background(), client, opts); err != nil {
t.Fatalf("sweep: %v", err)
}
}
current = sess("suspended", 20)
sweep() // end #1
current = sess("suspended", 30)
sweep() // updated_at bump, no observed resume -> NO new end
current = sess("working", 40)
sweep() // resumed -> reset
current = sess("suspended", 50)
sweep() // end #2
ends := 0
for _, l := range nonEmptyLines(t, logPath) {
if strings.Contains(l, `"action":"session.ended"`) {
ends++
}
}
if ends != 2 {
t.Fatalf("session.ended count = %d, want 2 (one per observed terminal episode)", ends)
}
}
func nonEmptyLines(t *testing.T, path string) []string {
t.Helper()
data, err := os.ReadFile(path)
if err != nil {
t.Fatalf("read %s: %v", path, err)
}
return nonEmptyLinesBytes(data)
}
func nonEmptyLinesBytes(data []byte) []string {
var out []string
for _, l := range strings.Split(string(data), "\n") {
if strings.TrimSpace(l) != "" {
out = append(out, l)
}
}
return out
}
func keys(m map[string][]byte) []string {
var out []string
for k := range m {
out = append(out, k)
}
return out
}