The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu
Go agent-ai-cliagent-harnessesagent-securitybrowser-extensionciclaude-codecloudcodexcursordetection-engineeringjevknowledge-basememorymobile-device-managementobservabilitysecuritysecurity-information-and-event-managementsecurity-toolstelemetrytraces
cursorusage: map Admin API events to token.usage and collect idempotently
Claudee9fc0be0d51c
diff --git a/cli/beacon/internal/cursorusage/collect.go b/cli/beacon/internal/cursorusage/collect.gonew file mode 100644index 00000000..1e6ad3ca--- /dev/null+++ b/cli/beacon/internal/cursorusage/collect.go@@ -0,0 +1,414 @@+package cursorusage++import (+ "context"+ "encoding/json"+ "errors"+ "fmt"+ "io"+ "os"+ "path/filepath"+ "sort"+ "strings"+ "time"++ "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/schema"+ "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/writer"+)++// StateVersion is the on-disk version of the collector state. A file of another version is+// discarded: the cost is one lookback window fetched again and suppressed by the fresh state's+// first-run window, while misreading a cursor would skip or repeat spend.+const StateVersion = 1++const (+ // DefaultLookback is how far behind the last sync each sync re-reads. Cursor posts some usage+ // events after the request that caused them, so a window that started exactly at the last+ // sync's end would miss a late event for good. Everything re-read is suppressed by key.+ DefaultLookback = 48 * time.Hour+ // DefaultInitialWindow is how far back a first sync reads when no start is given.+ DefaultInitialWindow = 7 * 24 * time.Hour+ // MaxWindow is the longest window one query asks for. Longer ranges are split, so a backfill+ // over months is a series of ordinary requests rather than one the API may refuse.+ MaxWindow = 30 * 24 * time.Hour+)++// Cursor is what the collector remembers about one scope between syncs.+//+// CoveredFromMS and HighWaterMS bound the span already collected. A sync fetches only what falls+// before CoveredFromMS (a backfill reaching further back than any earlier sync) and the lookback+// window behind HighWaterMS, and writes what is not already in Seen; the span between the two is+// already in the log and is never fetched again (see segments). Seen holds only the keys inside the lookback window, so the file stays the+// size of two days of requests however long the collector has run.+type Cursor struct {+ CoveredFromMS int64 `json:"covered_from_ms,omitempty"`+ HighWaterMS int64 `json:"high_water_ms,omitempty"`+ // SeenFromMS is where Seen starts being complete: every event collected at or after it has+ // its key in Seen. It is persisted rather than recomputed from the lookback, because a sync+ // run with a longer lookback than the last one would otherwise treat a span whose keys were+ // already pruned as never collected and write it again.+ SeenFromMS int64 `json:"seen_from_ms,omitempty"`+ Seen map[string]int64 `json:"seen,omitempty"`+ LastSyncMS int64 `json:"last_sync_ms,omitempty"`+}++// State is the collector's file: one cursor per scope. A scope is the filter the sync ran with+// (one member's email, one user id, or the whole team), because a cursor earned for one member+// says nothing about what has been collected for another.+type State struct {+ Version int `json:"version"`+ Scopes map[string]*Cursor `json:"scopes"`+}++// ScopeKey names the cursor a query uses.+func ScopeKey(q Query) string {+ if email := strings.ToLower(strings.TrimSpace(q.Email)); email != "" {+ return "email:" + email+ }+ if id := strings.TrimSpace(q.UserID); id != "" {+ return "user:" + id+ }+ return "team"+}++// LoadState reads the state file. A missing file or an empty path is an empty state.+func LoadState(path string) (*State, error) {+ state := &State{Version: StateVersion, Scopes: map[string]*Cursor{}}+ if path == "" {+ return state, nil+ }+ data, err := os.ReadFile(path)+ if err != nil {+ if errors.Is(err, os.ErrNotExist) {+ return state, nil+ }+ return nil, err+ }+ var stored State+ if err := json.Unmarshal(data, &stored); err != nil {+ return nil, fmt.Errorf("read Cursor usage collector state %s: %w", path, err)+ }+ if stored.Version != StateVersion || stored.Scopes == nil {+ return state, nil+ }+ state.Scopes = stored.Scopes+ return state, nil+}++// Save writes the state atomically. An empty path is a no-op.+func (s *State) Save(path string) error {+ if path == "" {+ return nil+ }+ if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {+ return err+ }+ s.Version = StateVersion+ data, err := json.MarshalIndent(s, "", " ")+ if err != nil {+ return err+ }+ tmp := path + ".tmp"+ if err := os.WriteFile(tmp, data, 0o600); err != nil {+ return err+ }+ return os.Rename(tmp, path)+}++// Cursor returns the scope's cursor, creating it.+func (s *State) Cursor(scope string) *Cursor {+ if s.Scopes == nil {+ s.Scopes = map[string]*Cursor{}+ }+ c := s.Scopes[scope]+ if c == nil {+ c = &Cursor{}+ s.Scopes[scope] = c+ }+ if c.Seen == nil {+ c.Seen = map[string]int64{}+ }+ return c+}++// DefaultStatePath is where the collector keeps its state for the current user. It holds event+// fingerprints and timestamps, never the key.+func DefaultStatePath() string {+ base := filepath.Join(os.TempDir(), "beacon")+ if home, err := os.UserHomeDir(); err == nil && strings.TrimSpace(home) != "" {+ base = filepath.Join(home, ".beacon")+ }+ return filepath.Join(base, "endpoint", "state", "cursor-usage.json")+}++// Fetcher is the part of Client the collector uses.+type Fetcher interface {+ FetchAll(ctx context.Context, q Query) ([]UsageEvent, error)+}++// CollectOptions configures one sync.+type CollectOptions struct {+ Fetcher Fetcher+ // Query carries the member filter and page size. Its dates are ignored; the collector chooses+ // the window.+ Query Query+ // Since forces the window to start no later than this. Zero means: from the cursor, or+ // DefaultInitialWindow back on a first sync.+ Since time.Time+ // Lookback overrides DefaultLookback.+ Lookback time.Duration+ // Now overrides the clock.+ Now func() time.Time++ // StatePath is the state file. Empty disables persistence.+ StatePath string+ // Write appends events to the runtime log at LogPath.+ Write bool+ LogPath string+ UserMode bool+ // Print writes each event to Out as JSON.+ Print bool+ Out io.Writer++ // emitHook replaces the writer in tests.+ emitHook func(schema.Event) error+}++// Summary reports what a sync did.+type Summary struct {+ Scope string `json:"scope"`+ WindowStart string `json:"window_start"`+ WindowEnd string `json:"window_end"`+ Fetched int `json:"fetched"`+ Emitted int `json:"emitted"`+ // Duplicates were already collected by an earlier sync.+ Duplicates int `json:"duplicates"`+ // NoUsage were requests with no tokens and no charge.+ NoUsage int `json:"no_usage"`+}++// Window returns the span a sync over the cursor would query.+func Window(c *Cursor, since, now time.Time, lookback time.Duration) (time.Time, time.Time) {+ if lookback <= 0 {+ lookback = DefaultLookback+ }+ var start time.Time+ if c != nil && c.HighWaterMS > 0 {+ start = time.UnixMilli(c.HighWaterMS).Add(-lookback)+ } else {+ start = now.Add(-DefaultInitialWindow)+ }+ if !since.IsZero() && (c == nil || c.HighWaterMS == 0 || since.Before(start)) {+ start = since+ }+ if start.After(now) {+ start = now+ }+ // Millisecond edges, because that is the API's resolution: a window edge between two+ // milliseconds would let adjacent chunks leave an event in neither.+ return start.UTC().Truncate(time.Millisecond), now.UTC().Truncate(time.Millisecond)+}++// CollectOnce runs one sync: fetch the window, drop what an earlier sync already wrote, write the+// rest oldest first, and advance the cursor.+//+// The cursor's high-water mark moves only after every event in the window was written. A sync+// that fails partway keeps the keys of the events it did write, so the retry skips them and+// writes the rest.+func CollectOnce(ctx context.Context, opts CollectOptions) (summary Summary, err error) {+ if opts.Fetcher == nil {+ return summary, errors.New("no Cursor usage fetcher")+ }+ now := time.Now+ if opts.Now != nil {+ now = opts.Now+ }+ lookback := opts.Lookback+ if lookback <= 0 {+ lookback = DefaultLookback+ }++ state, err := LoadState(opts.StatePath)+ if err != nil {+ return summary, err+ }+ scope := ScopeKey(opts.Query)+ cursor := state.Cursor(scope)+ summary.Scope = scope++ end := now().UTC()+ start, end := Window(cursor, opts.Since, end, lookback)+ summary.WindowStart = start.Format(time.RFC3339)+ summary.WindowEnd = end.Format(time.RFC3339)++ persist := func() {+ if saveErr := state.Save(opts.StatePath); saveErr != nil && err == nil {+ err = fmt.Errorf("save Cursor usage collector state: %w", saveErr)+ }+ }++ var events []UsageEvent+ seenFrom := cursor.seenFrom(lookback)+ for _, segment := range segments(cursor, start, end, seenFrom) {+ got, fetchErr := fetchSegment(ctx, opts.Fetcher, opts.Query, segment[0], segment[1])+ if fetchErr != nil {+ // Nothing has been written yet, so nothing is lost by stopping here: the next sync+ // asks for the same window.+ return summary, fetchErr+ }+ events = append(events, got...)+ }+ events = inWindow(events, start, end)+ summary.Fetched = len(events)++ sort.SliceStable(events, func(i, j int) bool {+ if !events[i].Timestamp.Equal(events[j].Timestamp) {+ return events[i].Timestamp.Before(events[j].Timestamp)+ }+ return Fingerprint(events[i]) < Fingerprint(events[j])+ })+ keys := DedupKeys(events)++ for i, ev := range events {+ ts := ev.Timestamp.UnixMilli()+ key := keys[i]+ if _, ok := cursor.Seen[key]; ok {+ summary.Duplicates+++ continue+ }+ if !HasUsage(ev) {+ summary.NoUsage+++ if !opts.Print {+ cursor.Seen[key] = ts+ }+ continue+ }+ if emitErr := emit(MapEvent(ev), opts); emitErr != nil {+ persist()+ return summary, emitErr+ }+ summary.Emitted+++ if !opts.Print {+ cursor.Seen[key] = ts+ }+ }++ if opts.Print {+ return summary, nil+ }+ if cursor.CoveredFromMS == 0 || start.UnixMilli() < cursor.CoveredFromMS {+ cursor.CoveredFromMS = start.UnixMilli()+ }+ if end.UnixMilli() > cursor.HighWaterMS {+ cursor.HighWaterMS = end.UnixMilli()+ }+ cursor.LastSyncMS = end.UnixMilli()+ prune := time.UnixMilli(cursor.HighWaterMS).Add(-lookback).UnixMilli()+ // The completeness boundary never moves back: keys behind it are already gone.+ if prune < cursor.SeenFromMS {+ prune = cursor.SeenFromMS+ }+ for key, ts := range cursor.Seen {+ if ts < prune {+ delete(cursor.Seen, key)+ }+ }+ cursor.SeenFromMS = prune+ persist()+ return summary, err+}++// segments lists the spans a sync has to fetch. Normally that is the whole window. A backfill+// that reaches before anything collected so far skips the span earlier syncs already read+// completely: fetching it again would only produce events the cursor then discards.+func segments(c *Cursor, start, end, seenFrom time.Time) [][2]time.Time {+ if c == nil || c.HighWaterMS == 0 || c.CoveredFromMS == 0 {+ return [][2]time.Time{{start, end}}+ }+ covered := time.UnixMilli(c.CoveredFromMS).UTC()+ recent := seenFrom+ if recent.Before(covered) {+ // Seen reaches past everything collected: there is no fully read middle span.+ recent = covered+ }+ tail := recent+ if tail.Before(start) {+ tail = start+ }+ if !start.Before(covered) {+ return [][2]time.Time{{tail, end}}+ }+ return [][2]time.Time{+ {start, covered.Add(-time.Millisecond)},+ {tail, end},+ }+}++// seenFrom is where the cursor's Seen set is complete. A state written before SeenFromMS existed+// falls back to the lookback behind the high-water mark, which is what it pruned to.+func (c *Cursor) seenFrom(lookback time.Duration) time.Time {+ if c.SeenFromMS > 0 {+ return time.UnixMilli(c.SeenFromMS).UTC()+ }+ if c.HighWaterMS > 0 {+ return time.UnixMilli(c.HighWaterMS).Add(-lookback).UTC()+ }+ return time.Time{}+}++// fetchSegment fetches [start, end] in pieces no longer than MaxWindow. The pieces do not share a+// millisecond: the API's end date is inclusive, and an event on a shared boundary would come back+// in both pieces and read as two requests.+func fetchSegment(ctx context.Context, f Fetcher, base Query, start, end time.Time) ([]UsageEvent, error) {+ var out []UsageEvent+ for chunkStart := start; !chunkStart.After(end); {+ chunkEnd := chunkStart.Add(MaxWindow)+ if chunkEnd.After(end) {+ chunkEnd = end+ }+ q := base+ q.StartDate = chunkStart+ q.EndDate = chunkEnd+ got, err := f.FetchAll(ctx, q)+ if err != nil {+ return out, err+ }+ out = append(out, inWindow(got, chunkStart, chunkEnd)...)+ chunkStart = chunkEnd.Add(time.Millisecond)+ }+ return out, nil+}++// inWindow drops events outside [start, end]. A host that ignores the requested dates would+// otherwise put events outside the window the cursor accounts for.+func inWindow(events []UsageEvent, start, end time.Time) []UsageEvent {+ out := events[:0]+ for _, ev := range events {+ if ev.Timestamp.Before(start) || ev.Timestamp.After(end) {+ continue+ }+ out = append(out, ev)+ }+ return out+}++func emit(event schema.Event, opts CollectOptions) error {+ if opts.emitHook != nil {+ return opts.emitHook(event)+ }+ if opts.Print && opts.Out != nil {+ data, err := json.Marshal(event)+ if err != nil {+ return err+ }+ if _, err := opts.Out.Write(append(data, '\n')); err != nil {+ return err+ }+ }+ if opts.Write && !opts.Print {+ if _, err := writer.AppendEvent(event, writer.Options{Path: opts.LogPath, UserMode: opts.UserMode}); err != nil {+ return err+ }+ }+ return nil+}diff --git a/cli/beacon/internal/cursorusage/collect_test.go b/cli/beacon/internal/cursorusage/collect_test.gonew file mode 100644index 00000000..cef55518--- /dev/null+++ b/cli/beacon/internal/cursorusage/collect_test.go@@ -0,0 +1,613 @@+package cursorusage++import (+ "bufio"+ "bytes"+ "context"+ "encoding/json"+ "errors"+ "os"+ "path/filepath"+ "testing"+ "time"++ "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/schema"+ "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/tokens"+)++// fakeFetcher serves a fixed set of events the way the API does: everything whose timestamp falls+// inside the inclusive [startDate, endDate] window of the query.+type fakeFetcher struct {+ events []UsageEvent+ queries []Query+ err error+ // ignoreDates makes the fake return every event regardless of the window.+ ignoreDates bool+}++func (f *fakeFetcher) FetchAll(_ context.Context, q Query) ([]UsageEvent, error) {+ f.queries = append(f.queries, q)+ if f.err != nil {+ return nil, f.err+ }+ var out []UsageEvent+ for _, ev := range f.events {+ if f.ignoreDates || (!ev.Timestamp.Before(q.StartDate) && !ev.Timestamp.After(q.EndDate)) {+ out = append(out, ev)+ }+ }+ return out, nil+}++var baseNow = time.Date(2026, 10, 5, 12, 0, 0, 0, time.UTC)++func f64(v float64) *float64 { return &v }++func usageAt(ts time.Time, input int64) UsageEvent {+ return UsageEvent{+ Timestamp: ts,+ Model: "claude-4.5-sonnet",+ Kind: "USAGE_EVENT_KIND_USAGE_BASED",+ TokenBased: true,+ TokenUsage: TokenUsage{InputTokens: input, OutputTokens: 10, CacheReadTokens: 100, CacheWriteTokens: 5, TotalCents: f64(1.5)},+ ChargedCents: f64(2),+ UserEmail: "[email protected]",+ UserID: "42",+ }+}++type harness struct {+ t *testing.T+ fetcher *fakeFetcher+ now time.Time+ state string+ log string+ query Query+ since time.Time+ lookback time.Duration+}++func newHarness(t *testing.T, events ...UsageEvent) *harness {+ dir := t.TempDir()+ return &harness{+ t: t,+ fetcher: &fakeFetcher{events: events},+ now: baseNow,+ state: filepath.Join(dir, "state", "cursor-usage.json"),+ log: filepath.Join(dir, "logs", "runtime.jsonl"),+ query: Query{Email: "[email protected]"},+ }+}++func (h *harness) sync() Summary {+ h.t.Helper()+ summary, err := CollectOnce(context.Background(), h.opts())+ if err != nil {+ h.t.Fatal(err)+ }+ return summary+}++func (h *harness) opts() CollectOptions {+ return CollectOptions{+ Fetcher: h.fetcher,+ Query: h.query,+ Since: h.since,+ Lookback: h.lookback,+ Now: func() time.Time { return h.now },+ StatePath: h.state,+ Write: true,+ LogPath: h.log,+ }+}++func (h *harness) logged() []schema.Event {+ h.t.Helper()+ f, err := os.Open(h.log)+ if errors.Is(err, os.ErrNotExist) {+ return nil+ }+ if err != nil {+ h.t.Fatal(err)+ }+ defer f.Close()+ var out []schema.Event+ scanner := bufio.NewScanner(f)+ for scanner.Scan() {+ var ev schema.Event+ if err := json.Unmarshal(scanner.Bytes(), &ev); err != nil {+ h.t.Fatalf("log line is not an event: %v", err)+ }+ out = append(out, ev)+ }+ return out+}++func TestMapEvent(t *testing.T) {+ ev := usageAt(time.UnixMilli(1750979225854).UTC(), 126)+ ev.CursorTokenFeeCents = f64(0.25)+ ev.Chargeable = new(bool)+ *ev.Chargeable = true+ ev.MaxMode = true+ out := MapEvent(ev)++ if err := out.Validate(); err != nil {+ t.Fatal(err)+ }+ if out.Event.Action != "token.usage" || out.Event.Category != "metric" || out.Event.Fidelity != schema.FidelityObserved {+ t.Errorf("event = %+v", out.Event)+ }+ if out.Harness.Name != "cursor" || out.Harness.CollectionMethod != schema.CollectionMethodPoll {+ t.Errorf("harness = %+v", out.Harness)+ }+ if out.Session != nil {+ t.Errorf("Admin API events name no conversation; session must be unset, got %+v", out.Session)+ }+ if out.Timestamp != schema.FormatTimestamp(ev.Timestamp) || out.Model != "claude-4.5-sonnet" {+ t.Errorf("timestamp/model = %s %s", out.Timestamp, out.Model)+ }+ u := out.GenAI.Usage+ if *u.InputTokens != 126 || *u.OutputTokens != 10 || *u.CacheRead.InputTokens != 100 || *u.CacheCreation.InputTokens != 5 {+ t.Errorf("usage = %+v", u)+ }+ if u.CostUSD == nil || *u.CostUSD != 0.02 {+ t.Errorf("cost = %v, want chargedCents/100", u.CostUSD)+ }+ raw := out.Raw["cursor"].(map[string]interface{})+ for key, want := range map[string]interface{}{+ "source": "admin_api", "token_cents": 1.5, "charged_cents": 2.0, "cursor_token_fee_cents": 0.25,+ "chargeable": true, "max_mode": true, "user_email": "[email protected]", "user_id": "42",+ } {+ if raw[key] != want {+ t.Errorf("raw.cursor.%s = %v, want %v", key, raw[key], want)+ }+ }+}++func TestMapEventLeavesAbsentValuesAbsent(t *testing.T) {+ out := MapEvent(UsageEvent{Timestamp: baseNow, Model: "auto", TokenUsage: TokenUsage{OutputTokens: 3}})+ u := out.GenAI.Usage+ if u.InputTokens != nil || u.CacheRead != nil || u.CacheCreation != nil || u.CostUSD != nil {+ t.Errorf("zero counts and an absent charge must not be written: %+v", u)+ }+ if *u.OutputTokens != 3 {+ t.Errorf("output = %v", u.OutputTokens)+ }+}++func TestHasUsage(t *testing.T) {+ if HasUsage(UsageEvent{}) {+ t.Error("an empty event has no usage")+ }+ if HasUsage(UsageEvent{ChargedCents: f64(0), RequestsCosts: f64(1)}) {+ t.Error("a request-billed call with a zero charge has nothing to sum")+ }+ if !HasUsage(UsageEvent{ChargedCents: f64(4)}) {+ t.Error("a charge alone is usage")+ }+ if !HasUsage(UsageEvent{TokenUsage: TokenUsage{CacheReadTokens: 1}}) {+ t.Error("cache tokens alone are usage")+ }+}++func TestDedupKeysSeparateIdenticalRequests(t *testing.T) {+ a := usageAt(baseNow, 1)+ b := usageAt(baseNow, 2)+ keys := DedupKeys([]UsageEvent{a, b, a})+ if keys[0] == keys[2] {+ t.Fatal("two identical requests must get two keys")+ }+ if keys[0] == keys[1] {+ t.Fatal("different requests must get different keys")+ }+ // Order among non-identical events does not change any key.+ again := DedupKeys([]UsageEvent{b, a, a})+ if again[0] != keys[1] || again[1] != keys[0] || again[2] != keys[2] {+ t.Fatalf("keys depend on order: %v vs %v", keys, again)+ }+}++func TestFingerprintDistinguishesAbsentFromZeroCharge(t *testing.T) {+ a := usageAt(baseNow, 1)+ b := a+ b.ChargedCents = nil+ if Fingerprint(a) == Fingerprint(b) {+ t.Fatal("an absent charge and a charge must fingerprint differently")+ }+}++func TestFirstSyncWritesTheInitialWindowAndARepeatWritesNothing(t *testing.T) {+ h := newHarness(t,+ usageAt(baseNow.Add(-8*24*time.Hour), 999), // before the default first window+ usageAt(baseNow.Add(-6*24*time.Hour), 1),+ usageAt(baseNow.Add(-time.Hour), 2),+ UsageEvent{Timestamp: baseNow.Add(-2 * time.Hour), Model: "auto", RequestsCosts: f64(1)},+ )+ first := h.sync()+ if first.Fetched != 3 || first.Emitted != 2 || first.NoUsage != 1 || first.Duplicates != 0 {+ t.Fatalf("first = %+v", first)+ }+ if first.Scope != "email:[email protected]" {+ t.Fatalf("scope = %q", first.Scope)+ }+ logged := h.logged()+ if len(logged) != 2 {+ t.Fatalf("logged %d events", len(logged))+ }+ // Oldest first.+ if *logged[0].GenAI.Usage.InputTokens != 1 || *logged[1].GenAI.Usage.InputTokens != 2 {+ t.Fatalf("order = %v, %v", *logged[0].GenAI.Usage.InputTokens, *logged[1].GenAI.Usage.InputTokens)+ }+ for _, ev := range logged {+ if ev.Event.ID == "" {+ t.Error("the writer must stamp event.id")+ }+ }++ h.now = h.now.Add(time.Minute)+ second := h.sync()+ if second.Emitted != 0 {+ t.Fatalf("a repeat sync wrote %d events: %+v", second.Emitted, second)+ }+ if len(h.logged()) != 2 {+ t.Fatal("the log grew on a repeat sync")+ }+ // The repeat asks only for the lookback window, not the whole first window again.+ q := h.fetcher.queries[len(h.fetcher.queries)-1]+ if got := baseNow.Sub(q.StartDate); got != DefaultLookback {+ t.Fatalf("repeat window starts %v before the last sync, want %v", got, DefaultLookback)+ }+}++func TestLateEventInsideTheLookbackIsCollected(t *testing.T) {+ h := newHarness(t, usageAt(baseNow.Add(-time.Hour), 1))+ h.sync()+ // Cursor posts an event for a request made before the last sync, after it ran.+ h.fetcher.events = append(h.fetcher.events, usageAt(baseNow.Add(-30*time.Minute), 2))+ h.now = baseNow.Add(10 * time.Minute)+ summary := h.sync()+ if summary.Emitted != 1 || summary.Duplicates != 1 {+ t.Fatalf("summary = %+v", summary)+ }+ if len(h.logged()) != 2 {+ t.Fatal("late event not written")+ }+}++func TestIdenticalRequestsAreBothWrittenOnceEach(t *testing.T) {+ ev := usageAt(baseNow.Add(-time.Hour), 7)+ h := newHarness(t, ev, ev)+ if s := h.sync(); s.Emitted != 2 {+ t.Fatalf("summary = %+v", s)+ }+ h.now = baseNow.Add(time.Minute)+ if s := h.sync(); s.Emitted != 0 || s.Duplicates != 2 {+ t.Fatalf("summary = %+v", s)+ }+}++func TestBackfillReachesFurtherBackWithoutRepeatingCollectedSpend(t *testing.T) {+ h := newHarness(t,+ usageAt(baseNow.Add(-20*24*time.Hour), 1),+ usageAt(baseNow.Add(-5*24*time.Hour), 2), // collected by the first sync, pruned from Seen later+ usageAt(baseNow.Add(-time.Hour), 3),+ )+ h.sync()+ // Several days pass with syncs, so the -5d event leaves the Seen window.+ for i := 1; i <= 6; i++ {+ h.now = baseNow.Add(time.Duration(i) * 24 * time.Hour)+ h.sync()+ }+ if len(h.logged()) != 2 {+ t.Fatalf("logged = %d", len(h.logged()))+ }++ h.fetcher.queries = nil+ h.since = baseNow.Add(-30 * 24 * time.Hour)+ summary := h.sync()+ if summary.Emitted != 1 {+ t.Fatalf("backfill wrote %d events, want only the one older than anything collected: %+v", summary.Emitted, summary)+ }+ if got := len(h.logged()); got != 3 {+ t.Fatalf("logged = %d, want 3", got)+ }+ // The span already collected is not fetched again.+ covered := baseNow.Add(-DefaultInitialWindow)+ for _, q := range h.fetcher.queries {+ if q.StartDate.Before(covered) && q.EndDate.After(covered.Add(time.Hour)) {+ t.Fatalf("query %v..%v re-reads the collected span", q.StartDate, q.EndDate)+ }+ }++ // And running the same backfill again writes nothing and fetches only the recent window.+ h.fetcher.queries = nil+ h.now = h.now.Add(time.Minute)+ if s := h.sync(); s.Emitted != 0 {+ t.Fatalf("second backfill wrote %d", s.Emitted)+ }+ for _, q := range h.fetcher.queries {+ if q.StartDate.Before(h.now.Add(-DefaultLookback - time.Hour)) {+ t.Fatalf("repeat backfill re-read %v..%v", q.StartDate, q.EndDate)+ }+ }+}++// A host that returns the collected span although it was not asked for it must not get it+// written twice: the span is never requested, and anything outside a request is dropped.+func TestCollectedSpanIsSkippedEvenWhenTheHostReturnsIt(t *testing.T) {+ h := newHarness(t, usageAt(baseNow.Add(-5*24*time.Hour), 1), usageAt(baseNow.Add(-time.Hour), 2))+ h.sync()+ for i := 1; i <= 6; i++ {+ h.now = baseNow.Add(time.Duration(i) * 24 * time.Hour)+ h.sync()+ }+ h.fetcher.ignoreDates = true+ h.lookback = 30 * 24 * time.Hour // fetch window now covers the -5d event again+ h.now = h.now.Add(time.Minute)+ s := h.sync()+ if s.Emitted != 0 {+ t.Fatalf("re-wrote %d collected events: %+v", s.Emitted, s)+ }+}++func TestSegments(t *testing.T) {+ ms := func(d time.Duration) time.Time { return baseNow.Add(d).Truncate(time.Millisecond) }+ c := &Cursor{CoveredFromMS: ms(-10 * 24 * time.Hour).UnixMilli(), HighWaterMS: baseNow.UnixMilli()}+ seen := ms(-48 * time.Hour)++ got := segments(nil, ms(-time.Hour), baseNow, time.Time{})+ if len(got) != 1 || !got[0][0].Equal(ms(-time.Hour)) {+ t.Fatalf("no cursor = %v", got)+ }+ got = segments(c, seen, baseNow, seen)+ if len(got) != 1 || !got[0][0].Equal(seen) {+ t.Fatalf("ordinary resume = %v", got)+ }+ got = segments(c, ms(-10*24*time.Hour), baseNow, seen)+ if len(got) != 1 || !got[0][0].Equal(seen) {+ t.Fatalf("since at the covered edge must fetch only the recent window: %v", got)+ }+ got = segments(c, ms(-20*24*time.Hour), baseNow, seen)+ if len(got) != 2 || !got[0][1].Equal(ms(-10*24*time.Hour).Add(-time.Millisecond)) || !got[1][0].Equal(seen) {+ t.Fatalf("backfill = %v", got)+ }+ got = segments(c, ms(-20*24*time.Hour), baseNow, ms(-15*24*time.Hour))+ if len(got) != 2 || !got[1][0].Equal(ms(-10*24*time.Hour)) {+ t.Fatalf("seen reaching past the covered edge = %v", got)+ }+}++func TestLongerLookbackDoesNotRewriteCollectedSpend(t *testing.T) {+ h := newHarness(t, usageAt(baseNow.Add(-5*24*time.Hour), 1), usageAt(baseNow.Add(-time.Hour), 2))+ h.sync()+ for i := 1; i <= 6; i++ {+ h.now = baseNow.Add(time.Duration(i) * 24 * time.Hour)+ h.sync()+ }+ h.lookback = 30 * 24 * time.Hour+ h.now = h.now.Add(time.Minute)+ if s := h.sync(); s.Emitted != 0 {+ t.Fatalf("a longer lookback re-wrote %d events: %+v", s.Emitted, s)+ }+ // And a later sync with the same longer lookback: the completeness boundary must not have+ // moved back over keys that were already pruned.+ h.now = h.now.Add(time.Hour)+ if s := h.sync(); s.Emitted != 0 {+ t.Fatalf("the next sync with the longer lookback re-wrote %d events: %+v", s.Emitted, s)+ }+ if len(h.logged()) != 2 {+ t.Fatalf("logged = %d", len(h.logged()))+ }+}++func TestLongWindowsAreSplitWithoutSharedBoundaries(t *testing.T) {+ h := newHarness(t)+ h.since = baseNow.Add(-75 * 24 * time.Hour)+ boundary := h.since.Add(MaxWindow)+ h.fetcher.events = []UsageEvent{usageAt(boundary, 1), usageAt(boundary.Add(time.Millisecond), 2)}+ summary := h.sync()+ if len(h.fetcher.queries) != 3 {+ t.Fatalf("queries = %d, want 3 for 75 days", len(h.fetcher.queries))+ }+ for i := 1; i < len(h.fetcher.queries); i++ {+ prev, cur := h.fetcher.queries[i-1], h.fetcher.queries[i]+ if cur.StartDate.Sub(prev.EndDate) != time.Millisecond {+ t.Fatalf("chunks %d and %d are not adjacent: %v .. %v", i-1, i, prev.EndDate, cur.StartDate)+ }+ if cur.EndDate.Sub(cur.StartDate) > MaxWindow {+ t.Fatalf("chunk %d is longer than MaxWindow", i)+ }+ }+ if summary.Emitted != 2 || len(h.logged()) != 2 {+ t.Fatalf("an event on a chunk boundary must be written once: %+v", summary)+ }+}++func TestEventsOutsideTheWindowAreDropped(t *testing.T) {+ h := newHarness(t, usageAt(baseNow.Add(-90*24*time.Hour), 1), usageAt(baseNow.Add(time.Hour), 2), usageAt(baseNow.Add(-time.Hour), 3))+ h.fetcher.ignoreDates = true+ if s := h.sync(); s.Emitted != 1 || s.Fetched != 1 {+ t.Fatalf("summary = %+v", s)+ }+}++func TestFetchErrorWritesNothingAndKeepsTheCursor(t *testing.T) {+ h := newHarness(t, usageAt(baseNow.Add(-time.Hour), 1))+ h.fetcher.err = errors.New("boom")+ if _, err := CollectOnce(context.Background(), h.opts()); err == nil {+ t.Fatal("expected the fetch error")+ }+ if len(h.logged()) != 0 {+ t.Fatal("wrote events after a failed fetch")+ }+ if _, err := os.Stat(h.state); !errors.Is(err, os.ErrNotExist) {+ t.Fatal("a failed fetch must not create state")+ }+ h.fetcher.err = nil+ if s := h.sync(); s.Emitted != 1 {+ t.Fatalf("retry = %+v", s)+ }+}++func TestPartialWriteFailureResumesWithoutDuplicates(t *testing.T) {+ h := newHarness(t, usageAt(baseNow.Add(-3*time.Hour), 1), usageAt(baseNow.Add(-2*time.Hour), 2), usageAt(baseNow.Add(-time.Hour), 3))+ var written []int64+ opts := h.opts()+ opts.emitHook = func(ev schema.Event) error {+ if len(written) == 2 {+ return errors.New("disk full")+ }+ written = append(written, *ev.GenAI.Usage.InputTokens)+ return nil+ }+ if _, err := CollectOnce(context.Background(), opts); err == nil {+ t.Fatal("expected the write error")+ }+ state, err := LoadState(h.state)+ if err != nil {+ t.Fatal(err)+ }+ cursor := state.Scopes["email:[email protected]"]+ if cursor == nil || len(cursor.Seen) != 2 || cursor.HighWaterMS != 0 {+ t.Fatalf("after a partial write the cursor must hold the written keys and not advance: %+v", cursor)+ }++ opts.emitHook = func(ev schema.Event) error {+ written = append(written, *ev.GenAI.Usage.InputTokens)+ return nil+ }+ if _, err := CollectOnce(context.Background(), opts); err != nil {+ t.Fatal(err)+ }+ if len(written) != 3 || written[0] != 1 || written[1] != 2 || written[2] != 3 {+ t.Fatalf("written = %v, want each event exactly once", written)+ }+}++func TestScopesKeepSeparateCursors(t *testing.T) {+ h := newHarness(t, usageAt(baseNow.Add(-time.Hour), 1))+ h.sync()+ h.query = Query{Email: "[email protected]"}+ if s := h.sync(); s.Emitted != 1 || s.Scope != "email:[email protected]" {+ t.Fatalf("a different member starts from its own first window: %+v", s)+ }+ h.query = Query{}+ if s := h.sync(); s.Scope != "team" || s.Emitted != 1 {+ t.Fatalf("team scope = %+v", s)+ }+ if got := ScopeKey(Query{UserID: " 42 "}); got != "user:42" {+ t.Fatalf("user scope = %q", got)+ }+ if got := ScopeKey(Query{Email: "[email protected]", UserID: "42"}); got != "email:[email protected]" {+ t.Fatalf("email wins and is case-folded: %q", got)+ }+}++func TestPrintModeWritesNoLogAndNoState(t *testing.T) {+ h := newHarness(t, usageAt(baseNow.Add(-time.Hour), 1))+ var out bytes.Buffer+ opts := h.opts()+ opts.Print = true+ opts.Write = false+ opts.StatePath = ""+ opts.Out = &out+ for i := 0; i < 2; i++ {+ s, err := CollectOnce(context.Background(), opts)+ if err != nil {+ t.Fatal(err)+ }+ if s.Emitted != 1 {+ t.Fatalf("print run %d emitted %d", i, s.Emitted)+ }+ }+ if len(h.logged()) != 0 {+ t.Fatal("print wrote the log")+ }+ if lines := bytes.Count(out.Bytes(), []byte("\n")); lines != 2 {+ t.Fatalf("printed %d lines", lines)+ }+}++func TestSeenIsPrunedToTheLookbackWindow(t *testing.T) {+ var events []UsageEvent+ for i := 0; i < 24*10; i++ {+ events = append(events, usageAt(baseNow.Add(-time.Duration(i)*time.Hour), int64(i+1)))+ }+ h := newHarness(t, events...)+ h.since = baseNow.Add(-11 * 24 * time.Hour)+ h.sync()+ state, err := LoadState(h.state)+ if err != nil {+ t.Fatal(err)+ }+ cursor := state.Scopes["email:[email protected]"]+ if len(cursor.Seen) > 49 {+ t.Fatalf("seen holds %d keys, want only the 48h lookback", len(cursor.Seen))+ }+ if cursor.SeenFromMS != baseNow.Add(-DefaultLookback).UnixMilli() {+ t.Fatalf("seen_from_ms = %d", cursor.SeenFromMS)+ }+ if info, err := os.Stat(h.state); err == nil && info.Mode().Perm() != 0o600 && os.PathSeparator == '/' {+ t.Fatalf("state mode = %v", info.Mode().Perm())+ }+}++func TestLoadStateResetsOtherVersionsAndRejectsGarbage(t *testing.T) {+ dir := t.TempDir()+ path := filepath.Join(dir, "s.json")+ if err := os.WriteFile(path, []byte(`{"version":99,"scopes":{"team":{"high_water_ms":5}}}`), 0o600); err != nil {+ t.Fatal(err)+ }+ state, err := LoadState(path)+ if err != nil || len(state.Scopes) != 0 {+ t.Fatalf("other version = %+v, %v", state, err)+ }+ if err := os.WriteFile(path, []byte(`{`), 0o600); err != nil {+ t.Fatal(err)+ }+ if _, err := LoadState(path); err == nil {+ t.Fatal("garbage state must be an error, not a silent reset")+ }+}++func TestWindow(t *testing.T) {+ now := baseNow.Add(123456 * time.Nanosecond)+ start, end := Window(nil, time.Time{}, now, 0)+ if !start.Equal(baseNow.Add(-DefaultInitialWindow)) || !end.Equal(baseNow) {+ t.Fatalf("first window = %v..%v (edges must be whole milliseconds)", start, end)+ }+ c := &Cursor{HighWaterMS: baseNow.UnixMilli()}+ start, _ = Window(c, time.Time{}, now, time.Hour)+ if !start.Equal(baseNow.Add(-time.Hour)) {+ t.Fatalf("resumed window starts %v", start)+ }+ start, _ = Window(c, baseNow.Add(-time.Minute), now, time.Hour)+ if !start.Equal(baseNow.Add(-time.Hour)) {+ t.Fatalf("since narrowed the window: %v", start)+ }+ start, end = Window(nil, baseNow.Add(time.Hour), now, 0)+ if !start.Equal(end) {+ t.Fatalf("future since = %v..%v", start, end)+ }+}++// The end-to-end check: what the collector writes is what `beacon token-usage` and the dashboard+// token view sum.+func TestCollectedEventsRollUpInTheTokenReport(t *testing.T) {+ h := newHarness(t, usageAt(baseNow.Add(-2*time.Hour), 100), usageAt(baseNow.Add(-time.Hour), 200))+ h.sync()+ report := tokens.Aggregate(h.logged(), tokens.Options{})+ got := report.Totals+ if got.InputTokens != 300 || got.OutputTokens != 20 || got.CacheReadInputTokens != 200 || got.CacheCreationInputTokens != 10 {+ t.Fatalf("totals = %+v", got)+ }+ if got.CostUSD < 0.0399 || got.CostUSD > 0.0401 {+ t.Fatalf("cost = %v, want 0.04", got.CostUSD)+ }+ if len(report.ByHarness) != 1 || report.ByHarness[0].Key != "cursor" {+ t.Fatalf("by harness = %+v", report.ByHarness)+ }+ if len(report.ByModel) != 1 || report.ByModel[0].Usage.InputTokens != 300 {+ t.Fatalf("by model = %+v", report.ByModel)+ }+}diff --git a/cli/beacon/internal/cursorusage/mapper.go b/cli/beacon/internal/cursorusage/mapper.gonew file mode 100644index 00000000..f9cd8630--- /dev/null+++ b/cli/beacon/internal/cursorusage/mapper.go@@ -0,0 +1,173 @@+package cursorusage++import (+ "crypto/sha256"+ "encoding/hex"+ "fmt"+ "strconv"+ "strings"++ "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/schema"+)++// Harness is the canonical harness name the events are recorded under: the same name Cursor's+// hooks and local session reader use, so a token report groups Admin API spend with the rest of+// Cursor's activity.+const Harness = "cursor"++// Source is recorded under raw.cursor.source so a reader can tell these events from anything a+// future local Cursor path might report.+const Source = "admin_api"++// HasUsage reports whether an event carries anything a token report can sum. The Admin API also+// lists requests billed per request rather than by token, which carry no token block and often no+// charge; an event with neither is a row of zeros and is not written.+func HasUsage(ev UsageEvent) bool {+ t := ev.TokenUsage+ if t.InputTokens > 0 || t.OutputTokens > 0 || t.CacheReadTokens > 0 || t.CacheWriteTokens > 0 {+ return true+ }+ return ev.ChargedCents != nil && *ev.ChargedCents > 0+}++// MapEvent converts one Admin API usage event into a Beacon token.usage event.+//+// The token counts go into gen_ai.usage exactly as Cursor reports them: inputTokens is the+// uncached input, cacheReadTokens and cacheWriteTokens are the cache buckets, so they map onto+// cache_read and cache_creation without double counting. cost_usd is chargedCents/100 -- what+// Cursor reports it charged, which is runtime-reported cost in the sense Beacon means. Cursor's+// token price (totalCents) is kept under raw.cursor rather than promoted, because a request+// included in a plan has a token price and no charge, and a report summing token prices would+// present a bill nobody received.+//+// No session is set. The Admin API reports requests per team member, not per conversation, and+// nothing in an event names the Composer it came from; attaching one by time overlap would be a+// guess presented as an observation. Fidelity is observed: Cursor names the request and its+// counts, Beacon derives nothing.+func MapEvent(ev UsageEvent) schema.Event {+ out := schema.NewEvent(schema.NewEventOptions{+ Action: "token.usage",+ Category: "metric",+ Severity: schema.SeverityInfo,+ Fidelity: schema.FidelityObserved,+ Message: "Cursor usage event",+ Origin: schema.OriginLocal,+ Harness: schema.HarnessInfo{+ Name: Harness,+ // Poll: a pull of records Cursor committed after the fact. It cannot hold or deny+ // anything, which is what the marker exists to say.+ CollectionMethod: schema.CollectionMethodPoll,+ },+ })+ out.Timestamp = schema.FormatTimestamp(ev.Timestamp)+ out.Model = ev.Model++ usage := &schema.GenAIUsageInfo{}+ if n := ev.TokenUsage.InputTokens; n > 0 {+ usage.InputTokens = &n+ }+ if n := ev.TokenUsage.OutputTokens; n > 0 {+ usage.OutputTokens = &n+ }+ if n := ev.TokenUsage.CacheReadTokens; n > 0 {+ usage.CacheRead = &schema.GenAIUsageCacheReadInfo{InputTokens: &n}+ }+ if n := ev.TokenUsage.CacheWriteTokens; n > 0 {+ usage.CacheCreation = &schema.GenAIUsageCacheCreationInfo{InputTokens: &n}+ }+ if ev.ChargedCents != nil {+ cost := *ev.ChargedCents / 100+ usage.CostUSD = &cost+ }+ out.GenAI = &schema.GenAIInfo{Usage: usage}++ raw := map[string]interface{}{+ "source": Source,+ "kind": ev.Kind,+ "max_mode": ev.MaxMode,+ "token_based": ev.TokenBased,+ "headless": ev.Headless,+ }+ if ev.TokenUsage.TotalCents != nil {+ raw["token_cents"] = *ev.TokenUsage.TotalCents+ }+ if ev.ChargedCents != nil {+ raw["charged_cents"] = *ev.ChargedCents+ }+ if ev.CursorTokenFeeCents != nil {+ raw["cursor_token_fee_cents"] = *ev.CursorTokenFeeCents+ }+ if ev.RequestsCosts != nil {+ raw["requests_costs"] = *ev.RequestsCosts+ }+ if ev.Chargeable != nil {+ raw["chargeable"] = *ev.Chargeable+ }+ if ev.UserID != "" {+ raw["user_id"] = ev.UserID+ }+ if ev.UserEmail != "" {+ raw["user_email"] = ev.UserEmail+ }+ out.Raw = map[string]interface{}{"cursor": raw}+ return out+}++// Fingerprint is an event's identity built from everything the Admin API says about it. The API+// gives events no id of their own, so this is the only stable name one has across two fetches of+// the same window.+func Fingerprint(ev UsageEvent) string {+ var b strings.Builder+ fmt.Fprintf(&b, "%d|%s|%s|%t|%t|%d|%d|%d|%d|%s|%s|%s|%s|%s|%t|%s|%s",+ ev.Timestamp.UnixMilli(),+ ev.Model,+ ev.Kind,+ ev.MaxMode,+ ev.TokenBased,+ ev.TokenUsage.InputTokens,+ ev.TokenUsage.OutputTokens,+ ev.TokenUsage.CacheWriteTokens,+ ev.TokenUsage.CacheReadTokens,+ optionalAmount(ev.TokenUsage.TotalCents),+ optionalAmount(ev.ChargedCents),+ optionalAmount(ev.CursorTokenFeeCents),+ optionalAmount(ev.RequestsCosts),+ optionalBool(ev.Chargeable),+ ev.Headless,+ ev.UserID,+ ev.UserEmail,+ )+ sum := sha256.Sum256([]byte(b.String()))+ return hex.EncodeToString(sum[:])+}++// DedupKeys names every event in a fetch. Two requests can be identical in every field the API+// reports -- same millisecond, model and counts -- and they are still two requests, so the nth+// copy of a fingerprint gets its own key rather than collapsing into the first. That is stable across+// fetches because identical events share a timestamp, so a window that includes one includes all+// of them.+func DedupKeys(events []UsageEvent) []string {+ seen := map[string]int{}+ keys := make([]string, len(events))+ for i, ev := range events {+ fp := Fingerprint(ev)+ n := seen[fp]+ seen[fp] = n + 1+ keys[i] = fp + "#" + strconv.Itoa(n)+ }+ return keys+}++func optionalAmount(v *float64) string {+ if v == nil {+ return "-"+ }+ return strconv.FormatFloat(*v, 'g', -1, 64)+}++func optionalBool(v *bool) string {+ if v == nil {+ return "-"+ }+ return strconv.FormatBool(*v)+}