The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu
Go agent-ai-cliagent-harnessesagent-securitybrowser-extensionciclaude-codecloudcodexcursordetection-engineeringjevknowledge-basememorymobile-device-managementobservabilitysecuritysecurity-information-and-event-managementsecurity-toolstelemetrytraces
feat(harness2session): read Harness² executor trajectories as endpoint events
Claude6a4e1525d37f
diff --git a/cli/beacon/internal/harness2session/codex.go b/cli/beacon/internal/harness2session/codex.gonew file mode 100644index 00000000..4a279822--- /dev/null+++ b/cli/beacon/internal/harness2session/codex.go@@ -0,0 +1,221 @@+package harness2session++import (+ "path/filepath"+ "strconv"+ "strings"++ "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/schema"+ "github.com/asymptote-labs/agent-beacon/pkg/asymptoteobserve"+)++// mapCodexExec maps Codex's `exec --json` stream (codex-rs/exec/src/exec_events.rs): thread.started+// names the thread, turn.completed carries usage, and each thread item is reported as it starts,+// updates and completes. Only item.completed is mapped, since that is the item's final state; an+// item still in progress when the stream ended -- a command the timeout killed -- has no outcome to+// record. Harness² gives the prompt as an argument, so it is not in the stream.+func (m *mapper) mapCodexExec(lines []Line) {+ // turn.completed reports the thread's running total (usage_from_last_total in+ // event_processor_with_jsonl_output.rs), so each turn's own usage is the difference from the+ // previous turn's. The previous total is tracked over every line, written or not, so a sweep+ // that resumes mid-stream differences against the right base.+ var previous codexUsage+ for _, line := range lines {+ switch str(line.Object, "type") {+ case "thread.started":+ m.emitStarted(line, 0, schema.FidelityObserved)+ case "turn.completed":+ total := codexUsageOf(object(line.Object, "usage"))+ if m.emits(line) {+ m.emitUsage(line, 0, total.minus(previous).genAI())+ }+ previous = total+ case "turn.failed":+ if m.emits(line) {+ m.emitError(line, 0, schema.SeverityHigh, "turn_failed", str(object(line.Object, "error"), "message"))+ }+ case "error":+ if m.emits(line) {+ m.emitError(line, 0, schema.SeverityHigh, "stream_error", str(line.Object, "message"))+ }+ case "item.completed":+ if m.emits(line) {+ m.emitCodexItem(line, object(line.Object, "item"))+ }+ }+ }+}++func (m *mapper) emitCodexItem(line Line, item map[string]interface{}) {+ id := str(item, "id")+ switch str(item, "type") {+ case "agent_message":+ m.emitText(line, 0, "agent.message", "assistant", str(item, "text"))+ case "reasoning":+ m.emitText(line, 0, "agent.reasoning", "reasoning", str(item, "text"))+ case "command_execution":+ m.emitCodexCommand(line, id, item)+ case "file_change":+ m.emitCodexFileChange(line, id, item)+ case "mcp_tool_call":+ m.emitCodexMCP(line, id, item)+ case "web_search":+ ev := m.base(line, 0, "tool.completed", "tool", schema.SeverityInfo, schema.FidelityObserved, "Harness² executor web search")+ ev.Tool = &schema.ToolInfo{Name: "web_search"}+ toolCall(&ev, "web_search", id, map[string]interface{}{"query": str(item, "query")}, nil)+ m.append(line, "tool."+id, ev)+ case "collab_tool_call":+ m.emitCodexCollab(line, id, item)+ case "error":+ // A thread item error is a warning Codex surfaced and carried on past; turn.failed and the+ // top-level error event are the ones that end a turn.+ m.emitError(line, 0, schema.SeverityMedium, "item_error", str(item, "message"))+ }+}++func (m *mapper) emitCodexCommand(line Line, id string, item map[string]interface{}) {+ status := str(item, "status")+ command := str(item, "command")+ action, severity, message := "command.executed", schema.SeverityInfo, "Harness² executor command executed"+ if status == "declined" {+ action, message = "tool.failed", "Harness² executor command declined"+ }+ ev := m.base(line, 0, action, "command", severity, schema.FidelityObserved, message)+ ev.Tool = &schema.ToolInfo{Name: "shell", Command: command}+ output := str(item, "aggregated_output")+ info := &schema.CommandInfo{Command: command, Output: output}+ if code, ok := num(item, "exit_code"); ok {+ exit := int(code)+ info.ExitCode = &exit+ }+ ev.Command = info+ var result interface{}+ if output != "" {+ result = output+ ev.Content = asymptoteobserve.RetainedContent(output, asymptoteobserve.DefaultStringLimit)+ }+ toolCall(&ev, "shell", id, map[string]interface{}{"command": command}, result)+ if status == "failed" || status == "declined" {+ ev.Error = &schema.ErrorInfo{Type: "command_" + status}+ }+ m.append(line, "tool."+id, ev)+}++// emitCodexFileChange writes one file event per path an apply_patch touched. The stream reports the+// paths and what happened to each, not the patch itself, so these events carry no diff.+func (m *mapper) emitCodexFileChange(line Line, id string, item map[string]interface{}) {+ changes, _ := item["changes"].([]interface{})+ failed := str(item, "status") == "failed"+ for i, raw := range changes {+ change, _ := raw.(map[string]interface{})+ path := str(change, "path")+ if path == "" {+ continue+ }+ action, operation := "file.modified", "modify"+ switch str(change, "kind") {+ case "add":+ action, operation = "file.created", "create"+ case "delete":+ // A deletion stays in the file family, with its own operation word, as the Kiro and fx+ // mappers record one: it is not a modification.+ operation = "delete"+ }+ ev := m.base(line, 0, action, "file", schema.SeverityInfo, schema.FidelityObserved, "Harness² executor file "+operation)+ ev.Tool = &schema.ToolInfo{Name: "apply_patch", Path: path}+ ev.File = &schema.FileInfo{Path: path, Operation: operation, Language: strings.TrimPrefix(filepath.Ext(path), ".")}+ toolCall(&ev, "apply_patch", id, map[string]interface{}{"path": path, "kind": str(change, "kind")}, nil)+ if failed {+ ev.Error = &schema.ErrorInfo{Type: "patch_failed"}+ }+ m.append(line, "file."+id+"."+strconv.Itoa(i), ev)+ }+}++func (m *mapper) emitCodexMCP(line Line, id string, item map[string]interface{}) {+ server, tool := str(item, "server"), str(item, "tool")+ ev := m.base(line, 0, "mcp.tool_invoked", "mcp", schema.SeverityInfo, schema.FidelityObserved, "Harness² executor MCP tool invoked")+ ev.Tool = &schema.ToolInfo{Name: firstNonEmpty(tool, "unknown")}+ ev.MCP = &schema.MCPInfo{Server: server, Tool: tool}+ var result interface{}+ if value, ok := item["result"]; ok && value != nil {+ result = value+ }+ toolCall(&ev, ev.Tool.Name, id, item["arguments"], result)+ if errObj := object(item, "error"); errObj != nil || str(item, "status") == "failed" {+ ev.Error = &schema.ErrorInfo{Type: "tool_error"}+ if text := str(errObj, "message"); text != "" {+ ev.Raw["error_message"] = truncate(text, 2000)+ }+ }+ m.append(line, "tool."+id, ev)+}++// emitCodexCollab records a call to one of Codex's agent-collaboration tools (spawn_agent,+// send_input, wait, close_agent). The agents it reaches run as their own threads, which this stream+// does not contain, so the call is recorded as a tool call with the thread ids it names.+func (m *mapper) emitCodexCollab(line Line, id string, item map[string]interface{}) {+ tool := firstNonEmpty(str(item, "tool"), "collab")+ action, severity := "tool.completed", schema.SeverityInfo+ if str(item, "status") == "failed" {+ action, severity = "tool.failed", schema.SeverityHigh+ }+ ev := m.base(line, 0, action, "tool", severity, schema.FidelityObserved, "Harness² executor agent collaboration")+ ev.Tool = &schema.ToolInfo{Name: tool}+ args := map[string]interface{}{}+ for _, key := range []string{"sender_thread_id", "receiver_thread_ids", "prompt"} {+ if value, ok := item[key]; ok && value != nil {+ args[key] = value+ }+ }+ toolCall(&ev, tool, id, args, item["agents_states"])+ m.append(line, "tool."+id, ev)+}++type codexUsage struct {+ input, cached, output, reasoning int64+}++func codexUsageOf(m map[string]interface{}) codexUsage {+ return codexUsage{+ input: positive(m, "input_tokens"),+ cached: positive(m, "cached_input_tokens"),+ output: positive(m, "output_tokens"),+ reasoning: positive(m, "reasoning_output_tokens"),+ }+}++// minus is the growth from previous. A total smaller than the one before it is not a running total+// any more -- a resumed thread restarting its count -- so it is taken whole.+func (u codexUsage) minus(previous codexUsage) codexUsage {+ if u.input < previous.input || u.cached < previous.cached || u.output < previous.output || u.reasoning < previous.reasoning {+ return u+ }+ return codexUsage{+ input: u.input - previous.input,+ cached: u.cached - previous.cached,+ output: u.output - previous.output,+ reasoning: u.reasoning - previous.reasoning,+ }+}++// genAI converts Codex's counts the way the Codex session collector does (codexsession+// usageFromCodex): Codex's input includes its cached input, which gen_ai.usage counts separately,+// and its output already includes reasoning, which gen_ai.usage counts inside output too.+func (u codexUsage) genAI() *schema.GenAIUsageInfo {+ if u == (codexUsage{}) {+ return nil+ }+ uncached := u.input - u.cached+ if uncached < 0 {+ uncached = 0+ }+ usage := &schema.GenAIUsageInfo{InputTokens: ptr(uncached), OutputTokens: ptr(u.output)}+ if u.cached > 0 {+ usage.CacheRead = &schema.GenAIUsageCacheReadInfo{InputTokens: ptr(u.cached)}+ }+ if u.reasoning > 0 {+ usage.Reasoning = &schema.GenAIUsageReasoningInfo{OutputTokens: ptr(u.reasoning)}+ }+ return usage+}diff --git a/cli/beacon/internal/harness2session/collect.go b/cli/beacon/internal/harness2session/collect.gonew file mode 100644index 00000000..0eb90ef4--- /dev/null+++ b/cli/beacon/internal/harness2session/collect.go@@ -0,0 +1,302 @@+package harness2session++import (+ "errors"+ "fmt"+ "io"+ "os"+ "path/filepath"+ "sort"+ "strings"+ "time"++ "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/writer"+ "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/sessionsync"+ "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/sessionwindow"+)++const StateVersion = 1++// Cursor records how far Beacon has read one trajectory file.+type Cursor struct {+ // SessionID is the session the file held when it was last read. Harness² re-rolls a run whose+ // output it cannot reuse by deleting the run directory and running again into the same path, so+ // a file that now names another session is a new stream, read from its start.+ SessionID string `json:"session_id,omitempty"`+ // SizeBytes and ModTimeUnixMS are the file as last read, a cheap check that skips files that+ // cannot have changed. They are not the record of what was written.+ SizeBytes int64 `json:"size_bytes,omitempty"`+ ModTimeUnixMS int64 `json:"mod_time_unix_ms,omitempty"`+ // Line is the last line whose events were all written.+ Line int `json:"line,omitempty"`+ // Started records that session.started was written for this file.+ Started bool `json:"started,omitempty"`+ // Events is the running total written for this file, across all sweeps.+ Events int `json:"events,omitempty"`+}++type State struct {+ Version int `json:"version"`+ // Roots are the runs directories earlier sweeps were pointed at with --runs-dir. A sweep with no+ // directory of its own reads these as well as the default, so a scheduled sync and the+ // install-time backfill keep reading a checkout that lives somewhere Beacon would not look.+ Roots []string `json:"roots,omitempty"`+ // Files is keyed by the trajectory's absolute path.+ Files map[string]*Cursor `json:"files"`+}++func DefaultStatePath() string {+ base := filepath.Join(os.TempDir(), "beacon")+ if home, err := os.UserHomeDir(); err == nil && strings.TrimSpace(home) != "" {+ base = filepath.Join(home, ".beacon")+ }+ return filepath.Join(base, "endpoint", "state", "harness2.json")+}++var stateFile = sessionsync.StateFile[State]{+ Name: "Harness²",+ New: func() *State { return &State{Version: StateVersion, Files: map[string]*Cursor{}} },+ Valid: func(s *State) bool { return s.Version == StateVersion && s.Files != nil },+ Stamp: func(s *State) {+ if s.Files == nil {+ s.Files = map[string]*Cursor{}+ }+ s.Version = StateVersion+ },+}++func LoadState(path string) (*State, error) { return stateFile.Load(path) }++func (s *State) Save(path string) error { return stateFile.Save(path, s) }++func (s *State) cursor(path string) *Cursor {+ if s.Files == nil {+ s.Files = map[string]*Cursor{}+ }+ c := s.Files[path]+ if c == nil {+ c = &Cursor{}+ s.Files[path] = c+ }+ return c+}++// remember adds roots to the ones later sweeps read.+func (s *State) remember(roots []string) {+ seen := map[string]bool{}+ for _, root := range s.Roots {+ seen[root] = true+ }+ for _, root := range roots {+ if !seen[root] {+ seen[root] = true+ s.Roots = append(s.Roots, root)+ }+ }+ sort.Strings(s.Roots)+}++type CollectOptions struct {+ // RunsDirs are the runs directories to read. Empty reads DefaultRunsDirs and every directory an+ // earlier sweep was given; given, they are read and remembered for later sweeps.+ RunsDirs []string+ StatePath string+ LogPath string+ Write bool+ UserMode bool+ Print bool+ Out io.Writer+ // Since, when set, limits the sweep to trajectories modified at or after it, newest first. Older+ // ones are not read and their cursors do not move, so a later sync still collects them. The+ // install-time backfill sets it; an ordinary sync leaves it zero.+ Since time.Time+ // Budget, when set, caps the bytes this sweep may append. The sweep stops at the first event+ // that does not fit and returns writer.ErrBudgetSpent, with cursors past only what was written.+ Budget *writer.Budget+ // RotateBytes is the size the runtime log rotates at while this sweep writes it; zero is+ // writer.DefaultRotateBytes.+ RotateBytes int64++ // guard keeps the sweep from rotating its own output out of the log; CollectOnce sets it.+ guard *writer.RetentionGuard+}++type Summary struct {+ Roots []string `json:"roots"`+ Trajectories int `json:"trajectories"`+ FilesChanged int `json:"files_changed"`+ EventsEmitted int `json:"events_emitted"`+ Errors int `json:"errors"`+ MalformedLines int `json:"malformed_lines"`+ // PartialFiles counts trajectories that ended in a line the executor was still writing. That+ // line is read on a later sweep.+ PartialFiles int `json:"partial_files"`+ sessionsync.Retention+}++// Roots resolves the runs directories a sweep with these options reads: the ones given, or else+// the defaults together with every directory an earlier sweep remembered.+func Roots(runsDirs []string, state *State) ([]string, error) {+ if len(runsDirs) > 0 {+ store, err := NewStore(runsDirs)+ if err != nil {+ return nil, err+ }+ return store.Roots, nil+ }+ var roots []string+ roots = append(roots, DefaultRunsDirs()...)+ if state != nil {+ roots = append(roots, state.Roots...)+ }+ store, err := NewStore(roots)+ if err != nil {+ return nil, err+ }+ return store.Roots, nil+}++func CollectOnce(opts CollectOptions) (summary Summary, err error) {+ if opts.Print && !opts.Write {+ opts.StatePath = ""+ }+ state, err := LoadState(opts.StatePath)+ if err != nil {+ return summary, err+ }+ roots, err := Roots(opts.RunsDirs, state)+ if err != nil {+ return summary, err+ }+ summary.Roots = roots+ store := &Store{Roots: roots}+ refs, err := store.List()+ if err != nil {+ return summary, err+ }+ if len(opts.RunsDirs) > 0 {+ state.remember(roots)+ }+ defer func() {+ if saveErr := state.Save(opts.StatePath); saveErr != nil && err == nil {+ err = fmt.Errorf("save Harness² collector state: %w", saveErr)+ }+ }()+ refs = sessionwindow.Recent(refs, opts.Since, func(r TrajectoryRef) int64 { return r.ModTimeUnixMS })+ summary.Trajectories = len(refs)++ var errs []error+ opts.guard = sessionsync.NewRetentionGuard(opts.Print, opts.Write)+ var stopErr error+ for _, ref := range refs {+ changed, collectErr := collectFile(store, ref, state, opts, &summary)+ if sessionsync.Stopped(collectErr) {+ if changed {+ summary.FilesChanged+++ }+ stopErr = collectErr+ errs = append(errs, collectErr)+ break+ }+ if collectErr != nil {+ summary.Errors+++ errs = append(errs, fmt.Errorf("Harness² trajectory %s: %w", ref.Path, collectErr))+ continue+ }+ if changed {+ summary.FilesChanged+++ }+ }+ if rotatedOut := summary.Finish(opts.sink(), stopErr); rotatedOut != nil {+ errs = append(errs, rotatedOut)+ }+ if len(errs) > 0 {+ return summary, errors.Join(errs...)+ }+ return summary, nil+}++func collectFile(store *Store, ref TrajectoryRef, state *State, opts CollectOptions, summary *Summary) (bool, error) {+ cursor := state.cursor(ref.Path)+ if cursor.SizeBytes > 0 && cursor.SizeBytes == ref.SizeBytes && cursor.ModTimeUnixMS == ref.ModTimeUnixMS {+ return false, nil+ }+ trajectory, stats, err := store.Read(ref)+ summary.MalformedLines += stats.Malformed+ if stats.PartialTail {+ summary.PartialFiles+++ }+ if err != nil {+ return false, err+ }+ sessionID := sessionIDOf(trajectory)+ // A file that now holds another session, or fewer lines than the cursor has passed, was+ // rewritten rather than appended to: Harness² re-rolled the run into the same path. Its lines+ // are read from the start. A new session id names new events, so nothing already written is+ // written again; a rewrite of the same session keeps session.started from being repeated.+ if cursor.SessionID != "" && cursor.SessionID != sessionID {+ *cursor = Cursor{}+ } else if trajectory.LineCount < cursor.Line {+ cursor.Line = 0+ }+ cursor.SessionID = sessionID+ mapped := MapTrajectory(trajectory, MapOptions{MinLine: cursor.Line, SkipStarted: cursor.Started})+ if len(mapped) == 0 {+ cursor.Line = maxInt(cursor.Line, trajectory.LineCount)+ // A file with nothing new is not a changed file, but its size is remembered so it is not+ // read again until it changes.+ cursor.SizeBytes, cursor.ModTimeUnixMS = ref.SizeBytes, ref.ModTimeUnixMS+ return false, nil+ }+ written := 0+ for _, item := range mapped {+ if err := opts.sink().Emit(item.Event); err != nil {+ commit(cursor, mapped, written, trajectory.LineCount)+ return written > 0, err+ }+ written+++ summary.EventsEmitted+++ }+ commit(cursor, mapped, written, trajectory.LineCount)+ cursor.SizeBytes, cursor.ModTimeUnixMS = ref.SizeBytes, ref.ModTimeUnixMS+ return true, nil+}++// commit advances the cursor over the lines whose events were all written. One line can map to+// several events -- an apply_patch touching three files is three -- so a line with an unwritten+// event left on it stays behind the cursor and is mapped again whole by the next sweep, whose ids+// match the ones already written. On a clean pass every complete line advances, including the ones+// that mapped to nothing.+func commit(cursor *Cursor, mapped []MappedEvent, written, lineCount int) {+ for _, item := range mapped[:written] {+ if item.Event.Event.Action == "session.started" {+ cursor.Started = true+ }+ }+ cursor.Events += written+ if written == len(mapped) {+ cursor.Line = maxInt(cursor.Line, lineCount)+ return+ }+ blocked := mapped[written].SourceLine+ for _, item := range mapped[:written] {+ if item.SourceLine < blocked && item.SourceLine > cursor.Line {+ cursor.Line = item.SourceLine+ }+ }+ // Lines between the last written event and the blocked one mapped to nothing; they are done too.+ if blocked-1 > cursor.Line {+ cursor.Line = blocked - 1+ }+}++func maxInt(a, b int) int {+ if a > b {+ return a+ }+ return b+}++func (o CollectOptions) sink() sessionsync.Sink {+ return sessionsync.Sink{Print: o.Print, Out: o.Out, Write: o.Write, Log: writer.Options{Path: o.LogPath, UserMode: o.UserMode, RotateSize: o.RotateBytes, Budget: o.Budget, Guard: o.guard}}+}diff --git a/cli/beacon/internal/harness2session/collect_test.go b/cli/beacon/internal/harness2session/collect_test.gonew file mode 100644index 00000000..f7c6bf0e--- /dev/null+++ b/cli/beacon/internal/harness2session/collect_test.go@@ -0,0 +1,313 @@+package harness2session++import (+ "bytes"+ "encoding/json"+ "errors"+ "fmt"+ "os"+ "path/filepath"+ "strings"+ "testing"+ "time"++ "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/schema"+ "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/writer"+ "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/sessionsync/sessionsynctest"+ "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/testenv"+)++type sweepEnv struct {+ root, statePath, logPath string+}++func newSweepEnv(t *testing.T) sweepEnv {+ t.Helper()+ // Keep the defaults off the real machine: an empty home and no HARNESS2_RUNS_DIR.+ testenv.SetHome(t, t.TempDir())+ t.Setenv("HARNESS2_RUNS_DIR", "")+ dir := t.TempDir()+ return sweepEnv{+ root: copyRuns(t),+ statePath: filepath.Join(dir, "state", "harness2.json"),+ logPath: filepath.Join(dir, "logs", "runtime.jsonl"),+ }+}++func (e sweepEnv) sweep(t *testing.T, mutate ...func(*CollectOptions)) Summary {+ t.Helper()+ opts := CollectOptions{RunsDirs: []string{e.root}, StatePath: e.statePath, LogPath: e.logPath, Write: true, UserMode: true}+ for _, m := range mutate {+ m(&opts)+ }+ summary, err := CollectOnce(opts)+ if err != nil {+ t.Fatalf("CollectOnce: %v", err)+ }+ return summary+}++func (e sweepEnv) logged(t *testing.T) []schema.Event {+ t.Helper()+ return sessionsynctest.RetainedEvents(t, e.logPath)+}++func countBy(events []schema.Event, pred func(schema.Event) bool) int {+ n := 0+ for _, ev := range events {+ if pred(ev) {+ n+++ }+ }+ return n+}++func appendLines(t *testing.T, path string, lines ...string) {+ t.Helper()+ f, err := os.OpenFile(path, os.O_APPEND|os.O_WRONLY, 0)+ if err != nil {+ t.Fatal(err)+ }+ defer f.Close()+ for _, line := range lines {+ if _, err := f.WriteString(line); err != nil {+ t.Fatal(err)+ }+ }+ // A growing file's size changes, but bump the time too so the test does not depend on the+ // filesystem's timestamp resolution.+ later := time.Now().Add(time.Minute)+ if err := os.Chtimes(path, later, later); err != nil {+ t.Fatal(err)+ }+}++func TestCollectOnceWritesEveryTrajectoryOnce(t *testing.T) {+ env := newSweepEnv(t)+ first := env.sweep(t)+ if first.Trajectories != 3 || first.FilesChanged != 3 || first.Errors != 0 || first.EventsEmitted != 21 {+ t.Fatalf("first sweep = %+v, want 3 trajectories and 21 events", first)+ }+ if first.MalformedLines != 2 {+ t.Fatalf("malformed = %d, want the WorkBuddy banner's two lines", first.MalformedLines)+ }+ events := env.logged(t)+ if len(events) != 21 {+ t.Fatalf("log holds %d events, want 21", len(events))+ }+ for _, ev := range events {+ if ev.Harness.Name != Harness || ev.Harness.CollectionMethod != schema.CollectionMethodPoll || ev.Event.ID == "" {+ t.Fatalf("logged event = %+v", ev.Harness)+ }+ }+ if got := countBy(events, func(ev schema.Event) bool { return ev.Event.Action == "session.started" }); got != 3 {+ t.Fatalf("session.started = %d, want one per trajectory", got)+ }++ second := env.sweep(t)+ if second.FilesChanged != 0 || second.EventsEmitted != 0 {+ t.Fatalf("second sweep = %+v, want nothing new", second)+ }+ if len(env.logged(t)) != 21 {+ t.Fatal("an unchanged runs directory wrote events again")+ }+}++// A LAB or WorkBuddy stream is written while the executor runs, so a sweep can see it half done. The+// next sweep writes only what was added, and a line still being written waits until it is whole.+func TestCollectOnceFollowsAGrowingStream(t *testing.T) {+ env := newSweepEnv(t)+ env.sweep(t)+ path := filepath.Join(env.root, filepath.FromSlash(labRel))++ appendLines(t, path,+ `{"type":"turn.started"}`+"\n",+ `{"type":"item.completed","item":{"id":"item_7","type":"agent_message","text":"Second turn."}}`+"\n",+ `{"type":"turn.completed","usage":{"input_tokens":8000,"cached_input_tokens":4500,"output_tokens":900,"reasoning_output_tokens":350}}`+"\n",+ `{"type":"item.completed","item":{"id":"item_8","type":"agent_mes`)+ summary := env.sweep(t)+ if summary.EventsEmitted != 2 || summary.FilesChanged != 1 || summary.PartialFiles != 1 {+ t.Fatalf("growth sweep = %+v, want the message and the turn's usage, with the partial line left", summary)+ }+ events := env.logged(t)+ last := events[len(events)-1]+ // The second turn's usage is its own, not the thread total.+ if last.GenAI == nil || last.GenAI.Usage == nil || *last.GenAI.Usage.InputTokens != 1500 || *last.GenAI.Usage.OutputTokens != 200 {+ t.Fatalf("second turn usage = %+v", last.GenAI)+ }++ appendLines(t, path, `sage","text":"Done."}}`+"\n")+ summary = env.sweep(t)+ if summary.EventsEmitted != 1 || summary.PartialFiles != 0 {+ t.Fatalf("completion sweep = %+v, want the finished line only", summary)+ }+ if got := countBy(env.logged(t), func(ev schema.Event) bool { return ev.Event.Action == "session.started" }); got != 3 {+ t.Fatalf("session.started = %d after growth, want it written once per trajectory", got)+ }+}++// Harness² re-rolls a run it cannot reuse into the same path. The new stream is a new session, read+// from its start, even though it is shorter than what the cursor had passed.+func TestCollectOnceReadsARerolledRunAsANewSession(t *testing.T) {+ env := newSweepEnv(t)+ env.sweep(t)+ path := filepath.Join(env.root, filepath.FromSlash(labRel))+ body := `{"type":"thread.started","thread_id":"rerolled-thread"}` + "\n" ++ `{"type":"item.completed","item":{"id":"item_0","type":"agent_message","text":"Again."}}` + "\n"+ if err := os.WriteFile(path, []byte(body), 0o644); err != nil {+ t.Fatal(err)+ }+ later := time.Now().Add(time.Hour)+ _ = os.Chtimes(path, later, later)+ summary := env.sweep(t)+ if summary.EventsEmitted != 2 {+ t.Fatalf("re-roll sweep = %+v, want session.started and the message", summary)+ }+ var rerolled []string+ for _, ev := range env.logged(t) {+ if ev.Session != nil && ev.Session.ID == "rerolled-thread" {+ rerolled = append(rerolled, ev.Event.Action)+ }+ }+ if strings.Join(rerolled, ",") != "session.started,agent.message" {+ t.Fatalf("re-rolled session events = %v", rerolled)+ }+}++// --print is a dry run in both directions: nothing is written and no cursor moves.+func TestCollectOncePrintWritesNothing(t *testing.T) {+ env := newSweepEnv(t)+ var out bytes.Buffer+ summary := env.sweep(t, func(o *CollectOptions) { o.Print, o.Write, o.Out = true, false, &out })+ if summary.EventsEmitted != 21 {+ t.Fatalf("print sweep = %+v", summary)+ }+ if lines := strings.Count(out.String(), "\n"); lines != 21 {+ t.Fatalf("printed %d lines, want 21", lines)+ }+ if _, err := os.Stat(env.statePath); !errors.Is(err, os.ErrNotExist) {+ t.Fatalf("print wrote a cursor file (err=%v)", err)+ }+ if _, err := os.Stat(env.logPath); !errors.Is(err, os.ErrNotExist) {+ t.Fatalf("print wrote the runtime log (err=%v)", err)+ }+ for _, line := range strings.Split(strings.TrimSpace(out.String()), "\n") {+ var ev schema.Event+ if err := json.Unmarshal([]byte(line), &ev); err != nil || ev.Harness.Name != Harness {+ t.Fatalf("printed line %q: %v", line, err)+ }+ }+}++// A checkout Beacon would not find by itself is named once; scheduled syncs and the install+// backfill, which name no directory, keep reading it.+func TestCollectOnceRemembersTheRunsDirectoriesItWasGiven(t *testing.T) {+ env := newSweepEnv(t)+ env.sweep(t, func(o *CollectOptions) { o.Write, o.Print = true, false })+ state, err := LoadState(env.statePath)+ if err != nil {+ t.Fatal(err)+ }+ if len(state.Roots) != 1 || state.Roots[0] != env.root {+ t.Fatalf("remembered roots = %v, want [%s]", state.Roots, env.root)+ }+ appendLines(t, filepath.Join(env.root, filepath.FromSlash(labRel)),+ `{"type":"item.completed","item":{"id":"item_9","type":"agent_message","text":"later"}}`+"\n")+ summary := env.sweep(t, func(o *CollectOptions) { o.RunsDirs = nil })+ if summary.EventsEmitted != 1 || len(summary.Roots) != 2 {+ t.Fatalf("sweep without a directory = %+v, want the remembered root read beside the default", summary)+ }+}++// The install backfill reads only recent trajectories, newest first, and leaves the rest -- cursors+// untouched -- for a manual sync.+func TestCollectOnceSinceLeavesOlderTrajectories(t *testing.T) {+ env := newSweepEnv(t)+ old := time.Now().Add(-90 * 24 * time.Hour)+ for _, rel := range []string{jbRel, wbRel} {+ if err := os.Chtimes(filepath.Join(env.root, filepath.FromSlash(rel)), old, old); err != nil {+ t.Fatal(err)+ }+ }+ summary := env.sweep(t, func(o *CollectOptions) { o.Since = time.Now().Add(-30 * 24 * time.Hour) })+ if summary.Trajectories != 1 || summary.EventsEmitted != 10 {+ t.Fatalf("windowed sweep = %+v, want only the recent LAB stream", summary)+ }+ summary = env.sweep(t)+ if summary.EventsEmitted != 11 {+ t.Fatalf("follow-up sweep = %+v, want the two older trajectories", summary)+ }+}++// A budget refusal stops the sweep with cursors past only what was written; the next sweep writes+// the rest and nothing twice.+func TestCollectOnceStopsOnTheBudgetAndResumes(t *testing.T) {+ env := newSweepEnv(t)+ opts := CollectOptions{RunsDirs: []string{env.root}, StatePath: env.statePath, LogPath: env.logPath, Write: true, UserMode: true, Budget: writer.NewBudget(6000)}+ summary, err := CollectOnce(opts)+ if !errors.Is(err, writer.ErrBudgetSpent) {+ t.Fatalf("err = %v, want the budget refusal", err)+ }+ if summary.EventsEmitted == 0 || summary.EventsEmitted >= 21 {+ t.Fatalf("budgeted sweep wrote %d events, want some but not all", summary.EventsEmitted)+ }+ env.sweep(t)+ events := env.logged(t)+ ids := map[string]bool{}+ for _, ev := range events {+ if ids[ev.Event.ID] {+ t.Fatalf("event %s (%s) written twice", ev.Event.ID, ev.Event.Action)+ }+ ids[ev.Event.ID] = true+ }+ if len(events) != 21 {+ t.Fatalf("log holds %d events after resuming, want 21", len(events))+ }+}++func TestCommitStopsAtTheFirstUnwrittenLine(t *testing.T) {+ mapped := []MappedEvent{{SourceLine: 2}, {SourceLine: 5}, {SourceLine: 5}, {SourceLine: 9}}+ for _, tc := range []struct {+ written, want int+ }{+ {0, 1}, // nothing written: lines before the first event mapped to nothing+ {1, 4}, // line 5 has an unwritten event, so it stays behind the cursor+ {2, 4}, // half of line 5 written is not line 5 written+ {3, 8},+ {4, 12}, // everything written: the whole file, trailing lines included+ } {+ cursor := &Cursor{}+ commit(cursor, mapped, tc.written, 12)+ if cursor.Line != tc.want {+ t.Errorf("written=%d: cursor line = %d, want %d", tc.written, cursor.Line, tc.want)+ }+ }+}++// A sync larger than the rotation window must not rotate its own first events away while moving+// every cursor past them (#973).+func TestCollectOnceKeepsWhatItWritesAcrossTheRotationWindow(t *testing.T) {+ testenv.SetHome(t, t.TempDir())+ t.Setenv("HARNESS2_RUNS_DIR", "")+ root := filepath.Join(t.TempDir(), "runs")+ src, err := os.ReadFile(filepath.Join("testdata", "runs", filepath.FromSlash(jbRel)))+ if err != nil {+ t.Fatal(err)+ }+ for i := 0; i < 40; i++ {+ dir := filepath.Join(root, "jb", "dataset", "p", fmt.Sprintf("task%d", i), "model_traj", "l")+ if err := os.MkdirAll(dir, 0o755); err != nil {+ t.Fatal(err)+ }+ body := strings.ReplaceAll(string(src), "ses_7a1f00c0ffee", fmt.Sprintf("ses_%03d", i))+ if err := os.WriteFile(filepath.Join(dir, "t_l_attempt1_x.jsonl"), []byte(body), 0o644); err != nil {+ t.Fatal(err)+ }+ }+ opts := CollectOptions{RunsDirs: []string{root}, Write: true, UserMode: true}+ sessionsynctest.CheckSweepKeepsItsOutput(t, func(statePath, logPath string, rotateBytes int64) (sessionsynctest.Result, error) {+ opts.StatePath, opts.LogPath, opts.RotateBytes = statePath, logPath, rotateBytes+ s, err := CollectOnce(opts)+ return sessionsynctest.Result{EventsEmitted: s.EventsEmitted, Errors: s.Errors, Retention: s.Retention}, err+ })+}diff --git a/cli/beacon/internal/harness2session/mapper.go b/cli/beacon/internal/harness2session/mapper.gonew file mode 100644index 00000000..cfc51820--- /dev/null+++ b/cli/beacon/internal/harness2session/mapper.go@@ -0,0 +1,315 @@+package harness2session++import (+ "crypto/sha1"+ "crypto/sha256"+ "encoding/hex"+ "encoding/json"+ "fmt"+ "io"+ "math"+ "strconv"+ "strings"+ "time"++ "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/schema"+ "github.com/asymptote-labs/agent-beacon/pkg/asymptoteobserve"+)++// MappedEvent is one event mapped from a trajectory, with the line it came from so the collector can+// advance its cursor only past lines whose events were all written.+type MappedEvent struct {+ DedupID string+ SourceLine int+ Event schema.Event+}++// MapOptions says how much of a trajectory an earlier sweep already wrote.+type MapOptions struct {+ // MinLine is the last line whose events were all written. Lines at or before it are still read,+ // because the session id and the running token totals come from the whole stream, but emit+ // nothing.+ MinLine int+ // SkipStarted is set once session.started was written for this trajectory.+ SkipStarted bool+}++// MapTrajectory maps a decoded trajectory to events in stream order.+func MapTrajectory(t Trajectory, opts MapOptions) []MappedEvent {+ m := &mapper{ref: t.Ref, opts: opts, sessionID: sessionIDOf(t)}+ switch t.Ref.Format {+ case FormatOpenCode:+ m.mapOpenCode(t.Lines)+ case FormatCodexExec:+ m.mapCodexExec(t.Lines)+ }+ return m.out+}++type mapper struct {+ ref TrajectoryRef+ opts MapOptions+ sessionID string+ out []MappedEvent+}++// sessionIDOf is the executor's own id for the session: OpenCode's sessionID, which every event+// carries, or the thread id Codex announces first. A stream that never names one -- a Codex run+// that failed before its thread started -- gets an id derived from the file's place in the runs+// directory, so its events still group as one session and keep their ids across sweeps.+func sessionIDOf(t Trajectory) string {+ for _, line := range t.Lines {+ switch t.Ref.Format {+ case FormatOpenCode:+ if id := str(line.Object, "sessionID"); id != "" {+ return id+ }+ case FormatCodexExec:+ if str(line.Object, "type") == "thread.started" {+ if id := str(line.Object, "thread_id"); id != "" {+ return id+ }+ }+ }+ }+ sum := sha256.Sum256([]byte(t.Ref.Format.Executor() + ":" + t.Ref.Rel))+ return "harness2-" + hex.EncodeToString(sum[:8])+}++func (m *mapper) emits(line Line) bool { return line.Number > m.opts.MinLine }++func (m *mapper) base(line Line, timestampMS int64, action, category string, severity schema.Severity, fidelity, message string) schema.Event {+ ev := schema.NewEvent(schema.NewEventOptions{+ Action: action,+ Category: category,+ Severity: severity,+ Fidelity: fidelity,+ Message: message,+ Origin: schema.OriginLocal,+ Harness: schema.HarnessInfo{+ Name: Harness,+ CollectionMethod: schema.CollectionMethodPoll,+ },+ })+ // Codex's exec stream carries no times at all, so its events take the file's modification time:+ // when Harness² last wrote the stream. That is the end of the execution rather than the moment+ // of each step, and the order within the session comes from the order the events are written.+ if timestampMS <= 0 {+ timestampMS = m.ref.ModTimeUnixMS+ }+ if timestampMS > 0 {+ ev.Timestamp = schema.FormatTimestamp(time.UnixMilli(timestampMS))+ }+ ev.Session = &schema.SessionInfo{ID: m.sessionID}+ ev.Raw = map[string]interface{}{"harness2": m.contextRaw(line)}+ return ev+}++// contextRaw is the Harness² context every event carries: which executor ran, and what the runs+// directory says about the rollout. It is the reason to read these files through this collector+// rather than as plain OpenCode or Codex sessions.+func (m *mapper) contextRaw(line Line) map[string]interface{} {+ ctx := m.ref.Context+ raw := map[string]interface{}{+ "executor": m.ref.Format.Executor(),+ "format": string(m.ref.Format),+ "trajectory": m.ref.Rel,+ "line": line.Number,+ }+ for key, value := range map[string]string{+ "benchmark": ctx.Benchmark,+ "subset": ctx.Subset,+ "task_id": ctx.TaskID,+ "label": ctx.Label,+ "run_id": ctx.RunID,+ "trial": ctx.Trial,+ "harness_fingerprint": ctx.HarnessFingerprint,+ } {+ if value != "" {+ raw[key] = value+ }+ }+ if ctx.Attempt > 0 {+ raw["attempt"] = ctx.Attempt+ }+ return raw+}++// emitStarted writes session.started once per trajectory, at the stream's first event. Neither+// stream format records a session start as such -- OpenCode's opens with its first step and Codex's+// with the thread id -- so the event is inferred from the stream having begun, and it is where the+// session's Harness² context is first stated.+func (m *mapper) emitStarted(line Line, timestampMS int64, fidelity string) {+ if m.opts.SkipStarted || !m.emits(line) {+ return+ }+ ev := m.base(line, timestampMS, "session.started", "session", schema.SeverityInfo, fidelity, "Harness² "+m.ref.Format.Executor()+" rollout started")+ m.append(line, "session.started", ev)+}++func (m *mapper) append(line Line, suffix string, ev schema.Event) {+ dedupID := fmt.Sprintf("%s:%s:%d:%s", m.ref.Format.Executor(), m.sessionID, line.Number, suffix)+ ev.Event.ID = harness2EventID(dedupID)+ m.out = append(m.out, MappedEvent{DedupID: dedupID, SourceLine: line.Number, Event: ev})+}++func (m *mapper) emitText(line Line, timestampMS int64, action, suffix, text string) {+ text = strings.TrimSpace(text)+ if text == "" {+ return+ }+ var ev schema.Event+ if action == "agent.reasoning" {+ ev = m.base(line, timestampMS, action, "session", schema.SeverityInfo, schema.FidelityObserved, "Harness² executor reasoning")+ ev.GenAI = &schema.GenAIInfo{Output: &schema.GenAIOutputInfo{Messages: []interface{}{map[string]interface{}{+ "role": "assistant",+ "parts": []interface{}{map[string]interface{}{"type": "reasoning", "content": text}},+ }}}}+ } else {+ ev = m.base(line, timestampMS, action, "session", schema.SeverityInfo, schema.FidelityObserved, "Harness² executor message")+ ev.GenAI = &schema.GenAIInfo{Output: &schema.GenAIOutputInfo{Messages: asymptoteobserve.TextOutputMessages(text)}}+ }+ ev.Content = asymptoteobserve.RetainedContent(text, asymptoteobserve.DefaultRawStringLimit)+ m.append(line, suffix, ev)+}++func (m *mapper) emitError(line Line, timestampMS int64, severity schema.Severity, errorType, message string) {+ ev := m.base(line, timestampMS, "session.error", "session", severity, schema.FidelityObserved, "Harness² executor error")+ ev.Error = &schema.ErrorInfo{Type: firstNonEmpty(errorType, "session_error")}+ if message = strings.TrimSpace(message); message != "" {+ ev.Raw["error_message"] = truncate(message, 2000)+ }+ m.append(line, "error", ev)+}++func usageEvent(usage *schema.GenAIUsageInfo) bool {+ return usage != nil && (usage.InputTokens != nil || usage.OutputTokens != nil || usage.Reasoning != nil ||+ usage.CacheRead != nil || usage.CacheCreation != nil || usage.CostUSD != nil)+}++func (m *mapper) emitUsage(line Line, timestampMS int64, usage *schema.GenAIUsageInfo) {+ if !usageEvent(usage) {+ return+ }+ ev := m.base(line, timestampMS, "token.usage", "metric", schema.SeverityInfo, schema.FidelityObserved, "Harness² executor token usage")+ ev.GenAI = &schema.GenAIInfo{Usage: usage}+ m.append(line, "usage", ev)+}++// toolCall fills the GenAI tool-call block every tool event carries.+func toolCall(ev *schema.Event, name, callID string, args interface{}, result interface{}) {+ call := &schema.GenAIToolCallInfo{ID: callID}+ if args != nil {+ call.Arguments = args+ }+ if result != nil {+ call.Result = result+ }+ ev.GenAI = &schema.GenAIInfo{+ Operation: &schema.GenAIOperationInfo{Name: "execute_tool"},+ Tool: &schema.GenAIToolInfo{Name: name, Call: call},+ }+}++var harness2IDNamespace = [16]byte{+ 0x3c, 0x91, 0x0e, 0x7f, 0x22, 0xd4, 0x4b, 0x58,+ 0x9a, 0x13, 0x6e, 0xc0, 0x45, 0xb7, 0x2f, 0xa8,+}++// harness2EventID is a name-based UUID over the event's coordinate in its session: the executor,+// the session id, the source line and what the event is. A re-read of the same stream names every+// event the same way, so a resent line is recognized rather than duplicated.+func harness2EventID(dedupID string) string {+ digest := sha1.New()+ digest.Write(harness2IDNamespace[:])+ io.WriteString(digest, dedupID)+ var id [16]byte+ copy(id[:], digest.Sum(nil))+ id[6] = (id[6] & 0x0f) | 0x50+ id[8] = (id[8] & 0x3f) | 0x80+ out := make([]byte, 32)+ hex.Encode(out, id[:])+ return fmt.Sprintf("%s-%s-%s-%s-%s", out[0:8], out[8:12], out[12:16], out[16:20], out[20:32])+}++func object(m map[string]interface{}, key string) map[string]interface{} {+ value, _ := m[key].(map[string]interface{})+ return value+}++func str(m map[string]interface{}, keys ...string) string {+ for _, key := range keys {+ if value, ok := m[key].(string); ok && strings.TrimSpace(value) != "" {+ return value+ }+ }+ return ""+}++// num reads a JSON number as an int64, refusing what is not a whole, finite count.+func num(m map[string]interface{}, key string) (int64, bool) {+ switch value := m[key].(type) {+ case float64:+ if math.IsNaN(value) || math.IsInf(value, 0) || value != math.Trunc(value) || math.Abs(value) > 1<<53 {+ return 0, false+ }+ return int64(value), true+ case json.Number:+ n, err := value.Int64()+ return n, err == nil+ case string:+ n, err := strconv.ParseInt(strings.TrimSpace(value), 10, 64)+ return n, err == nil+ default:+ return 0, false+ }+}++func positive(m map[string]interface{}, key string) int64 {+ if value, ok := num(m, key); ok && value > 0 {+ return value+ }+ return 0+}++func ptr(value int64) *int64 { return &value }++func truncate(s string, n int) string {+ if len(s) <= n {+ return s+ }+ cut := n+ for cut > 0 && (s[cut]&0xc0) == 0x80 {+ cut--+ }+ return s[:cut]+}++func firstNonEmpty(values ...string) string {+ for _, value := range values {+ if strings.TrimSpace(value) != "" {+ return value+ }+ }+ return ""+}++func outputText(value interface{}) string {+ switch v := value.(type) {+ case nil:+ return ""+ case string:+ return v+ default:+ data, err := json.Marshal(v)+ if err != nil {+ return ""+ }+ return string(data)+ }+}++func sha256Hex(value string) string {+ sum := sha256.Sum256([]byte(value))+ return hex.EncodeToString(sum[:])+}diff --git a/cli/beacon/internal/harness2session/mapper_test.go b/cli/beacon/internal/harness2session/mapper_test.gonew file mode 100644index 00000000..6eaf8c25--- /dev/null+++ b/cli/beacon/internal/harness2session/mapper_test.go@@ -0,0 +1,302 @@+package harness2session++import (+ "encoding/json"+ "strconv"+ "strings"+ "testing"+ "time"++ "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/schema"+ "github.com/asymptote-labs/agent-beacon/pkg/asymptoteobserve"+)++func mapFixture(t *testing.T, rel string, opts MapOptions) []MappedEvent {+ t.Helper()+ ref, ok := listFixtures(t, copyRuns(t))[rel]+ if !ok {+ t.Fatalf("%s not listed", rel)+ }+ store := &Store{Roots: []string{ref.Root}}+ trajectory, _, err := store.Read(ref)+ if err != nil {+ t.Fatal(err)+ }+ return MapTrajectory(trajectory, opts)+}++func actions(events []MappedEvent) []string {+ out := make([]string, len(events))+ for i, item := range events {+ out[i] = item.Event.Event.Action+ }+ return out+}++// Every event a collector writes must survive the trip through JSON as the release-contract shape.+func roundTrip(t *testing.T, ev schema.Event) map[string]interface{} {+ t.Helper()+ data, err := json.Marshal(ev)+ if err != nil {+ t.Fatal(err)+ }+ var out map[string]interface{}+ if err := json.Unmarshal(data, &out); err != nil {+ t.Fatal(err)+ }+ return out+}++func TestMapOpenCodeStream(t *testing.T) {+ events := mapFixture(t, jbRel, MapOptions{})+ want := []string{"session.started", "agent.reasoning", "file.read", "command.executed", "file.modified", "token.usage", "agent.message", "token.usage"}+ if got := actions(events); strings.Join(got, ",") != strings.Join(want, ",") {+ t.Fatalf("actions = %v\nwant %v", got, want)+ }+ for _, item := range events {+ ev := item.Event+ if ev.Harness.Name != Harness || ev.Harness.CollectionMethod != schema.CollectionMethodPoll {+ t.Fatalf("%s harness = %+v, want harness2 by poll", ev.Event.Action, ev.Harness)+ }+ // The collector writes the canonical name, so a normalizing reader leaves it alone.+ if got := asymptoteobserve.NormalizeHarnessName(ev.Harness.Name); got != Harness {+ t.Fatalf("NormalizeHarnessName(%q) = %q", ev.Harness.Name, got)+ }+ if ev.Session == nil || ev.Session.ID != "ses_7a1f00c0ffee" {+ t.Fatalf("%s session = %+v, want OpenCode's session id", ev.Event.Action, ev.Session)+ }+ ctx, _ := ev.Raw["harness2"].(map[string]interface{})+ if ctx["executor"] != "opencode" || ctx["benchmark"] != "jb" || ctx["task_id"] != "accountant/task3" ||+ ctx["label"] != "h2-final-1" || ctx["attempt"] != 2 || ctx["trajectory"] != jbRel {+ t.Fatalf("%s harness2 context = %v", ev.Event.Action, ctx)+ }+ if ev.Event.ID == "" || ev.Timestamp == "" {+ t.Fatalf("%s has no id or timestamp: %+v", ev.Event.Action, ev.Event)+ }+ }+ if events[0].Event.Event.Fidelity != schema.FidelityInferred {+ t.Fatalf("session.started fidelity = %q; the stream never states a start, so it is inferred", events[0].Event.Event.Fidelity)+ }++ read := events[2].Event+ if read.File == nil || read.File.Operation != "read" || !strings.HasSuffix(read.File.Path, "TASK_INSTRUCTIONS.txt") || read.File.Language != "txt" {+ t.Fatalf("file.read = %+v", read.File)+ }+ // A tool event is stamped when the tool finished, not when the line was printed.+ if read.Timestamp != schema.FormatTimestamp(timeMS(1793004902250)) {+ t.Fatalf("file.read timestamp = %s", read.Timestamp)+ }+ if read.GenAI == nil || read.GenAI.Tool == nil || read.GenAI.Tool.Call.ID != "call_read_1" || read.GenAI.Tool.Call.Result != "Prepare the Q3 reconciliation." {+ t.Fatalf("file.read gen_ai = %+v", read.GenAI)+ }++ cmd := events[3].Event+ if cmd.Command == nil || cmd.Command.Command != "python3 scripts/reconcile.py" || cmd.Command.Output != "wrote output/q3.xlsx\n" ||+ cmd.Command.ExitCode == nil || *cmd.Command.ExitCode != 0 {+ t.Fatalf("command = %+v", cmd.Command)+ }+ if cmd.Content == nil || !cmd.Content.Included {+ t.Fatalf("command content = %+v, want the output retained", cmd.Content)+ }++ edit := events[4].Event+ if edit.Error == nil || edit.Error.Type != "tool_error" || edit.Raw["error_message"] != "oldString not found in content" {+ t.Fatalf("failed edit error = %+v raw=%v", edit.Error, edit.Raw["error_message"])+ }+ if edit.File == nil || edit.File.Operation != "modify" || edit.File.Diff != "y" {+ t.Fatalf("failed edit file = %+v", edit.File)+ }++ usage := events[5].Event.GenAI.Usage+ // OpenCode keeps reasoning out of output when the provider total says so (1200+200+400+40 =+ // 1840); gen_ai counts it inside.+ if *usage.InputTokens != 1200 || *usage.OutputTokens != 440 || *usage.Reasoning.OutputTokens != 40 ||+ *usage.CacheRead.InputTokens != 200 || usage.CacheCreation != nil || *usage.CostUSD != 0.0123 {+ t.Fatalf("step usage = %s", mustJSON(t, usage))+ }+ if second := events[7].Event.GenAI.Usage; *second.OutputTokens != 100 || second.Reasoning != nil {+ t.Fatalf("second step usage = %s", mustJSON(t, second))+ }+ if msg := events[6].Event; msg.GenAI == nil || msg.GenAI.Output == nil || !strings.Contains(mustJSON(t, msg.GenAI.Output), "output/q3.xlsx") {+ t.Fatalf("agent.message = %+v", msg.GenAI)+ }+ for _, item := range events {+ roundTrip(t, item.Event)+ }+}++// WorkBuddy captures OpenCode's stdout whole, banner lines included, and a session error is the one+// OpenCode event with no part.+func TestMapOpenCodeCapturedStdout(t *testing.T) {+ events := mapFixture(t, wbRel, MapOptions{})+ if got := strings.Join(actions(events), ","); got != "session.started,agent.message,session.error" {+ t.Fatalf("actions = %s", got)+ }+ errEv := events[2].Event+ if errEv.Error == nil || errEv.Error.Type != "APIError" || errEv.Raw["error_message"] != "rate limited" || errEv.Severity != schema.SeverityHigh {+ t.Fatalf("session.error = %+v raw=%v", errEv.Error, errEv.Raw)+ }+ ctx := errEv.Raw["harness2"].(map[string]interface{})+ if ctx["subset"] != "office" || ctx["trial"] != "xlsx-budget__Ab3dE" || ctx["harness_fingerprint"] != "base" || ctx["run_id"] != "wb-office-xlsx-budget-base" {+ t.Fatalf("context = %v", ctx)+ }+}++func TestMapCodexExecStream(t *testing.T) {+ events := mapFixture(t, labRel, MapOptions{})+ want := []string{"session.started", "agent.reasoning", "command.executed", "command.executed",+ "file.created", "file.modified", "file.modified", "mcp.tool_invoked", "agent.message", "token.usage"}+ if got := actions(events); strings.Join(got, ",") != strings.Join(want, ",") {+ t.Fatalf("actions = %v\nwant %v", got, want)+ }+ for _, item := range events {+ if item.Event.Session.ID != "0199f1c2-7d1e-7c41-9a3b-5e2b8d0f6a11" {+ t.Fatalf("session id = %q, want the thread id", item.Event.Session.ID)+ }+ ctx := item.Event.Raw["harness2"].(map[string]interface{})+ if ctx["executor"] != "codex" || ctx["run_id"] != "antitrust-competition__t07__h2-cand-dom-1" || ctx["harness_fingerprint"] != "9f2c4e1a7b3d5f60" {+ t.Fatalf("context = %v", ctx)+ }+ }+ // item.started for the same command is not a second event.+ ok, failed := events[2].Event, events[3].Event+ if ok.Command.Command != "bash -lc 'ls documents'" || *ok.Command.ExitCode != 0 || ok.Error != nil {+ t.Fatalf("completed command = %+v err=%+v", ok.Command, ok.Error)+ }+ if *failed.Command.ExitCode != 1 || failed.Error == nil || failed.Error.Type != "command_failed" {+ t.Fatalf("failed command = %+v err=%+v", failed.Command, failed.Error)+ }+ for i, want := range []struct{ path, op string }{{"/work/memo.md", "create"}, {"/work/draft.md", "modify"}, {"/work/scratch.txt", "delete"}} {+ file := events[4+i].Event.File+ if file.Path != want.path || file.Operation != want.op {+ t.Errorf("file change %d = %+v, want %+v", i, file, want)+ }+ }+ mcp := events[7].Event+ if mcp.MCP == nil || mcp.MCP.Server != "docs" || mcp.MCP.Tool != "search" || mcp.Error != nil {+ t.Fatalf("mcp = %+v err=%+v", mcp.MCP, mcp.Error)+ }+ usage := events[9].Event.GenAI.Usage+ // Codex's input includes its cached input; gen_ai counts them apart.+ if *usage.InputTokens != 2000 || *usage.CacheRead.InputTokens != 3000 || *usage.OutputTokens != 700 || *usage.Reasoning.OutputTokens != 300 {+ t.Fatalf("usage = %s", mustJSON(t, usage))+ }+ // The exec stream has no times; every event takes the file's modification time.+ for _, item := range events {+ if item.Event.Timestamp != events[0].Event.Timestamp {+ t.Fatalf("timestamps differ: %s vs %s", item.Event.Timestamp, events[0].Event.Timestamp)+ }+ }+}++// turn.completed reports the thread's running total, so a two-turn stream must not count the first+// turn twice.+func TestCodexUsageIsDifferencedAcrossTurns(t *testing.T) {+ lines := []Line{+ {Number: 1, Object: obj(t, `{"type":"thread.started","thread_id":"th"}`)},+ {Number: 2, Object: obj(t, `{"type":"turn.completed","usage":{"input_tokens":100,"cached_input_tokens":40,"output_tokens":10,"reasoning_output_tokens":0}}`)},+ {Number: 3, Object: obj(t, `{"type":"turn.completed","usage":{"input_tokens":250,"cached_input_tokens":140,"output_tokens":30,"reasoning_output_tokens":5}}`)},+ // A total that went down is a restarted count, taken whole.+ {Number: 4, Object: obj(t, `{"type":"turn.completed","usage":{"input_tokens":20,"cached_input_tokens":0,"output_tokens":2,"reasoning_output_tokens":0}}`)},+ }+ events := MapTrajectory(Trajectory{Ref: TrajectoryRef{Format: FormatCodexExec, Rel: "x.jsonl"}, Lines: lines}, MapOptions{})+ var got []string+ for _, item := range events {+ if u := item.Event.GenAI; u != nil && u.Usage != nil {+ cache := int64(0)+ if u.Usage.CacheRead != nil {+ cache = *u.Usage.CacheRead.InputTokens+ }+ got = append(got, strings.Join([]string{itoa64(*u.Usage.InputTokens), itoa64(cache), itoa64(*u.Usage.OutputTokens)}, "/"))+ }+ }+ if strings.Join(got, " ") != "60/40/10 50/100/20 20/0/2" {+ t.Fatalf("per-turn usage (input/cache/output) = %v", got)+ }+ // Resuming after the first turn still differences against it.+ resumed := MapTrajectory(Trajectory{Ref: TrajectoryRef{Format: FormatCodexExec, Rel: "x.jsonl"}, Lines: lines[:3]}, MapOptions{MinLine: 2, SkipStarted: true})+ if len(resumed) != 1 || *resumed[0].Event.GenAI.Usage.InputTokens != 50 {+ t.Fatalf("resumed = %v", actions(resumed))+ }+}++// Ids are coordinates in the session, so a second read names every event the same way and a resumed+// read names the events it skips over the same way too.+func TestEventIDsAreStableAndResumeSkipsWrittenLines(t *testing.T) {+ first := mapFixture(t, jbRel, MapOptions{})+ second := mapFixture(t, jbRel, MapOptions{})+ seen := map[string]bool{}+ for i := range first {+ if first[i].Event.Event.ID != second[i].Event.Event.ID {+ t.Fatalf("event %d id changed between reads", i)+ }+ if seen[first[i].Event.Event.ID] {+ t.Fatalf("duplicate id %s", first[i].Event.Event.ID)+ }+ seen[first[i].Event.Event.ID] = true+ }+ resumed := mapFixture(t, jbRel, MapOptions{MinLine: 4, SkipStarted: true})+ if got := strings.Join(actions(resumed), ","); got != "file.modified,token.usage,agent.message,token.usage" {+ t.Fatalf("resumed actions = %s", got)+ }+ if resumed[0].Event.Event.ID != first[4].Event.Event.ID {+ t.Fatal("a resumed read named an event differently")+ }+}++// A Codex run that fails before its thread starts names no session; its events still group under+// one id derived from where the file is, the same on every read.+func TestStreamWithoutASessionIDGetsAStableOne(t *testing.T) {+ lines := []Line{{Number: 1, Object: obj(t, `{"type":"error","message":"401 Unauthorized"}`)}}+ ref := TrajectoryRef{Format: FormatCodexExec, Rel: "lab/results/cx/r1/codex_session.jsonl"}+ a := MapTrajectory(Trajectory{Ref: ref, Lines: lines}, MapOptions{})+ b := MapTrajectory(Trajectory{Ref: ref, Lines: lines}, MapOptions{})+ if len(a) != 1 || a[0].Event.Event.Action != "session.error" || a[0].Event.Error.Type != "stream_error" {+ t.Fatalf("events = %v", actions(a))+ }+ if id := a[0].Event.Session.ID; !strings.HasPrefix(id, "harness2-") || id != b[0].Event.Session.ID {+ t.Fatalf("session id = %q / %q", id, b[0].Event.Session.ID)+ }+}++func TestCodexItemsWithoutAnOutcomeAreSkipped(t *testing.T) {+ lines := []Line{+ {Number: 1, Object: obj(t, `{"type":"thread.started","thread_id":"th"}`)},+ {Number: 2, Object: obj(t, `{"type":"item.started","item":{"id":"i1","type":"command_execution","command":"sleep 999","aggregated_output":"","exit_code":null,"status":"in_progress"}}`)},+ {Number: 3, Object: obj(t, `{"type":"item.completed","item":{"id":"i2","type":"agent_message","text":" "}}`)},+ {Number: 4, Object: obj(t, `{"type":"item.completed","item":{"id":"i3","type":"todo_list","items":[]}}`)},+ {Number: 5, Object: obj(t, `{"type":"item.completed","item":{"id":"i4","type":"some_future_item"}}`)},+ {Number: 6, Object: obj(t, `{"type":"turn.completed","usage":{"input_tokens":0,"cached_input_tokens":0,"output_tokens":0,"reasoning_output_tokens":0}}`)},+ {Number: 7, Object: obj(t, `{"type":"item.completed","item":{"id":"i5","type":"error","message":"model fell back"}}`)},+ {Number: 8, Object: obj(t, `{"type":"turn.failed","error":{"message":"stream disconnected"}}`)},+ }+ events := MapTrajectory(Trajectory{Ref: TrajectoryRef{Format: FormatCodexExec, Rel: "x"}, Lines: lines}, MapOptions{})+ if got := strings.Join(actions(events), ","); got != "session.started,session.error,session.error" {+ t.Fatalf("actions = %s", got)+ }+ if events[1].Event.Severity != schema.SeverityMedium || events[2].Event.Severity != schema.SeverityHigh {+ t.Fatalf("severities = %s, %s; an item warning is not a failed turn", events[1].Event.Severity, events[2].Event.Severity)+ }+}++func obj(t *testing.T, s string) map[string]interface{} {+ t.Helper()+ m, ok := decodeObject([]byte(s))+ if !ok {+ t.Fatalf("bad fixture %s", s)+ }+ return m+}++func mustJSON(t *testing.T, v interface{}) string {+ t.Helper()+ data, err := json.Marshal(v)+ if err != nil {+ t.Fatal(err)+ }+ return string(data)+}++func timeMS(ms int64) time.Time { return time.UnixMilli(ms) }++func itoa64(n int64) string { return strconv.FormatInt(n, 10) }diff --git a/cli/beacon/internal/harness2session/opencode.go b/cli/beacon/internal/harness2session/opencode.gonew file mode 100644index 00000000..cbcab6fb--- /dev/null+++ b/cli/beacon/internal/harness2session/opencode.go@@ -0,0 +1,195 @@+package harness2session++import (+ "path/filepath"+ "strings"++ "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/schema"+ "github.com/asymptote-labs/agent-beacon/pkg/asymptoteobserve"+)++// mapOpenCode maps OpenCode's `run --format json` stream. Each line is+//+// {"type": <event>, "timestamp": <ms>, "sessionID": ..., "part": {...}}+//+// where the part is the message part OpenCode stores, in the shape it stores it+// (packages/opencode/src/cli/cmd/run.ts emits it as is). Only finished parts are emitted: a text or+// reasoning part once it has an end time, a tool part once it completed or failed, and a step's+// usage when the step finishes. The prompt is not in the stream -- it is an argument to+// `opencode run` -- so a rollout here has no prompt.submitted.+func (m *mapper) mapOpenCode(lines []Line) {+ for i, line := range lines {+ ts, _ := num(line.Object, "timestamp")+ if i == 0 {+ m.emitStarted(line, ts, schema.FidelityInferred)+ }+ if !m.emits(line) {+ continue+ }+ part := object(line.Object, "part")+ switch str(line.Object, "type") {+ case "text":+ m.emitText(line, ts, "agent.message", "assistant", str(part, "text"))+ case "reasoning":+ m.emitText(line, ts, "agent.reasoning", "reasoning", str(part, "text"))+ case "tool_use":+ m.emitOpenCodeTool(line, ts, part)+ case "step_finish":+ m.emitUsage(line, ts, openCodeUsage(part))+ case "error":+ errObj := object(line.Object, "error")+ m.emitError(line, ts, schema.SeverityHigh, str(errObj, "name"), firstNonEmpty(str(object(errObj, "data"), "message"), str(errObj, "message")))+ }+ }+}++func (m *mapper) emitOpenCodeTool(line Line, ts int64, part map[string]interface{}) {+ state := object(part, "state")+ name := firstNonEmpty(str(part, "tool"), "unknown")+ status := str(state, "status")+ if end := positive(object(state, "time"), "end"); end > 0 {+ ts = end+ }+ args := object(state, "input")+ action, category, severity, message := classifyOpenCodeTool(name, status)+ ev := m.base(line, ts, action, category, severity, schema.FidelityObserved, message)+ ev.Tool = &schema.ToolInfo{Name: name}+ if path := argString(args, "filePath", "file_path", "path"); path != "" {+ ev.Tool.Path = path+ }+ if command := argString(args, "command", "cmd"); command != "" {+ ev.Tool.Command = command+ }+ var result interface{}+ if output, ok := state["output"]; ok && output != nil {+ result = output+ }+ var callArgs interface{}+ if len(args) > 0 {+ callArgs = args+ }+ toolCall(&ev, name, firstNonEmpty(str(part, "callID"), str(part, "id")), callArgs, result)+ output := outputText(result)+ switch category {+ case "command":+ info := &schema.CommandInfo{Command: ev.Tool.Command, Output: output}+ if code, ok := num(object(state, "metadata"), "exit"); ok {+ exit := int(code)+ info.ExitCode = &exit+ }+ if output != "" {+ ev.Content = asymptoteobserve.RetainedContent(output, asymptoteobserve.DefaultStringLimit)+ }+ ev.Command = info+ case "file":+ ev.File = openCodeFile(action, ev.Tool.Path, args, object(state, "metadata"))+ if ev.File.Diff != "" {+ ev.Content = asymptoteobserve.RetainedContent(ev.File.Diff, asymptoteobserve.DefaultStringLimit)+ }+ case "mcp":+ ev.MCP = &schema.MCPInfo{Tool: name}+ }+ if status == "error" {+ ev.Error = &schema.ErrorInfo{Type: "tool_error"}+ if text := str(state, "error"); text != "" {+ ev.Raw["error_message"] = truncate(text, 2000)+ }+ }+ m.append(line, "tool."+firstNonEmpty(str(part, "callID"), str(part, "id")), ev)+}++// classifyOpenCodeTool sorts OpenCode's tool names into Beacon's event families the way the OpenCode+// session collector does (opencodesession.classifyTool), so a tool call reads the same whichever+// path recorded it.+func classifyOpenCodeTool(name, status string) (action, category string, severity schema.Severity, message string) {+ lower := strings.ToLower(strings.TrimSpace(name))+ switch {+ case nameHasToken(lower, "mcp"):+ return "mcp.tool_invoked", "mcp", schema.SeverityInfo, "Harness² executor MCP tool invoked"+ case nameHasToken(lower, "bash") || nameHasToken(lower, "shell") || nameHasToken(lower, "terminal") || lower == "powershell":+ return "command.executed", "command", schema.SeverityInfo, "Harness² executor command executed"+ case nameHasToken(lower, "read") || nameHasToken(lower, "view") || nameHasToken(lower, "list") || nameHasToken(lower, "grep") || nameHasToken(lower, "glob"):+ return "file.read", "file", schema.SeverityInfo, "Harness² executor file read"+ case nameHasToken(lower, "write") || nameHasToken(lower, "create"):+ return "file.created", "file", schema.SeverityInfo, "Harness² executor file created"+ case nameHasToken(lower, "edit") || nameHasToken(lower, "patch") || lower == "applypatch" || lower == "multiedit":+ return "file.modified", "file", schema.SeverityInfo, "Harness² executor file modified"+ case status == "error":+ return "tool.failed", "tool", schema.SeverityHigh, "Harness² executor tool failed"+ default:+ return "tool.completed", "tool", schema.SeverityInfo, "Harness² executor tool completed"+ }+}++func openCodeFile(action, path string, args, metadata map[string]interface{}) *schema.FileInfo {+ operation := "modify"+ switch action {+ case "file.read":+ operation = "read"+ case "file.created":+ operation = "create"+ }+ file := &schema.FileInfo{Path: path, Operation: operation}+ if path != "" {+ file.Language = strings.TrimPrefix(filepath.Ext(path), ".")+ }+ if operation != "read" {+ diff := firstNonEmpty(str(metadata, "diff"), argString(args, "diff", "patch", "patchText"), argString(args, "content", "newString", "new_string"))+ if diff != "" {+ file.Diff = diff+ file.DiffBytes = len(diff)+ file.DiffHash = sha256Hex(diff)+ }+ }+ return file+}++// openCodeUsage reads a finished step's tokens and cost. OpenCode's step usage has the shape it+// stores on messages, so output follows the rule the OpenCode session collector documents on+// opencodesession.TokenUsage.outputTokens: since v1.3.16 OpenCode stores output without reasoning,+// and gen_ai.usage counts reasoning inside output, so it is added back when the provider total says+// output left it out.+func openCodeUsage(part map[string]interface{}) *schema.GenAIUsageInfo {+ tokens := object(part, "tokens")+ cache := object(tokens, "cache")+ input, output, reasoning := positive(tokens, "input"), positive(tokens, "output"), positive(tokens, "reasoning")+ cacheRead, cacheWrite, total := positive(cache, "read"), positive(cache, "write"), positive(tokens, "total")+ if reasoning > 0 && (reasoning > output || (total > 0 && total >= input+cacheRead+cacheWrite+output+reasoning)) {+ output += reasoning+ }+ usage := &schema.GenAIUsageInfo{}+ if input > 0 {+ usage.InputTokens = ptr(input)+ }+ if output > 0 {+ usage.OutputTokens = ptr(output)+ }+ if reasoning > 0 {+ usage.Reasoning = &schema.GenAIUsageReasoningInfo{OutputTokens: ptr(reasoning)}+ }+ if cacheRead > 0 {+ usage.CacheRead = &schema.GenAIUsageCacheReadInfo{InputTokens: ptr(cacheRead)}+ }+ if cacheWrite > 0 {+ usage.CacheCreation = &schema.GenAIUsageCacheCreationInfo{InputTokens: ptr(cacheWrite)}+ }+ if cost, ok := part["cost"].(float64); ok && cost > 0 {+ usage.CostUSD = &cost+ }+ return usage+}++func argString(args map[string]interface{}, keys ...string) string {+ return str(args, keys...)+}++func nameHasToken(name, token string) bool {+ for _, part := range strings.FieldsFunc(name, func(r rune) bool {+ return !(r >= 'a' && r <= 'z') && !(r >= '0' && r <= '9')+ }) {+ if part == token {+ return true+ }+ }+ return false+}diff --git a/cli/beacon/internal/harness2session/recorded_test.go b/cli/beacon/internal/harness2session/recorded_test.gonew file mode 100644index 00000000..a90c71dc--- /dev/null+++ b/cli/beacon/internal/harness2session/recorded_test.go@@ -0,0 +1,87 @@+package harness2session++import (+ "os"+ "path/filepath"+ "strings"+ "testing"+)++// mapRecorded maps a stream recorded from a real executor run, placed where a benchmark would put it.+func mapRecorded(t *testing.T, name, rel string) []MappedEvent {+ t.Helper()+ data, err := os.ReadFile(filepath.Join("testdata", "recorded", name))+ if err != nil {+ t.Fatal(err)+ }+ root := filepath.Join(t.TempDir(), "runs")+ path := filepath.Join(root, filepath.FromSlash(rel))+ if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {+ t.Fatal(err)+ }+ if err := os.WriteFile(path, data, 0o644); err != nil {+ t.Fatal(err)+ }+ ref, ok := listFixtures(t, root)[rel]+ if !ok {+ t.Fatalf("recorded %s was not recognized as a trajectory", name)+ }+ trajectory, stats, err := (&Store{}).Read(ref)+ if err != nil || stats.Malformed != 0 || stats.PartialTail {+ t.Fatalf("read: stats=%+v err=%v", stats, err)+ }+ return MapTrajectory(trajectory, MapOptions{})+}++// codex-exec-0.144.6.jsonl is the stdout of the Codex release Harness² pins (config.CODEX_VERSION),+// run the way its reference argv runs it -- `codex exec --json --ephemeral+// --dangerously-bypass-approvals-and-sandbox` under a CODEX_HOME naming a custom provider -- against+// a local stand-in for the Responses endpoint that asked for one shell command and then answered.+// Two model calls made the turn, so its usage is their sum: turn.completed reports the thread total.+func TestRecordedCodexExecStream(t *testing.T) {+ events := mapRecorded(t, "codex-exec-0.144.6.jsonl", "lab/results/cx/recorded/codex_session.jsonl")+ if got := strings.Join(actions(events), ","); got != "session.started,session.error,command.executed,agent.message,token.usage" {+ t.Fatalf("actions = %s", got)+ }+ if id := events[0].Event.Session.ID; id != "01a1769f-fbe7-72b3-a4d2-a9b94a6e6dc4" {+ t.Fatalf("session id = %q", id)+ }+ // Codex reports a missing model-metadata entry as an error item and carries on.+ if events[1].Event.Error.Type != "item_error" || !strings.Contains(events[1].Event.Raw["error_message"].(string), "Model metadata") {+ t.Fatalf("warning = %+v %v", events[1].Event.Error, events[1].Event.Raw)+ }+ cmd := events[2].Event.Command+ if cmd.Command != "/bin/bash -lc 'echo harness2-real-run'" || cmd.Output != "harness2-real-run\n" || *cmd.ExitCode != 0 {+ t.Fatalf("command = %+v", cmd)+ }+ usage := events[4].Event.GenAI.Usage+ if *usage.InputTokens != 1201 || *usage.CacheRead.InputTokens != 800 || *usage.OutputTokens != 100 || *usage.Reasoning.OutputTokens != 20 {+ t.Fatalf("usage = %s", mustJSON(t, usage))+ }+}++// opencode-run-1.14.18.jsonl is the stdout of the OpenCode release Harness² pins (v1.14.18), run as+// its JobBench runner runs it -- `opencode run <prompt> --dir <ws> --model <provider/model> --title+// <task | label | aN> --format json` with OPENCODE_PERMISSION fencing the workspace -- against a local+// stand-in for an OpenAI-compatible provider that asked for one bash call and then answered. OpenCode+// reports each step's own usage, with cached input already taken out of input.+func TestRecordedOpenCodeRunStream(t *testing.T) {+ events := mapRecorded(t, "opencode-run-1.14.18.jsonl", "jb/dataset/p/task1/model_traj/l/p_task1_l_attempt1_x.jsonl")+ if got := strings.Join(actions(events), ","); got != "session.started,command.executed,token.usage,agent.message,token.usage" {+ t.Fatalf("actions = %s", got)+ }+ if id := events[0].Event.Session.ID; id != "ses_e895c12c2ffeRtDssTNzjq5b5M" {+ t.Fatalf("session id = %q", id)+ }+ cmd := events[1].Event+ if cmd.Command.Command != "echo harness2-real-run" || cmd.Command.Output != "harness2-real-run\n" || *cmd.Command.ExitCode != 0 ||+ cmd.GenAI.Tool.Call.ID != "call_bash_1" {+ t.Fatalf("command = %+v call=%+v", cmd.Command, cmd.GenAI.Tool.Call)+ }+ for i, want := range map[int]int64{2: 600, 4: 601} {+ usage := events[i].Event.GenAI.Usage+ if *usage.InputTokens != want || *usage.CacheRead.InputTokens != 300 || *usage.OutputTokens != 40 || usage.CostUSD != nil {+ t.Fatalf("step usage %d = %s", i, mustJSON(t, usage))+ }+ }+}diff --git a/cli/beacon/internal/harness2session/store.go b/cli/beacon/internal/harness2session/store.gonew file mode 100644index 00000000..f3d46b30--- /dev/null+++ b/cli/beacon/internal/harness2session/store.go@@ -0,0 +1,452 @@+package harness2session++import (+ "bufio"+ "bytes"+ "encoding/json"+ "errors"+ "fmt"+ "io"+ "io/fs"+ "os"+ "path/filepath"+ "regexp"+ "sort"+ "strconv"+ "strings"++ "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/sessionfs"+)++const (+ // MaxTrajectoryBytes bounds the file a sweep will read. A rollout's stream is megabytes at+ // most; a file far past that is not a trajectory worth holding in memory.+ MaxTrajectoryBytes = 256 << 20+ // MaxLineBytes bounds one event. An OpenCode tool part carries its whole output, so the limit+ // is generous, and a longer line is skipped as malformed rather than read.+ MaxLineBytes = 16 << 20+ // maxWalkDepth bounds how deep under a runs root the walk looks. The deepest layout Harness²+ // writes (WorkBuddy's trials) is under ten levels.+ maxWalkDepth = 12+ // sniffLines is how many lines the format check reads before giving up on a file. An OpenCode+ // stream captured from stdout can open with a few lines of banner before its first event.+ sniffLines = 64+ // markerName is the completion marker Harness² writes beside a finished rollout+ // (harness2/core/runmark.py).+ markerName = ".adapt_complete"+)++// skipDirs are directories under a runs root that hold no trajectories but can hold a great deal+// else: the task inputs and deliverables Harness² copies around, the executor's own private state,+// and anything a Python or Node toolchain leaves behind. Not walking them keeps a sweep proportional+// to the number of rollouts rather than to the size of the benchmark data.+var skipDirs = map[string]bool{+ ".git": true, "node_modules": true, "__pycache__": true, ".venv": true, "venv": true,+ ".ocdata": true, ".ocstate": true, "task_folder": true, "model_output": true, "documents": true,+ "dependencies": true,+}++// Store is the set of runs roots a sweep reads.+type Store struct {+ Roots []string+}++// DefaultRunsDirs is where Harness² writes when nothing says otherwise: $HARNESS2_RUNS_DIR when it+// is set, as Harness² itself reads it, and otherwise the runs directory of a checkout at+// ~/harness2. Harness² defaults to runs/ inside its own checkout, which can be anywhere, so a+// checkout elsewhere is named with --runs-dir.+func DefaultRunsDirs() []string {+ if dir := strings.TrimSpace(os.Getenv("HARNESS2_RUNS_DIR")); dir != "" {+ return []string{expandHome(dir)}+ }+ home, err := os.UserHomeDir()+ if err != nil || strings.TrimSpace(home) == "" {+ return nil+ }+ return []string{filepath.Join(home, "harness2", "runs")}+}++// NewStore returns a store over roots, or over DefaultRunsDirs when none are given. Roots are made+// absolute and deduplicated, so the same directory named two ways is read once.+func NewStore(roots []string) (*Store, error) {+ if len(roots) == 0 {+ roots = DefaultRunsDirs()+ }+ seen := map[string]bool{}+ store := &Store{}+ for _, root := range roots {+ root = strings.TrimSpace(root)+ if root == "" {+ continue+ }+ abs, err := filepath.Abs(expandHome(root))+ if err != nil {+ return nil, fmt.Errorf("Harness² runs directory %s: %w", root, err)+ }+ abs = filepath.Clean(abs)+ if seen[abs] {+ continue+ }+ seen[abs] = true+ store.Roots = append(store.Roots, abs)+ }+ return store, nil+}++// Exists reports whether any root is a directory.+func (s *Store) Exists() bool {+ for _, root := range s.Roots {+ if info, err := sessionfs.Stat(root); err == nil && info.IsDir() {+ return true+ }+ }+ return false+}++// List returns every trajectory under the store's roots, oldest first. A root that does not exist+// contributes nothing; one that cannot be walked is an error.+func (s *Store) List() ([]TrajectoryRef, error) {+ var refs []TrajectoryRef+ seen := map[string]bool{}+ for _, root := range s.Roots {+ found, err := listRoot(root)+ if err != nil {+ return refs, err+ }+ for _, ref := range found {+ // Nested roots (a runs directory and one of its benchmark directories) would otherwise+ // list the same file twice, under two cursors, and write its events twice.+ if seen[ref.Path] {+ continue+ }+ seen[ref.Path] = true+ refs = append(refs, ref)+ }+ }+ sort.Slice(refs, func(i, j int) bool {+ if refs[i].ModTimeUnixMS != refs[j].ModTimeUnixMS {+ return refs[i].ModTimeUnixMS < refs[j].ModTimeUnixMS+ }+ return refs[i].Path < refs[j].Path+ })+ return refs, nil+}++func listRoot(root string) ([]TrajectoryRef, error) {+ info, err := sessionfs.Stat(root)+ if err != nil {+ if errors.Is(err, os.ErrNotExist) {+ return nil, nil+ }+ return nil, err+ }+ if !info.IsDir() {+ return nil, fmt.Errorf("Harness² runs directory %s is not a directory", root)+ }+ var refs []TrajectoryRef+ err = sessionfs.WalkDir(root, func(path string, d fs.DirEntry, walkErr error) error {+ if walkErr != nil {+ if path == root {+ return walkErr+ }+ // One unreadable directory -- a container left it root-owned, a refused path under+ // confinement -- loses only what is inside it.+ if d != nil && d.IsDir() {+ return fs.SkipDir+ }+ return nil+ }+ rel, err := filepath.Rel(root, path)+ if err != nil {+ return nil+ }+ if d.IsDir() {+ if path == root {+ return nil+ }+ if skipDirs[d.Name()] || strings.Count(filepath.ToSlash(rel), "/") >= maxWalkDepth {+ return fs.SkipDir+ }+ return nil+ }+ if !d.Type().IsRegular() || !candidateName(d.Name()) {+ return nil+ }+ fi, err := d.Info()+ if err != nil || fi.Size() == 0 || fi.Size() > MaxTrajectoryBytes {+ return nil+ }+ format, err := sniffFormat(path)+ if err != nil || format == "" {+ return nil+ }+ slashRel := filepath.ToSlash(rel)+ refs = append(refs, TrajectoryRef{+ Path: path,+ Root: root,+ Rel: slashRel,+ Format: format,+ SizeBytes: fi.Size(),+ ModTimeUnixMS: fi.ModTime().UnixMilli(),+ Context: runContext(root, path, slashRel),+ })+ return nil+ })+ return refs, err+}++// candidateName is the cheap filter ahead of reading a file: JSONL, plus the one name WorkBuddy+// gives OpenCode's captured stdout.+func candidateName(name string) bool {+ return strings.HasSuffix(name, ".jsonl") || name == "oc-output.txt"+}++var openCodeEventTypes = map[string]bool{+ "step_start": true, "step_finish": true, "text": true, "reasoning": true, "tool_use": true, "error": true,+}++var codexExecEventTypes = map[string]bool{+ "thread.started": true, "turn.started": true, "turn.completed": true, "turn.failed": true,+ "item.started": true, "item.updated": true, "item.completed": true,+}++// sniffFormat reads the first event of a file and says which stream it is, or "" for a file that is+// not a trajectory. The first JSON object decides: lines before it that are not JSON are skipped,+// since captured stdout can open with a banner, but a file whose first object is something else --+// a result record, a judge's output -- is not read further.+func sniffFormat(path string) (Format, error) {+ f, err := sessionfs.Open(path)+ if err != nil {+ return "", err+ }+ defer f.Close()+ br := bufio.NewReaderSize(f, 64*1024)+ for i := 0; i < sniffLines; i++ {+ line, partial, oversized, err := readLine(br)+ if err != nil && !errors.Is(err, io.EOF) {+ return "", err+ }+ // A line without its newline is still being written; it decides nothing until it is whole.+ if !oversized && !partial {+ if format, decided := classifyFirstLine(line); decided {+ return format, nil+ }+ }+ if errors.Is(err, io.EOF) || partial {+ return "", nil+ }+ }+ return "", nil+}++func classifyFirstLine(line []byte) (Format, bool) {+ line = bytes.TrimSpace(line)+ if len(line) == 0 || line[0] != '{' {+ return "", false+ }+ var head struct {+ Type string `json:"type"`+ SessionID json.RawMessage `json:"sessionID"`+ }+ if err := json.Unmarshal(line, &head); err != nil {+ // A line that opens like JSON and does not parse is a fragment of something else. The+ // stream formats both write one whole object per line, so this is not one of them.+ return "", true+ }+ switch {+ case openCodeEventTypes[head.Type] && len(head.SessionID) > 0:+ return FormatOpenCode, true+ case codexExecEventTypes[head.Type]:+ return FormatCodexExec, true+ case head.Type == "error":+ // Codex writes a bare {"type":"error"} for a stream error; with no session id it is not+ // OpenCode's, and a stream that fails before its thread starts has nothing else to map.+ return FormatCodexExec, true+ default:+ return "", true+ }+}++// Read decodes every complete line of a trajectory.+func (s *Store) Read(ref TrajectoryRef) (Trajectory, Stats, error) {+ f, err := sessionfs.Open(ref.Path)+ if err != nil {+ return Trajectory{Ref: ref}, Stats{}, err+ }+ defer f.Close()+ return decodeTrajectory(ref, f)+}++func decodeTrajectory(ref TrajectoryRef, r io.Reader) (Trajectory, Stats, error) {+ out := Trajectory{Ref: ref}+ var stats Stats+ br := bufio.NewReaderSize(r, 64*1024)+ lineNo := 0+ for {+ line, partial, oversized, err := readLine(br)+ if err != nil {+ if errors.Is(err, io.EOF) {+ if partial {+ stats.PartialTail = true+ }+ break+ }+ return out, stats, fmt.Errorf("read %s: %w", ref.Path, err)+ }+ lineNo+++ out.LineCount = lineNo+ if oversized {+ stats.Malformed+++ continue+ }+ trimmed := bytes.TrimSpace(line)+ if len(trimmed) == 0 {+ continue+ }+ object, ok := decodeObject(trimmed)+ if !ok {+ stats.Malformed+++ continue+ }+ out.Lines = append(out.Lines, Line{Number: lineNo, Object: object})+ }+ return out, stats, nil+}++// readLine returns one complete line without its trailing newline. A trailing fragment with no+// newline is returned with partial set and io.EOF, so the caller leaves it for a later read; a line+// longer than MaxLineBytes is consumed and reported as oversized so a corrupt file cannot dictate+// memory use.+func readLine(br *bufio.Reader) (line []byte, partial, oversized bool, err error) {+ var buf []byte+ for {+ chunk, readErr := br.ReadSlice('\n')+ if oversized || len(buf)+len(chunk) > MaxLineBytes {+ oversized = true+ buf = nil+ } else {+ buf = append(buf, chunk...)+ }+ if errors.Is(readErr, bufio.ErrBufferFull) {+ continue+ }+ if errors.Is(readErr, io.EOF) {+ if oversized {+ return nil, false, true, nil+ }+ if len(buf) > 0 {+ return buf, true, false, io.EOF+ }+ return nil, false, false, io.EOF+ }+ if readErr != nil {+ return nil, false, false, readErr+ }+ break+ }+ if oversized {+ return nil, false, true, nil+ }+ return bytes.TrimSuffix(bytes.TrimSuffix(buf, []byte("\n")), []byte("\r")), false, false, nil+}++var (+ attemptPattern = regexp.MustCompile(`_attempt(\d+)_`)+ wbSubsets = map[string]bool{"office": true, "web": true, "code": true}+ benchmarks = map[string]bool{"jb": true, "lab": true, "wb": true}+)++// runContext reads what the layouts in the package comment say about a rollout from where its file+// sits, and the run id and harness fingerprint from the completion marker beside it.+func runContext(root, path, rel string) RunContext {+ var ctx RunContext+ segments := strings.Split(rel, "/")+ dirs := segments[:len(segments)-1]+ if len(dirs) > 0 && benchmarks[dirs[0]] {+ ctx.Benchmark = dirs[0]+ }+ switch ctx.Benchmark {+ case "jb":+ if i := indexOf(dirs, "model_traj"); i >= 2 {+ ctx.TaskID = dirs[i-2] + "/" + dirs[i-1]+ if i+1 < len(dirs) {+ ctx.Label = dirs[i+1]+ }+ }+ if m := attemptPattern.FindStringSubmatch(segments[len(segments)-1]); m != nil {+ ctx.Attempt, _ = strconv.Atoi(m[1])+ }+ case "lab":+ if len(dirs) >= 4 && dirs[1] == "results" {+ ctx.RunID = dirs[3]+ }+ case "wb":+ if len(dirs) > 1 && wbSubsets[dirs[1]] {+ ctx.Subset = dirs[1]+ }+ if i := lastIndexOf(dirs, "agent"); i >= 1 {+ ctx.Trial = dirs[i-1]+ }+ }+ // The marker sits in the run directory: the file's own directory for LAB, its parent for a+ // WorkBuddy trial's agent/ stream. Looking no higher keeps one run's marker from being read as+ // another's.+ dir := filepath.Dir(path)+ for i := 0; i < 2 && strings.HasPrefix(dir, root) && dir != root; i++ {+ if marker, ok := readMarker(filepath.Join(dir, markerName)); ok {+ if ctx.RunID == "" {+ ctx.RunID = marker.RunID+ }+ ctx.HarnessFingerprint = marker.Harness+ break+ }+ dir = filepath.Dir(dir)+ }+ return ctx+}++type completionMarker struct {+ RunID string `json:"run_id"`+ Harness string `json:"harness"`+}++func readMarker(path string) (completionMarker, bool) {+ data, err := sessionfs.ReadFile(path)+ if err != nil || len(data) > 64*1024 {+ return completionMarker{}, false+ }+ var marker completionMarker+ if err := json.Unmarshal(data, &marker); err != nil {+ return completionMarker{}, false+ }+ return marker, marker.RunID != "" || marker.Harness != ""+}++func indexOf(values []string, want string) int {+ for i, value := range values {+ if value == want {+ return i+ }+ }+ return -1+}++func lastIndexOf(values []string, want string) int {+ for i := len(values) - 1; i >= 0; i-- {+ if values[i] == want {+ return i+ }+ }+ return -1+}++func expandHome(path string) string {+ if path != "~" && !strings.HasPrefix(path, "~/") {+ return path+ }+ home, err := os.UserHomeDir()+ if err != nil || home == "" {+ return path+ }+ return filepath.Join(home, strings.TrimPrefix(path, "~"))+}diff --git a/cli/beacon/internal/harness2session/store_test.go b/cli/beacon/internal/harness2session/store_test.gonew file mode 100644index 00000000..f05cc1c7--- /dev/null+++ b/cli/beacon/internal/harness2session/store_test.go@@ -0,0 +1,215 @@+package harness2session++import (+ "io/fs"+ "os"+ "path/filepath"+ "strings"+ "testing"+)++// copyRuns copies testdata/runs into a fresh directory, so a test can grow, rewrite or remove files+// without touching the fixtures, and returns the copy's root.+func copyRuns(t *testing.T) string {+ t.Helper()+ dst := filepath.Join(t.TempDir(), "runs")+ err := filepath.WalkDir(filepath.Join("testdata", "runs"), func(path string, d fs.DirEntry, err error) error {+ if err != nil {+ return err+ }+ rel, _ := filepath.Rel(filepath.Join("testdata", "runs"), path)+ target := filepath.Join(dst, rel)+ if d.IsDir() {+ return os.MkdirAll(target, 0o755)+ }+ data, err := os.ReadFile(path)+ if err != nil {+ return err+ }+ return os.WriteFile(target, data, 0o644)+ })+ if err != nil {+ t.Fatalf("copy fixtures: %v", err)+ }+ return dst+}++func listFixtures(t *testing.T, root string) map[string]TrajectoryRef {+ t.Helper()+ store, err := NewStore([]string{root})+ if err != nil {+ t.Fatal(err)+ }+ refs, err := store.List()+ if err != nil {+ t.Fatalf("List: %v", err)+ }+ byRel := map[string]TrajectoryRef{}+ for _, ref := range refs {+ byRel[ref.Rel] = ref+ }+ return byRel+}++const (+ jbRel = "jb/dataset/accountant/task3/model_traj/h2-final-1/accountant_task3_h2-final-1_attempt2_20261026T101500.jsonl"+ labRel = "lab/results/cx/antitrust-competition__t07__h2-cand-dom-1/codex_session.jsonl"+ wbRel = "wb/office/jobs/2026-10-26__10-00-00/xlsx-budget__Ab3dE/agent/oc-output.txt"+)++// A runs directory holds far more JSONL than trajectories: graded results, a proposer's CLI output,+// task inputs. Only the executor streams are listed, recognized by their first event rather than by+// name, and the benchmark data Harness² copies around is not walked at all.+func TestListFindsOnlyExecutorTrajectories(t *testing.T) {+ refs := listFixtures(t, copyRuns(t))+ if len(refs) != 3 {+ names := []string{}+ for rel := range refs {+ names = append(names, rel)+ }+ t.Fatalf("listed %d trajectories, want 3: %v", len(refs), names)+ }+ for rel, want := range map[string]Format{jbRel: FormatOpenCode, labRel: FormatCodexExec, wbRel: FormatOpenCode} {+ ref, ok := refs[rel]+ if !ok {+ t.Fatalf("%s was not listed", rel)+ }+ if ref.Format != want {+ t.Errorf("%s format = %q, want %q", rel, ref.Format, want)+ }+ if !filepath.IsAbs(ref.Path) || ref.SizeBytes == 0 || ref.ModTimeUnixMS == 0 {+ t.Errorf("%s ref = %+v, want an absolute path, size and modification time", rel, ref)+ }+ }+}++// Each benchmark's layout names a different part of the rollout, and the completion marker beside a+// run names the harness that produced it.+func TestListReadsRunContextFromTheLayout(t *testing.T) {+ refs := listFixtures(t, copyRuns(t))+ for rel, want := range map[string]RunContext{+ jbRel: {Benchmark: "jb", TaskID: "accountant/task3", Label: "h2-final-1", Attempt: 2},+ labRel: {Benchmark: "lab", RunID: "antitrust-competition__t07__h2-cand-dom-1", HarnessFingerprint: "9f2c4e1a7b3d5f60"},+ wbRel: {Benchmark: "wb", Subset: "office", Trial: "xlsx-budget__Ab3dE", RunID: "wb-office-xlsx-budget-base", HarnessFingerprint: "base"},+ } {+ if got := refs[rel].Context; got != want {+ t.Errorf("%s context = %+v, want %+v", rel, got, want)+ }+ }+}++// A marker above the run directory belongs to some other run, and must not be read as this one's.+func TestRunContextIgnoresAMarkerAboveTheRun(t *testing.T) {+ root := copyRuns(t)+ if err := os.WriteFile(filepath.Join(root, "jb", markerName), []byte(`{"run_id":"elsewhere","harness":"0000"}`), 0o644); err != nil {+ t.Fatal(err)+ }+ if got := listFixtures(t, root)[jbRel].Context; got.RunID != "" || got.HarnessFingerprint != "" {+ t.Fatalf("context = %+v, want no marker fields", got)+ }+}++func TestSniffFormat(t *testing.T) {+ dir := t.TempDir()+ for name, tc := range map[string]struct {+ body string+ want Format+ }{+ "opencode": {`{"type":"step_start","timestamp":1,"sessionID":"ses_1","part":{}}` + "\n", FormatOpenCode},+ "opencode-banner": {"migrating...\n\n" + `{"type":"text","timestamp":1,"sessionID":"ses_1","part":{}}` + "\n", FormatOpenCode},+ "codex": {`{"type":"thread.started","thread_id":"t"}` + "\n", FormatCodexExec},+ "codex-error-only": {`{"type":"error","message":"unauthorized"}` + "\n", FormatCodexExec},+ // An OpenCode-shaped type without a session id is not OpenCode's stream.+ "no-session": {`{"type":"text","part":{}}` + "\n", ""},+ "result": {`{"type":"result","subtype":"success"}` + "\n", ""},+ "not-an-event": {`{"score":1}` + "\n", ""},+ "broken-json": {`{"type":"thread.started"` + "\n", ""},+ "plain-text": {"hello\nworld\n", ""},+ // A first event still being written decides nothing yet; the file is looked at again later.+ "partial": {`{"type":"thread.started","thread_id":"t"}`, ""},+ } {+ path := filepath.Join(dir, name+".jsonl")+ if err := os.WriteFile(path, []byte(tc.body), 0o644); err != nil {+ t.Fatal(err)+ }+ got, err := sniffFormat(path)+ if err != nil {+ t.Fatalf("%s: %v", name, err)+ }+ if got != tc.want {+ t.Errorf("%s: format = %q, want %q", name, got, tc.want)+ }+ }+}++// Two roots where one is inside the other must not list a file twice: two cursors over one file+// would write each of its events twice.+func TestListDeduplicatesNestedRoots(t *testing.T) {+ root := copyRuns(t)+ store, err := NewStore([]string{root, filepath.Join(root, "lab"), root + string(filepath.Separator)})+ if err != nil {+ t.Fatal(err)+ }+ if len(store.Roots) != 2 {+ t.Fatalf("roots = %v, want the runs root once and its lab directory", store.Roots)+ }+ refs, err := store.List()+ if err != nil {+ t.Fatal(err)+ }+ if len(refs) != 3 {+ t.Fatalf("listed %d trajectories, want 3", len(refs))+ }+}++func TestListToleratesAMissingRootAndRefusesAFile(t *testing.T) {+ missing := filepath.Join(t.TempDir(), "nope")+ store, _ := NewStore([]string{missing})+ if refs, err := store.List(); err != nil || len(refs) != 0 {+ t.Fatalf("missing root: refs=%v err=%v, want none and no error", refs, err)+ }+ if store.Exists() {+ t.Fatal("Exists() = true for a missing root")+ }+ file := filepath.Join(t.TempDir(), "runs")+ if err := os.WriteFile(file, []byte("x"), 0o644); err != nil {+ t.Fatal(err)+ }+ store, _ = NewStore([]string{file})+ if _, err := store.List(); err == nil || !strings.Contains(err.Error(), "not a directory") {+ t.Fatalf("file root: err = %v, want not a directory", err)+ }+}++func TestDefaultRunsDirsFollowsHarness2RunsDir(t *testing.T) {+ t.Setenv("HARNESS2_RUNS_DIR", "/srv/h2/runs")+ if got := DefaultRunsDirs(); len(got) != 1 || got[0] != "/srv/h2/runs" {+ t.Fatalf("DefaultRunsDirs() = %v, want [/srv/h2/runs]", got)+ }+ t.Setenv("HARNESS2_RUNS_DIR", "")+ home, _ := os.UserHomeDir()+ if got := DefaultRunsDirs(); len(got) != 1 || got[0] != filepath.Join(home, "harness2", "runs") {+ t.Fatalf("DefaultRunsDirs() = %v, want the checkout under the home directory", got)+ }+}++// A live stream ends in a line the executor is still writing; it is neither counted nor read.+func TestReadLeavesAPartialTailForLater(t *testing.T) {+ path := filepath.Join(t.TempDir(), "s.jsonl")+ body := `{"type":"thread.started","thread_id":"t"}` + "\n" + "not json\n\n" + `{"type":"turn.started"}` + "\n" + `{"type":"item.comp`+ if err := os.WriteFile(path, []byte(body), 0o644); err != nil {+ t.Fatal(err)+ }+ store := &Store{}+ got, stats, err := store.Read(TrajectoryRef{Path: path, Format: FormatCodexExec})+ if err != nil {+ t.Fatal(err)+ }+ if !stats.PartialTail || stats.Malformed != 1 {+ t.Fatalf("stats = %+v, want a partial tail and one malformed line", stats)+ }+ if got.LineCount != 4 || len(got.Lines) != 2 || got.Lines[1].Number != 4 {+ t.Fatalf("read %d lines (%d decoded, last #%d), want 4 lines with 2 decoded and the last numbered 4",+ got.LineCount, len(got.Lines), got.Lines[len(got.Lines)-1].Number)+ }+}diff --git a/cli/beacon/internal/harness2session/testdata/runs/lab/results/cx/antitrust-competition__t07__h2-cand-dom-1/.adapt_complete b/cli/beacon/internal/harness2session/testdata/runs/lab/results/cx/antitrust-competition__t07__h2-cand-dom-1/.adapt_completenew file mode 100644index 00000000..aad40905--- /dev/null+++ b/cli/beacon/internal/harness2session/testdata/runs/lab/results/cx/antitrust-competition__t07__h2-cand-dom-1/.adapt_complete@@ -0,0 +1,7 @@+{+ "at": 1793005000.5,+ "harness": "9f2c4e1a7b3d5f60",+ "output_fingerprint": "abc",+ "run_id": "antitrust-competition__t07__h2-cand-dom-1",+ "version": 1+}diff --git a/cli/beacon/internal/harness2session/testdata/runs/wb/office/jobs/2026-10-26__10-00-00/xlsx-budget__Ab3dE/.adapt_complete b/cli/beacon/internal/harness2session/testdata/runs/wb/office/jobs/2026-10-26__10-00-00/xlsx-budget__Ab3dE/.adapt_completenew file mode 100644index 00000000..c1b4e333--- /dev/null+++ b/cli/beacon/internal/harness2session/testdata/runs/wb/office/jobs/2026-10-26__10-00-00/xlsx-budget__Ab3dE/.adapt_complete@@ -0,0 +1 @@+{"version":1,"run_id":"wb-office-xlsx-budget-base","harness":"base","at":1793006100}diff --git a/cli/beacon/internal/harness2session/testdata/runs/wb/office/jobs/2026-10-26__10-00-00/xlsx-budget__Ab3dE/agent/oc-output.txt b/cli/beacon/internal/harness2session/testdata/runs/wb/office/jobs/2026-10-26__10-00-00/xlsx-budget__Ab3dE/agent/oc-output.txtnew file mode 100644index 00000000..befa0f1e--- /dev/null+++ b/cli/beacon/internal/harness2session/testdata/runs/wb/office/jobs/2026-10-26__10-00-00/xlsx-budget__Ab3dE/agent/oc-output.txt@@ -0,0 +1,5 @@+Performing one time database migration, may take a few minutes...+sqlite-migration:done+{"type":"step_start","timestamp":1793006000000,"sessionID":"ses_wb0001","part":{"id":"p1","sessionID":"ses_wb0001","messageID":"m1","type":"step-start"}}+{"type":"text","timestamp":1793006001000,"sessionID":"ses_wb0001","part":{"id":"p2","sessionID":"ses_wb0001","messageID":"m1","type":"text","text":"Done.","time":{"start":1793006000500,"end":1793006001000}}}+{"type":"error","timestamp":1793006002000,"sessionID":"ses_wb0001","error":{"name":"APIError","data":{"message":"rate limited","isRetryable":true}}}diff --git a/cli/beacon/internal/harness2session/types.go b/cli/beacon/internal/harness2session/types.gonew file mode 100644index 00000000..fcb271f4--- /dev/null+++ b/cli/beacon/internal/harness2session/types.go@@ -0,0 +1,130 @@+// Package harness2session reads the trajectories Harness² (google-research/harness2) records under+// its runs directory and turns them into Beacon endpoint telemetry.+//+// Harness² is not an agent. It is a framework for refining an agent's harness: for each benchmark+// task it runs an executor -- OpenCode or Codex -- several times over the same task with candidate+// harnesses, compares what those executions did, and composes a better harness from the contrast.+// Every one of those executions is a full agent session, and Harness² runs them in places Beacon's+// own integrations for those agents do not see: Codex with its own CODEX_HOME and --ephemeral, so+// no rollout file reaches ~/.codex/sessions, and OpenCode from a source checkout with its event+// stream captured to a file. What Harness² keeps of each execution is that stream, written beside+// the run's other artifacts:+//+// <runs>/jb/dataset/<profession>/<task>/model_traj/<label>/<task>_<label>_attempt<N>_<stamp>.jsonl+// <runs>/lab/results/<oc|cx>/<run-id>/opencode_session.jsonl | codex_session.jsonl+// <runs>/wb/<subset>/.../<trial>/agent/oc-output.txt | codex-output.jsonl+//+// So Beacon reads those files. Two stream formats occur, one per executor: OpenCode's+// `run --format json` events and Codex's `exec --json` thread events. A file is recognized by what+// its first event says it is rather than by its name, because the names differ per benchmark and a+// runs directory holds plenty of other JSONL -- graded results, judge output -- that is not a+// trajectory.+//+// Every event is marked harness.collection_method=poll, as every session-store collector's is:+// Beacon sees an execution once Harness² recorded it, and nothing here can hold or deny a tool call.+package harness2session++import "encoding/json"++// Harness is the canonical harness name every event from this collector carries.+const Harness = "harness2"++// Format names the stream format a trajectory file is written in.+type Format string++const (+ // FormatOpenCode is OpenCode's `opencode run --format json` output: one event per line, each+ // carrying the session id, a millisecond timestamp and the message part it reports.+ FormatOpenCode Format = "opencode_run"+ // FormatCodexExec is Codex's `codex exec --json` output: thread events wrapping thread items.+ // It carries no timestamps.+ FormatCodexExec Format = "codex_exec"+)++// Executor is the agent a format comes from, as Harness² itself names the two (its `--substrate oc`+// and `--substrate cx`) spelled out.+func (f Format) Executor() string {+ switch f {+ case FormatOpenCode:+ return "opencode"+ case FormatCodexExec:+ return "codex"+ default:+ return ""+ }+}++// RunContext is what a trajectory's place in the runs directory says about the execution: which+// benchmark it was for, which task, and which run of it. Every field is optional, because the+// layout differs per benchmark and a hand-run executor's stream need follow none of them.+type RunContext struct {+ // Benchmark is Harness²'s own short name for the benchmark (jb, lab, wb), the first directory+ // under the runs root.+ Benchmark string `json:"benchmark,omitempty"`+ // Subset is the WorkBuddy-Bench subset (office, web, code).+ Subset string `json:"subset,omitempty"`+ // TaskID is the benchmark's task id where the layout names it (JobBench's <profession>/<task>).+ TaskID string `json:"task_id,omitempty"`+ // Label is the harness label the rollout ran under (JobBench names its trajectory directory by it).+ Label string `json:"label,omitempty"`+ // RunID is the run id from the LAB results layout or from the run's completion marker.+ RunID string `json:"run_id,omitempty"`+ // Trial is the WorkBuddy-Bench trial directory the agent stream sits in.+ Trial string `json:"trial,omitempty"`+ // Attempt is the retry number JobBench puts in the file name.+ Attempt int `json:"attempt,omitempty"`+ // HarnessFingerprint is the digest Harness² records in the run's completion marker of the+ // harness tree that produced the rollout, or "base" for the inherited harness. It is what tells+ // two rollouts of the same task apart by the harness they ran with.+ HarnessFingerprint string `json:"harness_fingerprint,omitempty"`+}++// TrajectoryRef is one trajectory file found under a runs root.+type TrajectoryRef struct {+ // Path is the file's absolute path, and the key its cursor is stored under.+ Path string+ // Root is the runs root the file was found under; Rel is its slash-separated path inside it.+ Root string+ Rel string+ // Format is the stream format the file's first event identified.+ Format Format+ SizeBytes int64+ ModTimeUnixMS int64+ Context RunContext+}++// Line is one complete line of a trajectory, decoded as a JSON object.+type Line struct {+ // Number is the physical line number, counting blank and unparseable lines, so a half-written+ // line at the tail of a live stream does not shift the numbers of the lines before it.+ Number int+ Object map[string]interface{}+}++// Trajectory is a decoded trajectory file.+type Trajectory struct {+ Ref TrajectoryRef+ Lines []Line+ // LineCount is the number of complete lines the file held when it was read, which is what a+ // cursor is compared against to notice a file rewritten shorter.+ LineCount int+}++// Stats counts what a read could not use.+type Stats struct {+ // Malformed is how many complete, non-blank lines did not decode as a JSON object. OpenCode and+ // Codex print other output to the same stream (a migration banner, a log line), so a few are+ // expected.+ Malformed int+ // PartialTail is set when the file ended in a line with no newline: an event the executor is+ // still writing. It is not read until it is complete.+ PartialTail bool+}++func decodeObject(line []byte) (map[string]interface{}, bool) {+ var object map[string]interface{}+ if err := json.Unmarshal(line, &object); err != nil || object == nil {+ return nil, false+ }+ return object, true+}