The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu
Go agent-ai-cliagent-harnessesagent-securitybrowser-extensionciclaude-codecloudcodexcursordetection-engineeringjevknowledge-basememorymobile-device-managementobservabilitysecuritysecurity-information-and-event-managementsecurity-toolstelemetrytraces
fix(harness2session): commit the .jsonl fixtures and bound exit codes
Claudedf8e24b1c1f8
diff --git a/.gitignore b/.gitignoreindex a726cd99..7d3fb062 100644--- a/.gitignore+++ b/.gitignore@@ -49,6 +49,9 @@ cli/beacon-hooks/*.exe# during development. Without this exception the `*.jsonl` rule above silently drops it and the# tests that read it fail on a fresh clone while passing on the machine that captured it.!cli/beacon/internal/fxsession/testdata/**/*.jsonl+# Harness² trajectories: recorded from the real OpenCode and Codex releases it pins, and laid out as+# its benchmarks write them. The same rule would drop them.+!cli/beacon/internal/harness2session/testdata/**/*.jsonl!cli/beacon/internal/endpoint/falcon/pack/sample-event.jsonl!cli/beacon/internal/endpoint/gcs/pack/sample-event.jsonl*.logdiff --git a/cli/beacon/internal/harness2session/codex.go b/cli/beacon/internal/harness2session/codex.goindex 4a279822..bc127b58 100644--- a/cli/beacon/internal/harness2session/codex.go+++ b/cli/beacon/internal/harness2session/codex.go@@ -84,10 +84,7 @@ func (m *mapper) emitCodexCommand(line Line, id string, item map[string]interfacev.Tool = &schema.ToolInfo{Name: "shell", Command: command}output := str(item, "aggregated_output")info := &schema.CommandInfo{Command: command, Output: output}- if code, ok := num(item, "exit_code"); ok {- exit := int(code)- info.ExitCode = &exit- }+ info.ExitCode = exitCode(item, "exit_code")ev.Command = infovar result interface{}if output != "" {diff --git a/cli/beacon/internal/harness2session/mapper.go b/cli/beacon/internal/harness2session/mapper.goindex cfc51820..78705ffb 100644--- a/cli/beacon/internal/harness2session/mapper.go+++ b/cli/beacon/internal/harness2session/mapper.go@@ -265,6 +265,18 @@ func num(m map[string]interface{}, key string) (int64, bool) {}}+// exitCode reads a process exit status. Both executors report it as a 32-bit integer (Codex's+// Option<i32>, OpenCode's process exit code); anything outside that range is not one, and is left+// out rather than wrapped into a plausible-looking value.+func exitCode(m map[string]interface{}, key string) *int {+ code, ok := num(m, key)+ if !ok || code < math.MinInt32 || code > math.MaxInt32 {+ return nil+ }+ exit := int(code)+ return &exit+}+func positive(m map[string]interface{}, key string) int64 {if value, ok := num(m, key); ok && value > 0 {return valuediff --git a/cli/beacon/internal/harness2session/mapper_test.go b/cli/beacon/internal/harness2session/mapper_test.goindex 6eaf8c25..35a2085a 100644--- a/cli/beacon/internal/harness2session/mapper_test.go+++ b/cli/beacon/internal/harness2session/mapper_test.go@@ -300,3 +300,17 @@ func mustJSON(t *testing.T, v interface{}) string {func timeMS(ms int64) time.Time { return time.UnixMilli(ms) }func itoa64(n int64) string { return strconv.FormatInt(n, 10) }++// An exit status out of 32-bit range is not one, and must not be wrapped into a plausible value.+func TestExitCodeRefusesOutOfRangeValues(t *testing.T) {+ for raw, want := range map[string]string{`{"c":0}`: "0", `{"c":-1}`: "-1", `{"c":137}`: "137", `{"c":2147483647}`: "2147483647",+ `{"c":4294967297}`: "nil", `{"c":-2147483649}`: "nil", `{"c":1.5}`: "nil", `{"c":null}`: "nil", `{"c":"12"}`: "12", `{}`: "nil"} {+ got := "nil"+ if code := exitCode(obj(t, raw), "c"); code != nil {+ got = strconv.Itoa(*code)+ }+ if got != want {+ t.Errorf("exitCode(%s) = %s, want %s", raw, got, want)+ }+ }+}diff --git a/cli/beacon/internal/harness2session/opencode.go b/cli/beacon/internal/harness2session/opencode.goindex cbcab6fb..397d5991 100644--- a/cli/beacon/internal/harness2session/opencode.go+++ b/cli/beacon/internal/harness2session/opencode.go@@ -73,10 +73,7 @@ func (m *mapper) emitOpenCodeTool(line Line, ts int64, part map[string]interfaceswitch category {case "command":info := &schema.CommandInfo{Command: ev.Tool.Command, Output: output}- if code, ok := num(object(state, "metadata"), "exit"); ok {- exit := int(code)- info.ExitCode = &exit- }+ info.ExitCode = exitCode(object(state, "metadata"), "exit")if output != "" {ev.Content = asymptoteobserve.RetainedContent(output, asymptoteobserve.DefaultStringLimit)}diff --git a/cli/beacon/internal/harness2session/testdata/recorded/codex-exec-0.144.6.jsonl b/cli/beacon/internal/harness2session/testdata/recorded/codex-exec-0.144.6.jsonlnew file mode 100644index 00000000..a90850ea--- /dev/null+++ b/cli/beacon/internal/harness2session/testdata/recorded/codex-exec-0.144.6.jsonl@@ -0,0 +1,7 @@+{"type":"thread.started","thread_id":"01a1769f-fbe7-72b3-a4d2-a9b94a6e6dc4"}+{"type":"item.completed","item":{"id":"item_0","type":"error","message":"Model metadata for `gpt-5.1-codex` not found. Defaulting to fallback metadata; this can degrade performance and cause issues."}}+{"type":"turn.started"}+{"type":"item.started","item":{"id":"item_1","type":"command_execution","command":"/bin/bash -lc 'echo harness2-real-run'","aggregated_output":"","exit_code":null,"status":"in_progress"}}+{"type":"item.completed","item":{"id":"item_1","type":"command_execution","command":"/bin/bash -lc 'echo harness2-real-run'","aggregated_output":"harness2-real-run\n","exit_code":0,"status":"completed"}}+{"type":"item.completed","item":{"id":"item_2","type":"agent_message","text":"Ran the command."}}+{"type":"turn.completed","usage":{"input_tokens":2001,"cached_input_tokens":800,"output_tokens":100,"reasoning_output_tokens":20}}diff --git a/cli/beacon/internal/harness2session/testdata/recorded/opencode-run-1.14.18.jsonl b/cli/beacon/internal/harness2session/testdata/recorded/opencode-run-1.14.18.jsonlnew file mode 100644index 00000000..9129187b--- /dev/null+++ b/cli/beacon/internal/harness2session/testdata/recorded/opencode-run-1.14.18.jsonl@@ -0,0 +1,6 @@+{"type":"step_start","timestamp":1792991817442,"sessionID":"ses_e895c12c2ffeRtDssTNzjq5b5M","part":{"id":"prt_176a3eedc001DnldUaIpMZFjAT","messageID":"msg_176a3ee66001Ap7Fi4vXcKix02","sessionID":"ses_e895c12c2ffeRtDssTNzjq5b5M","type":"step-start"}}+{"type":"tool_use","timestamp":1792991817557,"sessionID":"ses_e895c12c2ffeRtDssTNzjq5b5M","part":{"type":"tool","tool":"bash","callID":"call_bash_1","state":{"status":"completed","input":{"command":"echo harness2-real-run","description":"Print a marker"},"output":"harness2-real-run\n","metadata":{"output":"harness2-real-run\n","exit":0,"description":"Print a marker","truncated":false},"title":"Print a marker","time":{"start":1792991817545,"end":1792991817550}},"id":"prt_176a3eedf001LqRWBoxQ6lbXpL","sessionID":"ses_e895c12c2ffeRtDssTNzjq5b5M","messageID":"msg_176a3ee66001Ap7Fi4vXcKix02"}}+{"type":"step_finish","timestamp":1792991817558,"sessionID":"ses_e895c12c2ffeRtDssTNzjq5b5M","part":{"id":"prt_176a3ef52001jgSOx4wTxCDXyk","reason":"tool-calls","messageID":"msg_176a3ee66001Ap7Fi4vXcKix02","sessionID":"ses_e895c12c2ffeRtDssTNzjq5b5M","type":"step-finish","tokens":{"total":940,"input":600,"output":40,"reasoning":0,"cache":{"write":0,"read":300}},"cost":0}}+{"type":"step_start","timestamp":1792991817616,"sessionID":"ses_e895c12c2ffeRtDssTNzjq5b5M","part":{"id":"prt_176a3ef8b001ZjgRRacy9c7YIx","messageID":"msg_176a3ef5c001zAJBae10zCsoRr","sessionID":"ses_e895c12c2ffeRtDssTNzjq5b5M","type":"step-start"}}+{"type":"text","timestamp":1792991817618,"sessionID":"ses_e895c12c2ffeRtDssTNzjq5b5M","part":{"id":"prt_176a3ef8d0018IyO14X623tWQg","messageID":"msg_176a3ef5c001zAJBae10zCsoRr","sessionID":"ses_e895c12c2ffeRtDssTNzjq5b5M","type":"text","text":"Ran the command.","time":{"start":1792991817614,"end":1792991817617}}}+{"type":"step_finish","timestamp":1792991817623,"sessionID":"ses_e895c12c2ffeRtDssTNzjq5b5M","part":{"id":"prt_176a3ef93001tpdlfK5757TcKB","reason":"stop","messageID":"msg_176a3ef5c001zAJBae10zCsoRr","sessionID":"ses_e895c12c2ffeRtDssTNzjq5b5M","type":"step-finish","tokens":{"total":941,"input":601,"output":40,"reasoning":0,"cache":{"write":0,"read":300}},"cost":0}}diff --git a/cli/beacon/internal/harness2session/testdata/runs/jb/dataset/accountant/task3/model_traj/h2-final-1/accountant_task3_h2-final-1_attempt2_20261026T101500.jsonl b/cli/beacon/internal/harness2session/testdata/runs/jb/dataset/accountant/task3/model_traj/h2-final-1/accountant_task3_h2-final-1_attempt2_20261026T101500.jsonlnew file mode 100644index 00000000..b0a69bee--- /dev/null+++ b/cli/beacon/internal/harness2session/testdata/runs/jb/dataset/accountant/task3/model_traj/h2-final-1/accountant_task3_h2-final-1_attempt2_20261026T101500.jsonl@@ -0,0 +1,8 @@+{"type":"step_start","timestamp":1793004900100,"sessionID":"ses_7a1f00c0ffee","part":{"id":"prt_01","sessionID":"ses_7a1f00c0ffee","messageID":"msg_01","type":"step-start","snapshot":"4b825dc"}}+{"type":"reasoning","timestamp":1793004901200,"sessionID":"ses_7a1f00c0ffee","part":{"id":"prt_02","sessionID":"ses_7a1f00c0ffee","messageID":"msg_01","type":"reasoning","text":"Read the instructions first.","time":{"start":1793004900200,"end":1793004901200}}}+{"type":"tool_use","timestamp":1793004902300,"sessionID":"ses_7a1f00c0ffee","part":{"id":"prt_03","sessionID":"ses_7a1f00c0ffee","messageID":"msg_01","type":"tool","callID":"call_read_1","tool":"read","state":{"status":"completed","input":{"filePath":"/tmp/hevo_ws/accountant_task3/task_folder/TASK_INSTRUCTIONS.txt"},"output":"Prepare the Q3 reconciliation.","title":"TASK_INSTRUCTIONS.txt","metadata":{"preview":"Prepare"},"time":{"start":1793004901300,"end":1793004902250}}}}+{"type":"tool_use","timestamp":1793004904400,"sessionID":"ses_7a1f00c0ffee","part":{"id":"prt_04","sessionID":"ses_7a1f00c0ffee","messageID":"msg_01","type":"tool","callID":"call_bash_1","tool":"bash","state":{"status":"completed","input":{"command":"python3 scripts/reconcile.py","description":"Run the helper"},"output":"wrote output/q3.xlsx\n","title":"Run the helper","metadata":{"output":"wrote output/q3.xlsx\n","exit":0,"description":"Run the helper"},"time":{"start":1793004902400,"end":1793004904350}}}}+{"type":"tool_use","timestamp":1793004905500,"sessionID":"ses_7a1f00c0ffee","part":{"id":"prt_05","sessionID":"ses_7a1f00c0ffee","messageID":"msg_01","type":"tool","callID":"call_edit_1","tool":"edit","state":{"status":"error","input":{"filePath":"/tmp/hevo_ws/accountant_task3/output/notes.md","oldString":"x","newString":"y"},"error":"oldString not found in content","time":{"start":1793004904500,"end":1793004905450}}}}+{"type":"step_finish","timestamp":1793004905600,"sessionID":"ses_7a1f00c0ffee","part":{"id":"prt_06","sessionID":"ses_7a1f00c0ffee","messageID":"msg_01","type":"step-finish","reason":"tool-calls","cost":0.0123,"tokens":{"total":1840,"input":1200,"output":400,"reasoning":40,"cache":{"read":200,"write":0}}}}+{"type":"text","timestamp":1793004907700,"sessionID":"ses_7a1f00c0ffee","part":{"id":"prt_07","sessionID":"ses_7a1f00c0ffee","messageID":"msg_02","type":"text","text":"The reconciliation is in output/q3.xlsx.","time":{"start":1793004906000,"end":1793004907700}}}+{"type":"step_finish","timestamp":1793004907800,"sessionID":"ses_7a1f00c0ffee","part":{"id":"prt_08","sessionID":"ses_7a1f00c0ffee","messageID":"msg_02","type":"step-finish","reason":"stop","cost":0.002,"tokens":{"total":1500,"input":1300,"output":100,"reasoning":0,"cache":{"read":100,"write":0}}}}diff --git a/cli/beacon/internal/harness2session/testdata/runs/jb/dataset/accountant/task3/task_folder/example.jsonl b/cli/beacon/internal/harness2session/testdata/runs/jb/dataset/accountant/task3/task_folder/example.jsonlnew file mode 100644index 00000000..1bd5f018--- /dev/null+++ b/cli/beacon/internal/harness2session/testdata/runs/jb/dataset/accountant/task3/task_folder/example.jsonl@@ -0,0 +1 @@+{"type":"text","timestamp":1,"sessionID":"ses_input","part":{"type":"text","text":"task input, not a rollout"}}diff --git a/cli/beacon/internal/harness2session/testdata/runs/jb/judged/proposer.jsonl b/cli/beacon/internal/harness2session/testdata/runs/jb/judged/proposer.jsonlnew file mode 100644index 00000000..0b54a078--- /dev/null+++ b/cli/beacon/internal/harness2session/testdata/runs/jb/judged/proposer.jsonl@@ -0,0 +1 @@+{"type":"result","subtype":"success"}diff --git a/cli/beacon/internal/harness2session/testdata/runs/jb/judged/scores.jsonl b/cli/beacon/internal/harness2session/testdata/runs/jb/judged/scores.jsonlnew file mode 100644index 00000000..91903106--- /dev/null+++ b/cli/beacon/internal/harness2session/testdata/runs/jb/judged/scores.jsonl@@ -0,0 +1 @@+{"task":"accountant/task3","label":"h2-final-1","score":0.82}diff --git a/cli/beacon/internal/harness2session/testdata/runs/lab/results/cx/antitrust-competition__t07__h2-cand-dom-1/codex_session.jsonl b/cli/beacon/internal/harness2session/testdata/runs/lab/results/cx/antitrust-competition__t07__h2-cand-dom-1/codex_session.jsonlnew file mode 100644index 00000000..6bbdd269--- /dev/null+++ b/cli/beacon/internal/harness2session/testdata/runs/lab/results/cx/antitrust-competition__t07__h2-cand-dom-1/codex_session.jsonl@@ -0,0 +1,11 @@+{"type":"thread.started","thread_id":"0199f1c2-7d1e-7c41-9a3b-5e2b8d0f6a11"}+{"type":"turn.started"}+{"type":"item.completed","item":{"id":"item_0","type":"reasoning","text":"**Reviewing the data room**"}}+{"type":"item.started","item":{"id":"item_1","type":"command_execution","command":"bash -lc 'ls documents'","aggregated_output":"","exit_code":null,"status":"in_progress"}}+{"type":"item.completed","item":{"id":"item_1","type":"command_execution","command":"bash -lc 'ls documents'","aggregated_output":"merger_agreement.pdf\n","exit_code":0,"status":"completed"}}+{"type":"item.completed","item":{"id":"item_2","type":"command_execution","command":"bash -lc 'pdftotext missing.pdf -'","aggregated_output":"I/O Error: Couldn't open file\n","exit_code":1,"status":"failed"}}+{"type":"item.completed","item":{"id":"item_3","type":"file_change","changes":[{"path":"/work/memo.md","kind":"add"},{"path":"/work/draft.md","kind":"update"},{"path":"/work/scratch.txt","kind":"delete"}],"status":"completed"}}+{"type":"item.completed","item":{"id":"item_4","type":"mcp_tool_call","server":"docs","tool":"search","arguments":{"q":"HSR filing"},"result":{"content":[{"type":"text","text":"3 hits"}],"structured_content":null},"error":null,"status":"completed"}}+{"type":"item.completed","item":{"id":"item_5","type":"todo_list","items":[{"text":"draft memo","completed":true}]}}+{"type":"item.completed","item":{"id":"item_6","type":"agent_message","text":"Memo written to memo.md."}}+{"type":"turn.completed","usage":{"input_tokens":5000,"cached_input_tokens":3000,"output_tokens":700,"reasoning_output_tokens":300}}