Skip to content

Every agent's model. One place. Codex on DeepSeek, Claude Code on Kimi, from the menu bar.

codex: a thread magpie compacted goes on after magpie is quit or disconnected (LukeMa on X)

yetone4896fc3ba148
diff --git a/internal/agent/codex.go b/internal/agent/codex.go
index d02aa0c74..54b376355 100644
--- a/internal/agent/codex.go
+++ b/internal/agent/codex.go
@@ -1116,6 +1116,12 @@ func codexIn(at place) *Agent {
}
dropList()
forget(at.key("codex.model"), at.key("codex.effort"), at.key("codex.provider"), at.key("codex.catalog"))
+ // its threads go on without magpie: the compactions magpie made
+ // in them written as Codex's own (a distro's Codex locks its
+ // files where this machine's locks can't reach)
+ if at.spell == nil {
+ portCodexHome(dir, codexPortUnwire)
+ }
return settle()
},
// signed in with ChatGPT, Codex reaches magpie by the base URL with
diff --git a/internal/agent/codex_portable.go b/internal/agent/codex_portable.go
new file mode 100644
index 000000000..1d58651df
--- /dev/null
+++ b/internal/agent/codex_portable.go
@@ -0,0 +1,67 @@
+package agent
+
+import (
+ "context"
+ "log"
+ "os"
+ "time"
+
+ "github.com/yetone/magpie/internal/sessions"
+)
+
+// A Codex conversation magpie compacted can't go on once Codex talks to
+// OpenAI itself, unless its rollout is written portable
+// (sessions.PortCodexRollouts, LukeMa on X): done while magpie runs, for
+// the threads no Codex has open, and again as magpie lets Codex go (quit,
+// Disconnect), for those it has just let go.
+
+// codexPortEvery is how often a running magpie makes the rollouts written
+// since portable; codexPortAfter how long after it starts it first does.
+var codexPortEvery, codexPortAfter = 10 * time.Minute, 2 * time.Minute
+
+// codexPortUnwire is how long Disconnect spends on it; what is left is
+// done by the next pass.
+const codexPortUnwire = 3 * time.Second
+
+// portCodexHome makes the rollouts of the Codex home dir portable, for
+// up to d.
+func portCodexHome(dir string, d time.Duration) {
+ ctx, cancel := context.WithTimeout(context.Background(), d)
+ defer cancel()
+ res, err := sessions.PortCodexRollouts(ctx, dir)
+ if err != nil {
+ log.Println("codex rollouts:", err)
+ }
+ if len(res.Rewritten) > 0 || len(res.Held) > 0 {
+ log.Printf("codex rollouts: %d written portable, %d left while a Codex has them open", len(res.Rewritten), len(res.Held))
+ }
+}
+
+// PortCodexRollouts makes this machine's Codex rollouts portable, for up
+// to d: as magpie quits.
+func PortCodexRollouts(d time.Duration) {
+ if home, err := os.UserHomeDir(); err == nil {
+ portCodexHome(here(home).codexHome(), d)
+ }
+}
+
+// KeepCodexRolloutsPortable makes this machine's Codex rollouts portable
+// every so often until ctx ends.
+func KeepCodexRolloutsPortable(ctx context.Context) {
+ home, err := os.UserHomeDir()
+ if err != nil {
+ return
+ }
+ dir := here(home).codexHome()
+ wait := codexPortAfter
+ for {
+ select {
+ case <-ctx.Done():
+ return
+ case <-time.After(wait):
+ }
+ wait = codexPortEvery
+ // a pass that is cut short goes on from where it was next time
+ portCodexHome(dir, codexPortEvery/2)
+ }
+}
diff --git a/internal/cli/providers_cli.go b/internal/cli/providers_cli.go
index 29ed8e25f..0d2c91be6 100644
--- a/internal/cli/providers_cli.go
+++ b/internal/cli/providers_cli.go
@@ -1256,6 +1256,7 @@ func serve() error {
// Codex's background app-server, restarted when it has the list from
// before a change and no codex session is on it
go agent.KeepCodexDaemonCurrent(context.Background())
+ go agent.KeepCodexRolloutsPortable(context.Background())
public := advertisedURL()
fmt.Println(green.Render("●"), "magpie gateway on", bold.Render(gateway.URL()))
fmt.Println(muted.Render(" OpenAI "), public+"/v1/chat/completions", muted.Render("·"), public+"/v1/responses")
diff --git a/internal/gui/app.go b/internal/gui/app.go
index 9fd474b3e..ea098dd77 100644
--- a/internal/gui/app.go
+++ b/internal/gui/app.go
@@ -375,6 +375,10 @@ func Run(version string, showMain bool, link string) error {
log.Println("codex failover:", err)
}
}
+ // the compactions magpie made in Codex's threads written as
+ // Codex's own, for the threads no Codex has open: with magpie
+ // gone they still go on (LukeMa on X)
+ agent.PortCodexRollouts(3 * time.Second)
updates.install(false)
},
// Wails exits on some webview errors; say why before it does.
diff --git a/internal/gui/backend.go b/internal/gui/backend.go
index 01aa8b680..abff47da5 100644
--- a/internal/gui/backend.go
+++ b/internal/gui/backend.go
@@ -43,6 +43,9 @@ func startBackend() (gw *gateway.Server) {
// Codex's background app-server, restarted when it has the list from
// before a change and no codex session is on it (luci)
go agent.KeepCodexDaemonCurrent(backendCtx)
+ // the compactions magpie made in Codex's threads written as Codex's
+ // own, so a thread goes on when Codex talks to OpenAI itself
+ go agent.KeepCodexRolloutsPortable(backendCtx)
// Model lists are fetched, never compiled in: whatever the agents can see
// comes from the models.dev catalog plus each vendor's own /models answer.
// Keep both halves warm without making the user click anything.
diff --git a/internal/sessions/codex_portable.go b/internal/sessions/codex_portable.go
new file mode 100644
index 000000000..2bf00fbd6
--- /dev/null
+++ b/internal/sessions/codex_portable.go
@@ -0,0 +1,416 @@
+package sessions
+
+import (
+ "bufio"
+ "bytes"
+ "context"
+ "encoding/json"
+ "errors"
+ "io"
+ "io/fs"
+ "log"
+ "os"
+ "path/filepath"
+ "regexp"
+ "strings"
+ "time"
+
+ "github.com/klauspost/compress/zstd"
+
+ "github.com/yetone/magpie/internal/codexsummary"
+ "github.com/yetone/magpie/internal/filelock"
+ "github.com/yetone/magpie/internal/settings"
+ "github.com/yetone/magpie/internal/steady"
+)
+
+// A Codex conversation magpie compacted keeps magpie's compaction item in
+// its rollout, and Codex sends it back with every later turn: to OpenAI
+// itself, once magpie is quit or disconnected, it is refused
+// (invalid_encrypted_content) and the conversation can't go on (LukeMa on
+// X). PortCodexRollouts writes, in each rollout that has one, the message
+// it stands for in its place (codexsummary.Message): what Codex's own
+// local compaction keeps, and what the gateway sends for it anyway, so a
+// rollout so written reads the same through magpie and goes on without it.
+//
+// Only magpie's compaction items change, in whichever line holds them (a
+// response_item, a compacted line's replacement_history); every other
+// line keeps its bytes, and the file its line count, mode and time. A
+// rollout is changed as Codex publishes a compressed one
+// (codex-rs/rollout/src/writer_lock.rs, try_acquire_for_publication):
+// under its home's coordination lock, and only when no Codex holds the
+// thread's writer lock, so never under a Codex that has the thread open.
+// One that does is left for a later pass. A Codex home without
+// thread-writer-locks (a Codex from before them) is left alone, as there
+// is no telling whether a Codex is writing there.
+
+// codexPortState is what PortCodexRollouts keeps between passes, per
+// Codex home: when its last whole pass began, and the rollouts it had to
+// leave for a Codex that held them.
+type codexPortState struct {
+ Homes map[string]codexPortHome `json:"homes"`
+}
+
+type codexPortHome struct {
+ Since time.Time `json:"since"`
+ Pending []string `json:"pending,omitempty"`
+}
+
+// codexPortFirst is the first day magpie wrote a compaction item of its
+// own (c3931c91d, 2026-09-25): no rollout last written before then has one.
+var codexPortFirst = time.Date(2026, 9, 24, 0, 0, 0, 0, time.UTC)
+
+// codexPortSlack is how far before a pass began the next one looks again:
+// a file's time can be coarser than the clock.
+const codexPortSlack = 2 * time.Minute
+
+func codexPortPath() string { return filepath.Join(settings.Dir(), "codex-portable.json") }
+
+// CodexPort is what a pass did.
+type CodexPort struct {
+ Rewritten []string // rollouts written portable
+ Held []string // rollouts with magpie's item a Codex holds, left for a later pass
+ Done bool // every rollout due was looked at
+}
+
+// PortCodexRollouts makes the rollouts of the Codex home home portable,
+// those written since the last whole pass and those it left before, until
+// ctx ends. The first pass reads every rollout written since magpie began
+// to compact; later ones only what changed since.
+func PortCodexRollouts(ctx context.Context, home string) (CodexPort, error) {
+ var res CodexPort
+ locks := filepath.Join(home, "thread-writer-locks")
+ if fi, err := os.Stat(locks); err != nil || !fi.IsDir() {
+ return res, nil
+ }
+ // one pass at a time across magpies
+ unlock, held := filelock.Lock(codexPortPath()+".lock", 0)
+ if !held {
+ return res, nil
+ }
+ defer unlock()
+ var st codexPortState
+ if b, err := os.ReadFile(codexPortPath()); err == nil {
+ if json.Unmarshal(b, &st) != nil {
+ st = codexPortState{} // unreadable: a whole pass again, which only costs time
+ }
+ } else if !errors.Is(err, fs.ErrNotExist) {
+ return res, err
+ }
+ if st.Homes == nil {
+ st.Homes = map[string]codexPortHome{}
+ }
+ key := filepath.Clean(home)
+ was := st.Homes[key]
+ since := codexPortFirst
+ if !was.Since.IsZero() {
+ since = was.Since.Add(-codexPortSlack)
+ }
+ start := time.Now()
+ var due []string
+ seen := map[string]bool{}
+ add := func(p string) {
+ if !seen[p] {
+ seen[p] = true
+ due = append(due, p)
+ }
+ }
+ for _, p := range was.Pending {
+ for _, q := range []string{p, rolloutTwin(p)} { // Codex may have compressed it since
+ if fi, err := os.Stat(q); err == nil && fi.Mode().IsRegular() {
+ add(q)
+ }
+ }
+ }
+ for _, folder := range []string{"sessions", "archived_sessions"} {
+ filepath.WalkDir(filepath.Join(home, folder), func(p string, d fs.DirEntry, err error) error {
+ if err != nil || d.IsDir() || codexThreadOf(d.Name()) == "" {
+ return nil
+ }
+ if fi, err := d.Info(); err == nil && fi.Mode().IsRegular() && !fi.ModTime().Before(since) {
+ add(p)
+ }
+ return nil
+ })
+ }
+ res.Done = true
+ var pending []string
+ for i, p := range due {
+ if ctx.Err() != nil {
+ res.Done = false
+ pending = append(pending, due[i:]...)
+ break
+ }
+ changed, err := portRollout(ctx, home, p)
+ switch {
+ case errors.Is(err, errRolloutHeld):
+ res.Held = append(res.Held, p)
+ pending = append(pending, p)
+ case err != nil:
+ if ctx.Err() != nil {
+ res.Done = false
+ pending = append(pending, due[i:]...)
+ break
+ }
+ log.Printf("codex rollout %s: %v", p, err)
+ case changed:
+ res.Rewritten = append(res.Rewritten, p)
+ }
+ if !res.Done {
+ break
+ }
+ }
+ now := codexPortHome{Since: was.Since, Pending: pending}
+ if res.Done {
+ now.Since = start
+ } else {
+ now.Pending = append(now.Pending, was.Pending...)
+ now.Pending = uniq(now.Pending)
+ }
+ st.Homes[key] = now
+ b, _ := json.MarshalIndent(st, "", " ")
+ if err := os.MkdirAll(settings.Dir(), 0o700); err != nil {
+ return res, err
+ }
+ return res, steady.WriteFile(codexPortPath(), b, 0o600)
+}
+
+func uniq(list []string) []string {
+ var out []string
+ for _, s := range list {
+ if !contains(out, s) {
+ out = append(out, s)
+ }
+ }
+ return out
+}
+
+// errRolloutHeld is a rollout a Codex has open: its thread's writer lock
+// is held, or the file changed while it was read.
+var errRolloutHeld = errors.New("held by a Codex")
+
+// codexRolloutName is Codex's name for a rollout (rollout_file_name.rs):
+// its time, its thread and, for a later rollout of the thread, its own id.
+var codexRolloutName = regexp.MustCompile(`^rollout-\d{4}-\d{2}-\d{2}T\d{2}-\d{2}-\d{2}-([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12})(?:_[0-9a-fA-F-]{36})?\.jsonl(?:\.zst)?$`)
+
+// codexThreadOf is the thread a rollout's file name says it is of; "" for
+// a file that isn't one.
+func codexThreadOf(name string) string {
+ m := codexRolloutName.FindStringSubmatch(name)
+ if m == nil {
+ return ""
+ }
+ return strings.ToLower(m[1])
+}
+
+var magpieMark = []byte(codexsummary.Mark)
+
+// portRollout writes the rollout at path portable, if it has magpie's
+// compaction item, and says whether it did.
+func portRollout(ctx context.Context, home, path string) (bool, error) {
+ thread := codexThreadOf(filepath.Base(path))
+ if thread == "" {
+ return false, nil
+ }
+ fi, err := os.Stat(path)
+ if err != nil {
+ return false, err
+ }
+ in, err := openLines(path)
+ if err != nil {
+ return false, err
+ }
+ defer in.Close()
+ tmp, err := os.CreateTemp(filepath.Dir(path), ".magpie-"+filepath.Base(path)+"-*")
+ if err != nil {
+ return false, err
+ }
+ done := false
+ defer func() {
+ if !done {
+ tmp.Close()
+ os.Remove(tmp.Name())
+ }
+ }()
+ var w io.Writer = tmp
+ var z *zstd.Encoder
+ if packed(path) {
+ if z, err = zstd.NewWriter(tmp); err != nil {
+ return false, err
+ }
+ w = z
+ }
+ bw := bufio.NewWriterSize(w, 64<<10)
+ br := bufio.NewReaderSize(in, 64<<10)
+ changed := false
+ for n := 0; ; n++ {
+ if n%4096 == 0 && ctx.Err() != nil {
+ return false, ctx.Err()
+ }
+ line, err := br.ReadBytes('\n')
+ if len(line) > 0 {
+ out, ok := portLine(line)
+ changed = changed || ok
+ if _, werr := bw.Write(out); werr != nil {
+ return false, werr
+ }
+ }
+ if err == io.EOF {
+ break
+ }
+ if err != nil {
+ return false, err
+ }
+ }
+ if !changed {
+ return false, nil
+ }
+ if err := bw.Flush(); err != nil {
+ return false, err
+ }
+ if z != nil {
+ if err := z.Close(); err != nil {
+ return false, err
+ }
+ }
+ if err := tmp.Chmod(fi.Mode().Perm()); err != nil {
+ return false, err
+ }
+ if err := steady.Sync(tmp); err != nil {
+ return false, err
+ }
+ if err := tmp.Close(); err != nil {
+ return false, err
+ }
+ in.Close()
+ os.Chtimes(tmp.Name(), fi.ModTime(), fi.ModTime())
+
+ // published as Codex publishes a compressed rollout: under the
+ // coordination lock every Codex writer takes before its thread's
+ // lock, with that lock free (or never made)
+ locks := filepath.Join(home, "thread-writer-locks")
+ unlock, held := filelock.Lock(filepath.Join(locks, ".coordination.lock"), filelock.Wait)
+ if !held {
+ return false, errRolloutHeld
+ }
+ defer unlock()
+ if lf, err := os.OpenFile(filepath.Join(locks, thread+".lock"), os.O_RDWR, 0); err == nil {
+ got, err := filelock.TryLock(lf)
+ if err != nil || !got {
+ lf.Close()
+ return false, errRolloutHeld
+ }
+ defer func() { filelock.Unlock(lf); lf.Close() }()
+ } else if !errors.Is(err, fs.ErrNotExist) {
+ return false, errRolloutHeld
+ }
+ // written to since it was read: a Codex had it after all
+ if now, err := os.Stat(path); err != nil || now.Size() != fi.Size() || !now.ModTime().Equal(fi.ModTime()) {
+ return false, errRolloutHeld
+ }
+ if err := steady.Rename(tmp.Name(), path); err != nil {
+ return false, err
+ }
+ done = true
+ return true, nil
+}
+
+// portLine is a rollout line with each of magpie's compaction items in it
+// written as the message it stands for; any other line as it is, byte for
+// byte. One of magpie's items whose summary can't be read is left out of
+// a list (and its entry of replacement_history_metadata with it), as the
+// gateway leaves it out; a line that is one alone stays as it is, as
+// leaving a line out would move every line after it.
+func portLine(line []byte) ([]byte, bool) {
+ if !bytes.Contains(line, magpieMark) {
+ return line, false
+ }
+ body := bytes.TrimRight(line, "\r\n")
+ end := line[len(body):]
+ d := json.NewDecoder(bytes.NewReader(body))
+ d.UseNumber()
+ var v any
+ if d.Decode(&v) != nil || d.More() {
+ return line, false
+ }
+ v, changed, _ := portValue(v)
+ if !changed {
+ return line, false
+ }
+ var buf bytes.Buffer
+ e := json.NewEncoder(&buf)
+ e.SetEscapeHTML(false)
+ if e.Encode(v) != nil {
+ return line, false
+ }
+ return append(bytes.TrimRight(buf.Bytes(), "\n"), end...), true
+}
+
+// portValue is v with magpie's compaction items in it written as their
+// messages; drop says v is one of magpie's that can't be read.
+func portValue(v any) (_ any, changed, drop bool) {
+ switch x := v.(type) {
+ case map[string]any:
+ if t, _ := x["type"].(string); t == "compaction" || t == "compaction_summary" {
+ enc, _ := x["encrypted_content"].(string)
+ if msg, ours := codexsummary.Message(enc); ours {
+ if msg == nil {
+ return x, false, true
+ }
+ return msg, true, false
+ }
+ return x, false, false
+ }
+ for k, c := range x {
+ list, isList := c.([]any)
+ if !isList {
+ if nv, ch, dr := portValue(c); ch && !dr {
+ x[k], changed = nv, true
+ }
+ continue
+ }
+ // Codex reads replacement_history_metadata beside
+ // replacement_history, one entry for each item: an item is
+ // left out only with its entry, and where the two don't line
+ // up (Codex refuses that line anyway) none is
+ meta, hasMeta := x[k+"_metadata"]
+ metaList, _ := meta.([]any)
+ drop := !hasMeta || meta == nil || len(metaList) == len(list)
+ kept, gone, ch := portList(list, drop)
+ if !ch {
+ continue
+ }
+ x[k], changed = kept, true
+ if len(gone) > 0 && metaList != nil {
+ m := []any{}
+ for i, e := range metaList {
+ if !gone[i] {
+ m = append(m, e)
+ }
+ }
+ x[k+"_metadata"] = m
+ }
+ }
+ return x, changed, false
+ case []any:
+ kept, _, ch := portList(x, true)
+ return kept, ch, false
+ }
+ return v, false, false
+}
+
+func portList(list []any, drop bool) (_ []any, gone map[int]bool, changed bool) {
+ out := make([]any, 0, len(list))
+ for i, e := range list {
+ nv, ch, dr := portValue(e)
+ if dr && drop {
+ if gone == nil {
+ gone = map[int]bool{}
+ }
+ gone[i], changed = true, true
+ continue
+ }
+ changed = changed || ch
+ out = append(out, nv)
+ }
+ return out, gone, changed
+}
diff --git a/internal/sessions/codex_portable_test.go b/internal/sessions/codex_portable_test.go
new file mode 100644
index 000000000..7084966c1
--- /dev/null
+++ b/internal/sessions/codex_portable_test.go
@@ -0,0 +1,239 @@
+package sessions
+
+import (
+ "bytes"
+ "context"
+ "encoding/json"
+ "os"
+ "path/filepath"
+ "reflect"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/klauspost/compress/zstd"
+
+ "github.com/yetone/magpie/internal/codexsummary"
+ "github.com/yetone/magpie/internal/filelock"
+)
+
+const portThread = "01a15820-ff81-7840-bb3f-e18771c53aa8"
+
+// portRolloutLines is a Codex 0.162 rollout of a thread magpie compacted,
+// in the shapes Codex writes (codex-rs/history/src/rollout_payload.rs,
+// CompactedItemWire; the compaction item as magpie's gateway streams it):
+// a session_meta, a user message that talks about "magpie1:" (text, never
+// changed), magpie's compaction item as a response_item, the compacted
+// checkpoint holding it in replacement_history with its metadata beside,
+// a checkpoint with an unreadable one of magpie's, OpenAI's own sealed
+// compaction, and a later turn.
+func portRolloutLines(summary string) []string {
+ enc := codexsummary.Encrypted(summary)
+ return []string{
+ `{"timestamp":"2026-10-20T07:05:20.118Z","type":"session_meta","payload":{"id":"` + portThread + `","timestamp":"2026-10-20T07:05:20.101Z","cwd":"/Users/me/w","originator":"codex_cli_rs","cli_version":"0.162.1","source":"cli","model_provider":"openai"}}`,
+ `{"timestamp":"2026-10-20T07:05:31.002Z","type":"response_item","payload":{"type":"message","role":"user","content":[{"type":"input_text","text":"why does \"magpie1:\" show in my rollout? <b>&</b>"}]}}`,
+ `{"timestamp":"2026-10-20T07:40:02.551Z","type":"response_item","payload":{"type":"compaction","id":"cmp_magpie_1761000002","encrypted_content":"` + enc + `"},"metadata":{"compaction_model_hash":"h1"}}`,
+ `{"timestamp":"2026-10-20T07:40:02.553Z","type":"compacted","payload":{"message":"","replacement_history":[{"type":"message","role":"developer","content":[{"type":"input_text","text":"<permissions instructions>"}]},{"type":"message","role":"user","content":[{"type":"input_text","text":"fix the build"}]},{"type":"compaction","id":"cmp_magpie_1761000002","encrypted_content":"` + enc + `"}],"replacement_history_metadata":[{"client_authored":true},{},{"compaction_model_hash":"h1"}],"window_number":2,"compaction_response_id":"resp_magpie_1761000002","latest_token_usage_record":{"input_tokens":182734,"output_tokens":1204}}}`,
+ `{"timestamp":"2026-10-20T08:10:00.000Z","type":"compacted","payload":{"message":"","replacement_history":[{"type":"message","role":"user","content":[{"type":"input_text","text":"go on"}]},{"type":"compaction","encrypted_content":"magpie1:%%%"}],"replacement_history_metadata":[{"client_authored":true},{"compaction_model_hash":"h2"}]}}`,
+ `{"timestamp":"2026-10-20T09:00:00.000Z","type":"response_item","payload":{"type":"compaction","id":"cmp_0a1b","encrypted_content":"gAAAAABo1x2y3z_OpenAIsOwn=="}}`,
+ `{"timestamp":"2026-10-20T09:00:05.000Z","type":"response_item","payload":{"type":"message","role":"assistant","content":[{"type":"output_text","text":"Done: 1.5e+300 tokens, <ok>"}]}}`,
+ }
+}
+
+// portHome is a Codex home with thread-writer-locks, as Codex 0.162 keeps
+// it, and the rollout at sessions/2026/10/20, plain or packed, last
+// written an hour ago with mode 0640.
+func portHome(t *testing.T, lines []string, zst bool) (home, path string) {
+ t.Helper()
+ t.Setenv("XDG_CONFIG_HOME", t.TempDir())
+ home = filepath.Join(t.TempDir(), ".codex")
+ if err := os.MkdirAll(filepath.Join(home, "thread-writer-locks"), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ dir := filepath.Join(home, "sessions", "2026", "10", "20")
+ if err := os.MkdirAll(dir, 0o755); err != nil {
+ t.Fatal(err)
+ }
+ path = filepath.Join(dir, "rollout-2026-10-20T15-05-20-"+portThread+".jsonl")
+ b := []byte(strings.Join(lines, "\n") + "\n")
+ if zst {
+ path += ".zst"
+ z, _ := zstd.NewWriter(nil)
+ b = z.EncodeAll(b, nil)
+ }
+ if err := os.WriteFile(path, b, 0o640); err != nil {
+ t.Fatal(err)
+ }
+ old := time.Now().Add(-time.Hour).Truncate(time.Second)
+ if err := os.Chtimes(path, old, old); err != nil {
+ t.Fatal(err)
+ }
+ return home, path
+}
+
+func readRollout(t *testing.T, path string) []string {
+ t.Helper()
+ in, err := openLines(path)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer in.Close()
+ var buf bytes.Buffer
+ buf.ReadFrom(in)
+ return strings.Split(strings.TrimSuffix(buf.String(), "\n"), "\n")
+}
+
+func decode(t *testing.T, line string) map[string]any {
+ t.Helper()
+ var v map[string]any
+ if err := json.Unmarshal([]byte(line), &v); err != nil {
+ t.Fatalf("%v: %s", err, line)
+ }
+ return v
+}
+
+// LukeMa (X): a Codex conversation magpie compacted errored once magpie
+// was quit, as OpenAI can't read magpie's compaction item
+// (invalid_encrypted_content). PortCodexRollouts writes each of magpie's
+// items as Codex's own summary message, the one the gateway sends for it,
+// and changes nothing else: the other lines keep their bytes, the file
+// its line count, mode and time, and a compacted line's metadata stays
+// beside its items.
+func TestPortCodexRollouts(t *testing.T) {
+ for _, zst := range []bool{false, true} {
+ t.Run(map[bool]string{false: "plain", true: "zst"}[zst], func(t *testing.T) {
+ summary := "Progress: build fixed in internal/x.\nNext: run the suite."
+ lines := portRolloutLines(summary)
+ home, path := portHome(t, lines, zst)
+ before, _ := os.Stat(path)
+
+ res, err := PortCodexRollouts(context.Background(), home)
+ if err != nil || !res.Done || len(res.Rewritten) != 1 || len(res.Held) != 0 {
+ t.Fatalf("pass: %+v, %v", res, err)
+ }
+ after, err := os.Stat(path)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !after.ModTime().Equal(before.ModTime()) || after.Mode() != before.Mode() {
+ t.Errorf("time/mode %v %v, was %v %v", after.ModTime(), after.Mode(), before.ModTime(), before.Mode())
+ }
+ got := readRollout(t, path)
+ if len(got) != len(lines) {
+ t.Fatalf("%d lines, was %d", len(got), len(lines))
+ }
+ for _, i := range []int{0, 1, 5, 6} {
+ if got[i] != lines[i] {
+ t.Errorf("line %d changed:\n%s\nwas\n%s", i, got[i], lines[i])
+ }
+ }
+ want := codexsummary.UserMessage(codexsummary.Prefix + "\n" + summary)
+ wantJSON, _ := json.Marshal(want)
+ var wantAny any
+ json.Unmarshal(wantJSON, &wantAny)
+
+ l2 := decode(t, got[2])
+ if !reflect.DeepEqual(l2["payload"], wantAny) {
+ t.Errorf("response_item payload %v, want %v", l2["payload"], wantAny)
+ }
+ if !reflect.DeepEqual(l2["metadata"], map[string]any{"compaction_model_hash": "h1"}) || l2["type"] != "response_item" {
+ t.Errorf("the rest of the line changed: %s", got[2])
+ }
+ p3 := decode(t, got[3])["payload"].(map[string]any)
+ hist := p3["replacement_history"].([]any)
+ if len(hist) != 3 || !reflect.DeepEqual(hist[2], wantAny) {
+ t.Errorf("replacement_history %v", hist)
+ }
+ if meta := p3["replacement_history_metadata"].([]any); len(meta) != 3 {
+ t.Errorf("metadata %v", meta)
+ }
+ if p3["window_number"] != float64(2) || p3["compaction_response_id"] != "resp_magpie_1761000002" {
+ t.Errorf("checkpoint fields changed: %s", got[3])
+ }
+ // the unreadable one is left out with its metadata entry, as the
+ // gateway leaves it out of what it sends
+ p4 := decode(t, got[4])["payload"].(map[string]any)
+ if h := p4["replacement_history"].([]any); len(h) != 1 {
+ t.Errorf("replacement_history %v", h)
+ }
+ if m := p4["replacement_history_metadata"].([]any); len(m) != 1 || !reflect.DeepEqual(m[0], map[string]any{"client_authored": true}) {
+ t.Errorf("metadata %v", m)
+ }
+ // nothing of magpie's that OpenAI would refuse is left
+ for i, l := range got {
+ if i != 1 && strings.Contains(l, codexsummary.Mark) {
+ t.Errorf("line %d still has magpie's item: %s", i, l)
+ }
+ }
+ // a second pass has nothing to do
+ if res, err := PortCodexRollouts(context.Background(), home); err != nil || len(res.Rewritten) != 0 {
+ t.Errorf("second pass: %+v, %v", res, err)
+ }
+ })
+ }
+}
+
+// A rollout whose thread a Codex has open (its writer lock held) is never
+// written under it: it is left as it was, kept for a later pass, and
+// written once the Codex lets it go. A Codex home without
+// thread-writer-locks, where there is no telling, is left alone.
+func TestPortCodexRolloutsLeavesAnOpenThread(t *testing.T) {
+ lines := portRolloutLines("s")
+ home, path := portHome(t, lines, false)
+ orig, _ := os.ReadFile(path)
+ lf, err := os.OpenFile(filepath.Join(home, "thread-writer-locks", portThread+".lock"), os.O_CREATE|os.O_RDWR, 0o644)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer lf.Close()
+ if got, err := filelock.TryLock(lf); !got || err != nil {
+ t.Fatalf("lock: %v %v", got, err)
+ }
+ res, err := PortCodexRollouts(context.Background(), home)
+ if err != nil || len(res.Held) != 1 || len(res.Rewritten) != 0 {
+ t.Fatalf("pass with the thread open: %+v, %v", res, err)
+ }
+ if b, _ := os.ReadFile(path); !bytes.Equal(b, orig) {
+ t.Fatal("written while a Codex had the thread open")
+ }
+ if left, _ := filepath.Glob(filepath.Join(filepath.Dir(path), ".magpie-*")); len(left) > 0 {
+ t.Fatalf("left behind: %v", left)
+ }
+ filelock.Unlock(lf)
+ // the pass after looks again at what it left, though its time is old
+ res, err = PortCodexRollouts(context.Background(), home)
+ if err != nil || len(res.Rewritten) != 1 {
+ t.Fatalf("pass after: %+v, %v", res, err)
+ }
+
+ home2, path2 := portHome(t, lines, false)
+ os.Remove(filepath.Join(home2, "thread-writer-locks"))
+ orig2, _ := os.ReadFile(path2)
+ if res, err := PortCodexRollouts(context.Background(), home2); err != nil || len(res.Rewritten) != 0 {
+ t.Fatalf("no writer locks: %+v, %v", res, err)
+ }
+ if b, _ := os.ReadFile(path2); !bytes.Equal(b, orig2) {
+ t.Fatal("written without Codex's writer locks")
+ }
+}
+
+// A line is parsed only for magpie's item; text that only mentions
+// "magpie1:" keeps its bytes, and a line that is one unreadable item of
+// magpie's alone stays (leaving it out would move the lines after it).
+func TestPortLineLeavesOthers(t *testing.T) {
+ for _, l := range []string{
+ `{"type":"response_item","payload":{"type":"message","role":"user","content":[{"type":"input_text","text":"magpie1:abc"}]}}` + "\n",
+ `{"type":"response_item","payload":{"type":"compaction","encrypted_content":"magpie1:%%%"}}` + "\r\n",
+ `{"type":"response_item","payload":{"type":"reasoning","encrypted_content":"magpie1:xyz"}}` + "\n",
+ `not json magpie1:` + "\n",
+ } {
+ if got, ok := portLine([]byte(l)); ok || string(got) != l {
+ t.Errorf("%q became %q", l, got)
+ }
+ }
+ l := `{"type":"response_item","payload":{"type":"compaction_summary","encrypted_content":"` + codexsummary.Encrypted("x") + `"}}` + "\r\n"
+ got, ok := portLine([]byte(l))
+ if !ok || !bytes.HasSuffix(got, []byte("}\r\n")) || bytes.Contains(got, []byte("magpie1:")) {
+ t.Errorf("compaction_summary: %q", got)
+ }
+}