Skip to content

Every agent's model. One place. Codex on DeepSeek, Claude Code on Kimi, from the menu bar.

15.7 KB
Raw Blame
package library

import (
	"bytes"
	"cmp"
	"errors"
	"fmt"
	"io"
	"io/fs"
	"maps"
	"os"
	"path/filepath"
	"slices"
	"strings"
	"time"
)

// Bundle is the library as a backup carries it to another computer: the
// instruction sets and the one on, what each agent gets besides, the MCP
// servers and the skills with their files. What magpie wrote into each
// agent (Applied) is this computer's own and stays out. It is made the
// same way every time — no times, everything in order — so two computers
// with the same library carry the same bundle.
type Bundle struct {
	Instructions Instructions      `json:"instructions"`
	Texts        map[string]string `json:"texts,omitempty"` // set → its text
	Extra        map[string]string `json:"extra,omitempty"` // agent → what it gets besides
	MCP          []*Server         `json:"mcp"`
	Skills       []*CarriedSkill   `json:"skills"`
	Icons        map[string]string `json:"icons,omitempty"`
}

// CarriedSkill is a skill with its folder's files.
type CarriedSkill struct {
	Name   string            `json:"name"`
	Source *Source           `json:"source,omitempty"`
	Agents []string          `json:"agents"`
	Files  map[string][]byte `json:"files"`          // by path in the folder, with /
	Exec   []string          `json:"exec,omitempty"` // the files that run
	Left   []string          `json:"left,omitempty"` // files too big to carry
}

// A skill's file bigger than maxCarried is left out, and so are the
// biggest once the skills come to maxCarriedAll. A skill's own files go
// with it, its scripts and its templates (a PowerPoint template with its
// pictures is several MB), but a sync reads at most 64 MiB of a backup
// (davsync), and a skill's bytes are base64 twice in one (the bundle's
// JSON, then the sealed envelope's): 32 MiB of files is about 57 MiB.
const (
	maxCarried    = 16 << 20
	maxCarriedAll = 32 << 20
)

// Collect is the library as it is now; nil when there is none yet.
func Collect() (*Bundle, error) {
	mu.Lock()
	defer mu.Unlock()
	if _, err := os.Stat(path()); errors.Is(err, fs.ErrNotExist) {
		return nil, nil
	} else if err != nil { // there, but unknown: never carried as no library (#2269)
		return nil, err
	}
	l, err := load()
	if err != nil {
		return nil, err
	}
	b := &Bundle{Instructions: l.Instructions, Texts: map[string]string{}, Extra: extras(), MCP: []*Server{}, Skills: []*CarriedSkill{}, Icons: l.Icons}
	b.Instructions.Agents = orNone(b.Instructions.Agents)
	// a set's own file is this computer's: its text goes, where it is
	// doesn't
	b.Instructions.Sets = slices.Clone(b.Instructions.Sets)
	for i := range b.Instructions.Sets {
		b.Instructions.Sets[i].File = ""
	}
	for _, s := range l.sets() {
		if text := setText(l.textPath(s.ID)); text != "" {
			b.Texts[s.ID] = text
		}
	}
	if len(b.Texts) == 0 {
		b.Texts = nil
	}
	if len(b.Extra) == 0 {
		b.Extra = nil
	}
	// a server kept to this computer stays out; another computer's by its
	// name, which a sync brought, goes on as it came (mcp_local.go)
	for _, s := range sorted(l.carriedServers(), func(s *Server) string { return s.Name }) {
		c := *carried(s) // magpie's own by name, each computer's own binary
		c.Agents = orNone(slices.Sorted(slices.Values(s.Agents)))
		b.MCP = append(b.MCP, &c)
	}
	skills := sorted(l.Skills, func(s *Skill) string { return s.Name })
	dirs := []string{}
	for _, s := range skills {
		dirs = append(dirs, realDir(skillDir(s.Name)))
	}
	for i, c := range readSkills(dirs) {
		s := skills[i]
		c.Name, c.Source, c.Agents = s.Name, s.Source, orNone(slices.Sorted(slices.Values(s.Agents)))
		b.Skills = append(b.Skills, c)
	}
	return b, nil
}

func sorted[T any](xs []T, key func(T) string) []T {
	return slices.SortedFunc(slices.Values(xs), func(a, b T) int { return strings.Compare(key(a), key(b)) })
}

func orNone(xs []string) []string {
	if xs == nil {
		return []string{}
	}
	return xs
}

// readSkills is each skill folder's files. Links in it, .git and what
// isn't a plain file stay out. What goes is chosen smallest first across
// every skill, so the text of them all comes before any big file.
func readSkills(dirs []string) []*CarriedSkill {
	type file struct {
		skill  int
		rel, p string
		size   int64
		runs   bool
	}
	var files []file
	out := make([]*CarriedSkill, len(dirs))
	for i, dir := range dirs {
		out[i] = &CarriedSkill{Files: map[string][]byte{}}
		filepath.WalkDir(dir, func(p string, d fs.DirEntry, err error) error {
			if err != nil {
				return nil
			}
			if d.IsDir() {
				if p != dir && (d.Name() == ".git" || d.Name() == marker) {
					return filepath.SkipDir
				}
				return nil
			}
			if !d.Type().IsRegular() || d.Name() == marker {
				return nil
			}
			rel, err := filepath.Rel(dir, p)
			if err != nil {
				return nil
			}
			fi, err := d.Info()
			if err != nil {
				return nil
			}
			files = append(files, file{i, filepath.ToSlash(rel), p, fi.Size(), fi.Mode().Perm()&0o111 != 0})
			return nil
		})
	}
	// smallest first, and the same order for the same files every time
	slices.SortFunc(files, func(a, b file) int {
		return cmp.Or(cmp.Compare(a.size, b.size), cmp.Compare(a.skill, b.skill), strings.Compare(a.rel, b.rel))
	})
	budget := int64(maxCarriedAll)
	for _, f := range files {
		c := out[f.skill]
		if f.size > maxCarried || f.size > budget {
			c.Left = append(c.Left, f.rel)
			continue
		}
		data, err := os.ReadFile(f.p)
		if err != nil {
			continue
		}
		if n := int64(len(data)); n > maxCarried || n > budget { // grew since
			c.Left = append(c.Left, f.rel)
			continue
		}
		budget -= int64(len(data))
		c.Files[f.rel] = data
		if f.runs {
			c.Exec = append(c.Exec, f.rel)
		}
	}
	for _, c := range out {
		slices.Sort(c.Exec)
		slices.Sort(c.Left)
	}
	return out
}

// check refuses a bundle that would write outside the library: a name
// that isn't one, a file's path that climbs out of its skill.
func (b *Bundle) check() error {
	for _, s := range b.Instructions.Sets {
		if err := checkName("set", s.ID); err != nil {
			return err
		}
	}
	for id := range b.Texts {
		if id != defaultSet && !slices.ContainsFunc(b.Instructions.Sets, func(s InstrSet) bool { return s.ID == id }) {
			return fmt.Errorf("a text for no set, %q", id)
		}
	}
	for id := range b.Extra {
		if err := checkAgent(id); err != nil {
			return err
		}
	}
	for _, s := range b.MCP {
		if s == nil {
			return fmt.Errorf("an empty server")
		}
		if err := s.check(); err != nil {
			return err
		}
	}
	seen := map[string]bool{}
	for _, s := range b.Skills {
		if s == nil {
			return fmt.Errorf("an empty skill")
		}
		if err := checkName("skill", s.Name); err != nil {
			return err
		}
		if seen[s.Name] {
			return fmt.Errorf("the skill %s twice", s.Name)
		}
		seen[s.Name] = true
		for _, rel := range slices.Concat(slices.Collect(maps.Keys(s.Files)), s.Left) {
			if !localFile(rel) {
				return fmt.Errorf("skill %s: %q is not a file in its folder", s.Name, rel)
			}
		}
	}
	return nil
}

// localFile reports whether rel, with /, names a file inside a folder:
// not absolute, no .., no \ to be read as a separator elsewhere.
func localFile(rel string) bool {
	return rel != "" && !strings.ContainsAny(rel, "\\:\x00") && filepath.IsLocal(filepath.FromSlash(rel)) &&
		!slices.Contains(strings.Split(rel, "/"), "..")
}

// WithoutSecrets empties the values of the servers' environment variables
// and headers that secret says hold one; their names stay, to be filled in.
func (b *Bundle) WithoutSecrets(secret func(string) bool) {
	for i, s := range b.MCP {
		c := *s
		c.Env, c.Headers = blank(s.Env, secret), blank(s.Headers, secret)
		// an OAuth app's secret is a key too; its client ID isn't
		if s.OAuth != nil && s.OAuth.ClientSecret != "" {
			a := *s.OAuth
			a.ClientSecret = ""
			c.OAuth = &a
		}
		b.MCP[i] = &c
	}
}

func blank(m map[string]string, secret func(string) bool) map[string]string {
	if m == nil {
		return nil
	}
	out := map[string]string{}
	for k, v := range m {
		if secret(k) {
			v = ""
		}
		out[k] = v
	}
	return out
}

// WithSecrets is b with the values WithoutSecrets emptied filled from the
// servers of the same name in have, so a bundle made without keys doesn't
// take away the ones kept here.
func (b *Bundle) WithSecrets(have *Bundle, secret func(string) bool) *Bundle {
	if have == nil {
		return b
	}
	out := *b
	out.MCP = nil
	for _, s := range b.MCP {
		c := *s
		i := slices.IndexFunc(have.MCP, func(h *Server) bool { return h.Name == s.Name })
		if i >= 0 {
			c.Env, c.Headers = filled(s.Env, have.MCP[i].Env, secret), filled(s.Headers, have.MCP[i].Headers, secret)
			// the same app's secret, kept here
			if h := have.MCP[i].OAuth; s.OAuth != nil && s.OAuth.ClientSecret == "" && h != nil && h.ClientID == s.OAuth.ClientID {
				a := *s.OAuth
				a.ClientSecret = h.ClientSecret
				c.OAuth = &a
			}
		}
		out.MCP = append(out.MCP, &c)
	}
	return &out
}

func filled(m, have map[string]string, secret func(string) bool) map[string]string {
	if m == nil {
		return nil
	}
	out := maps.Clone(m)
	for k, v := range m {
		if v == "" && secret(k) && have[k] != "" {
			out[k] = have[k]
		}
	}
	return out
}

// Put makes the library what b says — its instructions, servers and
// skills in place of the ones here — and writes it into the agents, which
// keep whatever of theirs isn't magpie's. The skills are written aside
// first, and only once all of them are there does any go in; a skill
// replaced or taken out is kept with the backups, and so is an
// instructions file that changes. A skill that comes linked to a folder
// this computer has too is linked to it; else it comes as its files. A
// skill whose files are as they are here is left alone.
func Put(b *Bundle) (*Result, error) {
	if err := b.check(); err != nil {
		return nil, err
	}
	return change(func(l *Library) error {
		if err := os.MkdirAll(skillsDir(), 0o755); err != nil {
			return err
		}
		stage, err := os.MkdirTemp(Dir(), ".incoming-") // beside skills, to be renamed in
		if err != nil {
			return err
		}
		defer os.RemoveAll(stage)

		keep, link := map[string]bool{}, map[string]string{}
		for _, s := range b.Skills {
			if cur := l.skill(s.Name); cur != nil && sameSource(cur.Source, s.Source) {
				if have := readSkills([]string{realDir(skillDir(s.Name))})[0]; sameFiles(have, s) {
					keep[s.Name] = true
					continue
				}
			}
			if src := s.Source; src != nil && src.Kind == "folder" && src.Dir != "" && isDir(src.Dir) {
				if to, err := os.Readlink(skillDir(s.Name)); err == nil && filepath.Clean(to) == filepath.Clean(src.Dir) {
					keep[s.Name] = true
				} else {
					link[s.Name] = src.Dir
				}
				continue
			}
			if err := writeSkill(filepath.Join(stage, s.Name), s); err != nil {
				return fmt.Errorf("skill %s: %w", s.Name, err)
			}
			if err := keepLeft(filepath.Join(stage, s.Name), realDir(skillDir(s.Name)), s); err != nil {
				return fmt.Errorf("skill %s: %w", s.Name, err)
			}
		}

		// every skill's files are ready: out with the old ones
		aside := filepath.Join(BackupDir(), time.Now().Format("2006-01-02_15-04-05.000"), "skills")
		out := func(name string) error {
			p := skillDir(name)
			fi, err := os.Lstat(p)
			if err != nil {
				return nil
			}
			if linkEntry(fi) {
				return os.Remove(p) // a folder of the user's: only the link goes
			}
			if err := os.MkdirAll(aside, 0o700); err != nil {
				return err
			}
			return move(p, filepath.Join(aside, name))
		}
		for _, s := range l.Skills {
			if !keep[s.Name] {
				if err := out(s.Name); err != nil {
					return err
				}
			}
		}
		skills := []*Skill{}
		for _, s := range b.Skills {
			// turned off here stays off here: that is this computer's own
			off := l.skill(s.Name) != nil && l.skill(s.Name).OffHere
			skills = append(skills, &Skill{Name: s.Name, Source: s.Source, Agents: slices.Clone(orNone(s.Agents)), OffHere: off})
			if keep[s.Name] {
				continue
			}
			if err := out(s.Name); err != nil { // one of no skill's, in the way
				return err
			}
			if dir, ok := link[s.Name]; ok {
				if err := dirLink(dir, skillDir(s.Name)); err != nil {
					if err := copyDir(dir, skillDir(s.Name)); err != nil {
						return err
					}
				}
				continue
			}
			if err := os.Rename(filepath.Join(stage, s.Name), skillDir(s.Name)); err != nil {
				return err
			}
		}
		l.Skills = skills

		// the instructions: every text here that b hasn't goes, kept first
		kept := newBackups()
		l.kept = kept
		texts := map[string]string{}
		// a set linked here to a file of the user's keeps it: the file is
		// given the text b has for the set, and never emptied for a text
		// b hasn't
		files := map[string]string{}
		for _, s := range l.sets() {
			if s.File != "" {
				files[s.ID] = s.File
			} else {
				texts[setPath(s.ID)] = ""
			}
		}
		for id := range extras() {
			texts[extraPath(id)] = ""
		}
		for id, text := range b.Texts {
			if files[id] == "" {
				texts[setPath(id)] = text
			}
		}
		for id, text := range b.Extra {
			texts[extraPath(id)] = text
		}
		for _, p := range slices.Sorted(maps.Keys(texts)) {
			if readText(p) == strings.TrimSpace(texts[p]) {
				continue
			}
			if err := kept.keep("library", p); err != nil {
				return err
			}
			if err := writeText(p, texts[p]); err != nil {
				return err
			}
		}
		l.Instructions = Instructions{Agents: slices.Sorted(slices.Values(orNone(b.Instructions.Agents))),
			Sets: slices.Clone(b.Instructions.Sets), Active: b.Instructions.Active}
		for i := range l.Instructions.Sets {
			s := &l.Instructions.Sets[i]
			s.File = files[s.ID]
			if text := strings.TrimSpace(b.Texts[s.ID]); s.File != "" && text != "" {
				if err := l.putText(s.File, text); err != nil {
					return err
				}
			}
		}

		l.putServers(b.MCP)
		l.Icons = maps.Clone(b.Icons)
		return nil
	})
}

func sameSource(a, b *Source) bool {
	return (a == nil) == (b == nil) && (a == nil || *a == *b)
}

// sameFiles reports whether a folder here has what c carries.
func sameFiles(have, c *CarriedSkill) bool {
	return maps.EqualFunc(have.Files, c.Files, bytes.Equal) && slices.Equal(have.Exec, sorted(c.Exec, func(s string) string { return s }))
}

// writeSkill writes a carried skill's files into a new folder.
func writeSkill(dir string, s *CarriedSkill) error {
	if err := os.MkdirAll(dir, 0o755); err != nil {
		return err
	}
	for _, rel := range slices.Sorted(maps.Keys(s.Files)) {
		p := filepath.Join(dir, filepath.FromSlash(rel))
		if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
			return err
		}
		mode := fs.FileMode(0o644)
		if slices.Contains(s.Exec, rel) {
			mode = 0o755
		}
		if err := os.WriteFile(p, s.Files[rel], mode); err != nil {
			return err
		}
	}
	return nil
}

// keepLeft copies into a skill being put (at dir) the files the other
// computer left out as too big to carry that the skill here has, so it
// is put with them, not without.
func keepLeft(dir, here string, s *CarriedSkill) error {
	for _, rel := range s.Left {
		if _, ok := s.Files[rel]; ok {
			continue
		}
		from := filepath.Join(here, filepath.FromSlash(rel))
		fi, err := os.Lstat(from)
		if err != nil || !fi.Mode().IsRegular() {
			continue
		}
		to := filepath.Join(dir, filepath.FromSlash(rel))
		if err := os.MkdirAll(filepath.Dir(to), 0o755); err != nil {
			return err
		}
		in, err := os.Open(from)
		if err != nil {
			return err
		}
		out, err := os.OpenFile(to, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, fi.Mode().Perm())
		if err == nil {
			_, err = io.Copy(out, in)
			err = errors.Join(err, out.Close())
		}
		in.Close()
		if err != nil {
			return err
		}
	}
	return nil
}

func isDir(p string) bool {
	fi, err := os.Stat(p)
	return err == nil && fi.IsDir()
}

// Changed is when the library was last changed: library.json, or a file
// of its own (links not followed).
func Changed() time.Time {
	var t time.Time
	if fi, err := os.Stat(path()); err == nil {
		t = fi.ModTime()
	}
	filepath.WalkDir(Dir(), func(p string, d fs.DirEntry, err error) error {
		if err != nil {
			return nil
		}
		if fi, err := d.Info(); err == nil && fi.ModTime().After(t) {
			t = fi.ModTime()
		}
		return nil
	})
	return t
}