Every agent's model. One place. Codex on DeepSeek, Claude Code on Kimi, from the menu bar.
package library
import (
"crypto/sha256"
_ "embed"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"html"
"io"
"maps"
"net/http"
"net/url"
"os"
"path/filepath"
"regexp"
"slices"
"sort"
"strings"
"sync"
"time"
"github.com/yetone/magpie/internal/appdir"
"github.com/yetone/magpie/internal/iconproxy"
)
// The market is where the page finds MCP servers and skills to add in one
// click: servers magpie picked and checked itself, then the MCP Registry's;
// skills from skills.sh, which counts how often each is installed.
// ---- servers ----------------------------------------------------------------
// Input is something a server needs from the user before it can start: a
// key in its environment, a header it sends, or an argument.
type Input struct {
Key string `json:"key"`
Where string `json:"where"` // env, header or arg
Label string `json:"label"`
Description string `json:"description,omitempty"`
Placeholder string `json:"placeholder,omitempty"`
Secret bool `json:"secret,omitempty"`
Required bool `json:"required,omitempty"`
format string // a header's value around the input, "Bearer {}"
}
// MarketServer is a server the market offers.
type MarketServer struct {
ID string `json:"id"`
Name string `json:"name"` // what the library calls it
Title string `json:"title"`
Description string `json:"description"`
Publisher string `json:"publisher,omitempty"`
Icon string `json:"icon,omitempty"`
Homepage string `json:"homepage,omitempty"`
Transport string `json:"transport"`
// Runs is what starts a local one: npx, uvx or docker
Runs string `json:"runs,omitempty"`
// SignIn is a remote one that has the agent sign in, in the browser,
// the first time it's used
SignIn bool `json:"signIn,omitempty"`
Featured bool `json:"featured,omitempty"`
// OptIn is one no agent is given unless it is picked for it
OptIn bool `json:"optIn,omitempty"`
Inputs []Input `json:"inputs"`
// Have is the library's server that is this one
Have string `json:"have,omitempty"`
// Match is one found at the address searched for: its repository, page,
// endpoint or package is it
Match bool `json:"match,omitempty"`
server Server
addrs []string // where it is, normalised (normAddr): matched by an address
}
func gh(owner string) string { return "https://github.com/" + owner + ".png?size=96" }
func remoteServer(id, title, publisher, icon, u, desc string, signIn bool, inputs ...Input) MarketServer {
return MarketServer{ID: id, Name: id, Title: title, Publisher: publisher, Icon: icon, Description: desc, Transport: "http",
SignIn: signIn, Featured: true, Inputs: inputs, server: Server{Name: id, Transport: "http", URL: u}}
}
func localServer(id, title, publisher, icon, homepage, desc, command string, args []string, inputs ...Input) MarketServer {
return MarketServer{ID: id, Name: id, Title: title, Publisher: publisher, Icon: icon, Homepage: homepage, Description: desc,
Transport: "stdio", Runs: command, Featured: true, Inputs: inputs, server: Server{Name: id, Transport: "stdio", Command: command, Args: args}}
}
func secretEnv(key, label, where string) Input {
return Input{Key: key, Where: "env", Label: label, Description: where, Secret: true, Required: true}
}
// featured are the servers magpie offers first. Every one was checked: its
// package is published, its endpoint answers.
var featured = []MarketServer{
selfServer(),
remoteServer("context7", "Context7", "Upstash", gh("upstash"), "https://mcp.context7.com/mcp",
"Up-to-date documentation and code examples for any library, straight into the prompt.", false,
Input{Key: "CONTEXT7_API_KEY", Where: "header", Label: "API key", Description: "Optional — raises the rate limit. From context7.com/dashboard", Secret: true, format: "{}"}),
localServer("playwright", "Playwright", "Microsoft", "https://playwright.dev/img/playwright-logo.svg", "https://github.com/microsoft/playwright-mcp",
"Drive a real browser: open pages, click, type and read them through the accessibility tree.", "npx", []string{"-y", "@playwright/mcp@latest"}),
localServer("chrome-devtools", "Chrome DevTools", "Google", gh("ChromeDevTools"), "https://github.com/ChromeDevTools/chrome-devtools-mcp",
"Control and inspect a live Chrome: console, network, performance traces and screenshots.", "npx", []string{"-y", "chrome-devtools-mcp@latest"}),
remoteServer("github", "GitHub", "GitHub", gh("github"), "https://api.githubcopilot.com/mcp/",
"Repositories, issues, pull requests, Actions and code search on GitHub.", false,
Input{Key: "Authorization", Where: "header", Label: "Personal access token", Description: "github.com/settings/personal-access-tokens", Placeholder: "github_pat_…", Secret: true, Required: true, format: "Bearer {}"}),
remoteServer("deepwiki", "DeepWiki", "Cognition", gh("CognitionAI"), "https://mcp.deepwiki.com/mcp",
"Ask questions about any public GitHub repository and read its generated wiki.", false),
remoteServer("exa", "Exa", "Exa", gh("exa-labs"), "https://mcp.exa.ai/mcp",
"Web search and code search built for agents, with clean page contents.", false),
localServer("filesystem", "Filesystem", "Model Context Protocol", gh("modelcontextprotocol"), "https://github.com/modelcontextprotocol/servers/tree/main/src/filesystem",
"Read, write and search files, limited to the folders you allow.", "npx", []string{"-y", "@modelcontextprotocol/server-filesystem"},
Input{Key: "dir", Where: "arg", Label: "Folder", Description: "The folder it may read and write", Placeholder: "~/code", Required: true}),
localServer("fetch", "Fetch", "Model Context Protocol", gh("modelcontextprotocol"), "https://github.com/modelcontextprotocol/servers/tree/main/src/fetch",
"Fetch a web page and hand it over as Markdown.", "uvx", []string{"mcp-server-fetch"}),
localServer("memory", "Memory", "Model Context Protocol", gh("modelcontextprotocol"), "https://github.com/modelcontextprotocol/servers/tree/main/src/memory",
"A knowledge graph the agent keeps across conversations.", "npx", []string{"-y", "@modelcontextprotocol/server-memory"}),
localServer("sequential-thinking", "Sequential Thinking", "Model Context Protocol", gh("modelcontextprotocol"), "https://github.com/modelcontextprotocol/servers/tree/main/src/sequentialthinking",
"Think a problem through step by step, revising and branching as it goes.", "npx", []string{"-y", "@modelcontextprotocol/server-sequential-thinking"}),
localServer("git", "Git", "Model Context Protocol", gh("modelcontextprotocol"), "https://github.com/modelcontextprotocol/servers/tree/main/src/git",
"Read, search and change Git repositories.", "uvx", []string{"mcp-server-git"}),
localServer("time", "Time", "Model Context Protocol", gh("modelcontextprotocol"), "https://github.com/modelcontextprotocol/servers/tree/main/src/time",
"The current time anywhere, and conversions between time zones.", "uvx", []string{"mcp-server-time"}),
remoteServer("sentry", "Sentry", "Sentry", gh("getsentry"), "https://mcp.sentry.dev/mcp",
"Issues, errors and traces from Sentry, and Seer's analysis of them.", true),
remoteServer("linear", "Linear", "Linear", gh("linear"), "https://mcp.linear.app/mcp",
"Find, create and update Linear issues, projects and comments.", true),
remoteServer("notion", "Notion", "Notion", gh("makenotion"), "https://mcp.notion.com/mcp",
"Search, read and write pages and databases in your Notion workspace.", true),
remoteServer("atlassian", "Atlassian", "Atlassian", gh("atlassian"), "https://mcp.atlassian.com/v1/mcp",
"Jira issues and Confluence pages from Atlassian Cloud.", true),
remoteServer("supabase", "Supabase", "Supabase", gh("supabase"), "https://mcp.supabase.com/mcp",
"Manage Supabase projects: tables, SQL, migrations, logs and edge functions.", true),
remoteServer("stripe", "Stripe", "Stripe", gh("stripe"), "https://mcp.stripe.com",
"Customers, payments, subscriptions and Stripe's documentation.", true),
remoteServer("vercel", "Vercel", "Vercel", gh("vercel"), "https://mcp.vercel.com",
"Projects, deployments and logs on Vercel, and its documentation.", true),
remoteServer("neon", "Neon", "Neon", gh("neondatabase"), "https://mcp.neon.tech/mcp",
"Serverless Postgres on Neon: projects, branches and SQL.", true),
remoteServer("cloudflare-docs", "Cloudflare Docs", "Cloudflare", gh("cloudflare"), "https://docs.mcp.cloudflare.com/mcp",
"Search Cloudflare's documentation.", false),
remoteServer("microsoft-learn", "Microsoft Learn", "Microsoft", gh("microsoft"), "https://learn.microsoft.com/api/mcp",
"Official Microsoft and Azure documentation and code samples.", false),
remoteServer("huggingface", "Hugging Face", "Hugging Face", gh("huggingface"), "https://huggingface.co/mcp",
"Search models, datasets, Spaces and papers on the Hugging Face Hub.", false),
localServer("brave-search", "Brave Search", "Brave", gh("brave"), "https://github.com/brave/brave-search-mcp-server",
"Web, news, image and local search from Brave's independent index.", "npx", []string{"-y", "@brave/brave-search-mcp-server"},
secretEnv("BRAVE_API_KEY", "API key", "brave.com/search/api")),
localServer("firecrawl", "Firecrawl", "Firecrawl", gh("firecrawl"), "https://github.com/firecrawl/firecrawl-mcp-server",
"Scrape, crawl and search the web into clean Markdown.", "npx", []string{"-y", "firecrawl-mcp"},
secretEnv("FIRECRAWL_API_KEY", "API key", "firecrawl.dev/app/api-keys")),
localServer("tavily", "Tavily", "Tavily", gh("tavily-ai"), "https://github.com/tavily-ai/tavily-mcp",
"Search and extract from the web with Tavily.", "npx", []string{"-y", "tavily-mcp"},
secretEnv("TAVILY_API_KEY", "API key", "app.tavily.com")),
}
// magpieCommand stands for this magpie's own binary in a featured server's
// command, which InstallServer puts in when it is added, and in a library
// carried to another computer, which reads it as its own (self_here.go).
const magpieCommand = "magpie"
// selfServerName is the name magpie's own image generation server is written
// under, in every agent's config.
const selfServerName = "magpie-image"
// selfServer is magpie's own image generation server: generate_image makes
// an image with the model Settings → Images → Image generation names, saved
// in the project. It costs what the model does, so it is given only to the
// agents picked for it.
func selfServer() MarketServer {
m := localServer(selfServerName, "Magpie Image", "Magpie", "https://usemagpie.ai/favicon.png", "https://usemagpie.ai",
"Generate and edit images with the image model set in Magpie (Settings → Images), and make short videos with a video model set up in Magpie (Grok Imagine, Seedance), saved in the project.", magpieCommand, []string{"mcp", "image"})
m.OptIn = true
return m
}
// selfCommand is the server's command and env for this magpie: its binary
// by full path, and the gateway's address when it isn't the default one.
func selfCommand(s *Server) {
if s.Command != magpieCommand {
return
}
if exe, err := selfExe(); err == nil {
s.Command = exe
}
if a := os.Getenv("MAGPIE_ADDR"); a != "" {
if s.Env == nil {
s.Env = map[string]string{}
}
s.Env["MAGPIE_ADDR"] = a
}
}
// homepages are the remote servers' pages, where their endpoints aren't.
var homepages = map[string]string{
"context7": "https://context7.com",
"github": "https://github.com/github/github-mcp-server",
"deepwiki": "https://deepwiki.com",
"exa": "https://docs.exa.ai/reference/exa-mcp",
"sentry": "https://docs.sentry.io/product/sentry-mcp/",
"linear": "https://linear.app/docs/mcp",
"notion": "https://developers.notion.com/docs/mcp",
"atlassian": "https://www.atlassian.com/platform/remote-mcp-server",
"supabase": "https://supabase.com/docs/guides/getting-started/mcp",
"stripe": "https://docs.stripe.com/mcp",
"vercel": "https://vercel.com/docs/mcp/vercel-mcp",
"neon": "https://neon.com/docs/ai/neon-mcp-server",
"cloudflare-docs": "https://github.com/cloudflare/mcp-server-cloudflare",
"microsoft-learn": "https://github.com/MicrosoftDocs/mcp",
"huggingface": "https://huggingface.co/settings/mcp",
}
func init() {
for i := range featured {
m := &featured[i]
if m.Homepage == "" {
m.Homepage = homepages[m.ID]
}
if m.Inputs == nil {
m.Inputs = []Input{}
}
m.addrs = addrsOf(m.Homepage, m.server.URL, packageOf(&m.server))
}
}
// registryURL is the MCP Registry's; a variable for tests.
var registryURL = "https://registry.modelcontextprotocol.io/v0/servers"
// seenServers are the registry's servers the page was last shown, for one to
// be installed by its id.
var seenServers = struct {
sync.Mutex
m map[string]MarketServer
}{m: map[string]MarketServer{}}
// MarketServers is what the market shows for a search: the featured servers
// that match it, then the registry's. A search that is an address — a
// repository, a page, an endpoint, a package — finds the ones at it.
func MarketServers(q string) ([]MarketServer, error) {
q = strings.ToLower(strings.TrimSpace(q))
addr := address(q)
var out []MarketServer
for _, m := range featured {
switch {
case addr != "":
if addrScore(addr, m.addrs) > 0 {
m.Match = true
out = append(out, m)
}
case q == "" || strings.Contains(strings.ToLower(m.Title+" "+m.ID+" "+m.Publisher+" "+m.Description+" "+packageOf(&m.server)), q):
out = append(out, m)
}
}
var err error
if q != "" {
var found []MarketServer
if addr != "" {
found, err = searchRegistryAt(addr)
} else {
found, err = searchRegistry(q)
}
for _, f := range found {
if !slices.ContainsFunc(out, func(m MarketServer) bool { return serverKey(&m.server) == serverKey(&f.server) }) {
out = append(out, f)
}
}
}
if out == nil {
out = []MarketServer{}
}
if l, lerr := loadLocked(); lerr == nil {
for i := range out {
out[i].Have = l.haveServer(&out[i].server)
}
}
return out, err
}
// serverKey is what a server is, whatever it's called: its URL, or the
// command and the package it starts.
func serverKey(s *Server) string {
if s.Remote() {
return strings.TrimSuffix(strings.ToLower(s.URL), "/")
}
if len(s.Args) == 2 && s.Args[0] == "mcp" && s.Args[1] == "image" {
return "magpie mcp image" // magpie's own, wherever its binary is
}
for _, a := range s.Args {
if a == "" || strings.HasPrefix(a, "-") || a == "run" {
continue
}
if i := strings.LastIndex(a, "@"); i > 0 { // @scope/pkg@latest → @scope/pkg
a = a[:i]
}
return s.Command + " " + a
}
return s.Command
}
func (l *Library) haveServer(s *Server) string {
k := serverKey(s)
for _, x := range l.MCP {
if serverKey(x) == k {
return x.Name
}
}
return ""
}
type registryServer struct {
Name string `json:"name"`
Title string `json:"title"`
Description string `json:"description"`
WebsiteURL string `json:"websiteUrl"`
Repository struct {
URL string `json:"url"`
} `json:"repository"`
Icons []struct {
Src string `json:"src"`
} `json:"icons"`
Remotes []struct {
Type string `json:"type"`
URL string `json:"url"`
Headers []registryVar `json:"headers"`
} `json:"remotes"`
Packages []struct {
RegistryType string `json:"registryType"`
Identifier string `json:"identifier"`
Transport struct{ Type string } `json:"transport"`
Env []registryVar `json:"environmentVariables"`
Arguments []struct {
Type string `json:"type"`
Name string `json:"name"`
Value string `json:"value"`
Default string `json:"default"`
ValueHint string `json:"valueHint"`
Description string `json:"description"`
IsRequired bool `json:"isRequired"`
} `json:"packageArguments"`
} `json:"packages"`
}
type registryVar struct {
Name string `json:"name"`
Value string `json:"value"`
Description string `json:"description"`
IsRequired bool `json:"isRequired"`
IsSecret bool `json:"isSecret"`
}
var placeholderRe = regexp.MustCompile(`\{[^{}]*\}`)
// listed is an active server the registry lists, as the market
// offers it.
type listed struct {
m MarketServer
r registryServer
}
// fetchRegistry is what the registry lists for a search, which it matches
// against servers' names; the same server once.
func fetchRegistry(q string) ([]listed, error) {
u := registryURL + "?" + url.Values{"search": {q}, "limit": {"100"}, "version": {"latest"}}.Encode()
var body struct {
Servers []struct {
Server registryServer `json:"server"`
Meta map[string]struct {
Status string `json:"status"`
} `json:"_meta"`
} `json:"servers"`
}
if err := getJSONBy(registryClient, u, &body); err != nil {
return nil, &SearchError{Source: "the MCP Registry", Err: err}
}
var list []listed
seen := map[string]bool{}
for _, e := range body.Servers {
if st := e.Meta["io.modelcontextprotocol.registry/official"].Status; st != "" && st != "active" {
continue
}
m, ok := fromRegistry(&e.Server)
if !ok || seen[serverKey(&m.server)] {
continue
}
seen[serverKey(&m.server)] = true
list = append(list, listed{m, e.Server})
}
return list, nil
}
type scored struct {
m MarketServer
score int
}
func searchRegistry(q string) ([]MarketServer, error) {
found, err := fetchRegistry(q)
if err != nil {
return nil, err
}
var list []scored
for _, e := range found {
list = append(list, scored{e.m, rank(&e.r, e.m, q)})
}
return offer(list), nil
}
// searchRegistryAt finds the registry's servers at an address. The registry
// searches names only, so it's asked for the names the address suggests (a
// repository's, its owner's), and what it lists is kept when its repository,
// page, endpoint or package is the address. When none is, what it lists for
// the first name is offered, as a search for that name would.
func searchRegistryAt(addr string) ([]MarketServer, error) {
var matched, byName []scored
seen := map[string]bool{}
for i, term := range addrTerms(addr) {
found, err := fetchRegistry(term)
if err != nil {
if i == 0 {
return nil, err
}
break
}
for _, e := range found {
if i == 0 {
byName = append(byName, scored{e.m, rank(&e.r, e.m, term)})
}
k := serverKey(&e.m.server)
if seen[k] {
continue
}
if sc := addrScore(addr, e.m.addrs); sc > 0 {
seen[k] = true
e.m.Match = true
matched = append(matched, scored{e.m, sc*1000 + rank(&e.r, e.m, term)})
}
}
}
if len(matched) > 0 {
return offer(matched), nil
}
return offer(byName), nil
}
// offer is the best of a search's servers, best first, kept for one to be
// installed by its id.
func offer(list []scored) []MarketServer {
sort.SliceStable(list, func(i, j int) bool { return list[i].score > list[j].score })
out := []MarketServer{}
seenServers.Lock()
defer seenServers.Unlock()
if len(seenServers.m) > 2000 {
clear(seenServers.m)
}
for i, s := range list {
if i == 30 {
break
}
seenServers.m[s.m.ID] = s.m
out = append(out, s.m)
}
return out
}
// rank puts first what's called what was searched for, then what says it
// is, with an icon and a repository; the registry lists them by name.
func rank(r *registryServer, m MarketServer, q string) int {
s := 0
short := strings.ToLower(r.Name[strings.LastIndex(r.Name, "/")+1:])
switch {
case m.Name == q || strings.ToLower(r.Title) == q || short == q:
s += 100
case strings.Contains(short, q):
s += 40
}
if strings.Contains(strings.ToLower(r.Title), q) {
s += 20
}
if strings.Contains(strings.ToLower(r.Description), q) {
s += 10
}
if len(r.Icons) > 0 {
s += 8
}
if r.Repository.URL != "" {
s += 5
}
if strings.HasPrefix(r.Name, "ai.smithery/") { // wrappers around others' servers
s -= 60
}
return s
}
// fromRegistry is what a registry entry becomes in the library: its remote
// when it has one, else the package it's published as.
func fromRegistry(r *registryServer) (MarketServer, bool) {
m := MarketServer{ID: r.Name, Name: shortName(r.Name), Title: r.Title, Description: r.Description, Homepage: r.WebsiteURL, Inputs: []Input{}}
m.addrs = addrsOf(r.Name, r.WebsiteURL, r.Repository.URL)
for _, rm := range r.Remotes {
m.addrs = append(m.addrs, addrsOf(rm.URL)...)
}
for _, p := range r.Packages {
m.addrs = append(m.addrs, addrsOf(p.Identifier)...)
}
if m.Title == "" {
m.Title = m.Name
}
if m.Homepage == "" {
m.Homepage = r.Repository.URL
}
owner := githubOwner(r.Repository.URL)
if owner == "" {
if rest, ok := strings.CutPrefix(r.Name, "io.github."); ok {
owner, _, _ = strings.Cut(rest, "/")
}
}
switch {
case owner != "":
m.Publisher = owner
default:
ns, _, _ := strings.Cut(r.Name, "/")
parts := strings.Split(ns, ".")
slices.Reverse(parts)
m.Publisher = strings.Join(parts, ".")
}
for _, ic := range r.Icons {
if strings.HasPrefix(ic.Src, "https://") {
m.Icon = ic.Src
break
}
}
if m.Icon == "" && owner != "" {
m.Icon = gh(owner)
}
s := &m.server
s.Name = m.Name
for _, want := range []string{"streamable-http", "sse"} {
for _, rm := range r.Remotes {
if rm.Type != want || strings.Contains(rm.URL, "{") || !strings.HasPrefix(rm.URL, "https://") {
continue
}
s.Transport, s.URL = map[string]string{"streamable-http": "http", "sse": "sse"}[want], rm.URL
for _, h := range rm.Headers {
switch {
case h.Value != "" && !placeholderRe.MatchString(h.Value):
if s.Headers == nil {
s.Headers = map[string]string{}
}
s.Headers[h.Name] = h.Value
case h.IsRequired || h.IsSecret:
f := "{}"
if h.Value != "" {
f = placeholderRe.ReplaceAllString(h.Value, "{}")
}
m.Inputs = append(m.Inputs, Input{Key: h.Name, Where: "header", Label: h.Name, Description: h.Description,
Secret: h.IsSecret, Required: h.IsRequired, format: f})
}
}
m.Transport = s.Transport
return m, true
}
}
for _, want := range []string{"npm", "pypi", "oci"} {
for _, p := range r.Packages {
if p.RegistryType != want || p.Identifier == "" || (p.Transport.Type != "" && p.Transport.Type != "stdio") {
continue
}
s.Transport, m.Transport = "stdio", "stdio"
switch want {
case "npm":
s.Command, s.Args = "npx", []string{"-y", p.Identifier}
case "pypi":
s.Command, s.Args = "uvx", []string{p.Identifier}
case "oci":
s.Command, s.Args = "docker", []string{"run", "-i", "--rm"}
}
m.Runs = s.Command
for _, e := range p.Env {
if !e.IsRequired && !e.IsSecret {
continue
}
m.Inputs = append(m.Inputs, Input{Key: e.Name, Where: "env", Label: e.Name, Description: e.Description, Secret: e.IsSecret, Required: e.IsRequired})
if want == "oci" {
s.Args = append(s.Args, "-e", e.Name)
}
}
if want == "oci" {
s.Args = append(s.Args, p.Identifier)
}
for i, a := range p.Arguments {
v := a.Value
if v == "" {
v = a.Default
}
if placeholderRe.MatchString(v) {
v = ""
}
switch {
case v != "" && a.Type == "named":
s.Args = append(s.Args, a.Name, v)
case v != "":
s.Args = append(s.Args, v)
case a.IsRequired && a.Type != "named":
label := a.ValueHint
if label == "" {
label = fmt.Sprintf("Argument %d", i+1)
}
m.Inputs = append(m.Inputs, Input{Key: fmt.Sprintf("arg%d", i), Where: "arg", Label: label, Description: a.Description, Required: true})
case a.IsRequired:
return MarketServer{}, false // a named argument magpie can't fill
}
}
return m, true
}
}
return MarketServer{}, false
}
// shortName is a registry name as the library would call it:
// io.github.upstash/context7-mcp is context7.
func shortName(n string) string {
n = strings.ToLower(n[strings.LastIndex(n, "/")+1:])
orig := n
for _, p := range []string{"mcp-server-", "server-", "mcp-"} {
n = strings.TrimPrefix(n, p)
}
for _, s := range []string{"-mcp-server", "-mcp", "-server", "_mcp", ".mcp"} {
n = strings.TrimSuffix(n, s)
}
if n == "" {
n = orig
}
n = strings.Trim(unsafe.ReplaceAllString(n, "-"), "-_")
if len(n) > 64 {
n = n[:64]
}
if n == "" || !nameRe.MatchString(n) {
n = "server"
}
return n
}
func lastPart(p string) string { return p[strings.LastIndex(p, "/")+1:] }
func githubOwner(u string) string {
p, err := url.Parse(u)
if err != nil || (p.Host != "github.com" && p.Host != "www.github.com") {
return ""
}
owner, _, _ := strings.Cut(strings.Trim(p.Path, "/"), "/")
return owner
}
func marketServer(id string) (MarketServer, bool) {
if i := slices.IndexFunc(featured, func(m MarketServer) bool { return m.ID == id }); i >= 0 {
return featured[i], true
}
seenServers.Lock()
defer seenServers.Unlock()
m, ok := seenServers.m[id]
return m, ok
}
// InstallServer adds a market server to the library, filled in with values
// (by input key), and gives it to the agents named — to every agent that can
// have it when none are.
func InstallServer(id string, values map[string]string, agents []string) (*Result, error) {
m, ok := marketServer(id)
if !ok {
return nil, fmt.Errorf("the market has no server %s; search for it again", id)
}
s := m.server
s.Args = slices.Clone(s.Args)
s.Env, s.Headers = maps.Clone(s.Env), maps.Clone(s.Headers)
selfCommand(&s)
for _, in := range m.Inputs {
v := strings.TrimSpace(values[in.Key])
if v == "" {
if in.Required {
return nil, fmt.Errorf("%s needs %s", m.Title, in.Label)
}
continue
}
switch in.Where {
case "env":
if s.Env == nil {
s.Env = map[string]string{}
}
s.Env[in.Key] = v
case "header":
if s.Headers == nil {
s.Headers = map[string]string{}
}
f := in.format
if f == "" {
f = "{}"
}
s.Headers[in.Key] = strings.Replace(f, "{}", v, 1)
case "arg":
s.Args = append(s.Args, expand(v))
}
}
if agents == nil && m.OptIn {
agents = []string{}
}
if agents == nil {
for _, t := range Targets() {
if t.MCP != nil && t.MCP.supports(&s) == nil {
agents = append(agents, t.Agent.ID)
}
}
}
s.Agents = agents
if err := s.check(); err != nil {
return nil, err
}
return change(func(l *Library) error {
if have := l.haveServer(&s); have != "" {
return fmt.Errorf("the library has it already, as %s", have)
}
base := s.Name
for i := 2; l.server(s.Name) != nil; i++ {
s.Name = fmt.Sprintf("%s-%d", base, i)
}
s.Agents = slices.Sorted(slices.Values(s.Agents))
l.MCP = append(l.MCP, &s)
if m.Icon != "" {
if l.Icons == nil {
l.Icons = map[string]string{}
}
l.Icons[serverKey(&s)] = m.Icon
}
return nil
})
}
// serverIcon is a server's own icon, where the market knows it: the
// featured server it is, the one it was added from, or the featured one it
// is named after. None is made up for the rest.
func serverIcon(l *Library, s *Server) string {
k := serverKey(s)
for _, m := range featured {
if serverKey(&m.server) == k {
return m.Icon
}
}
if u := l.Icons[k]; u != "" {
return u
}
for _, m := range featured {
if strings.EqualFold(m.ID, s.Name) {
return m.Icon
}
}
return ""
}
// skillIcon is its repository owner's avatar, for a skill from GitHub.
func skillIcon(s *Skill) string {
if s.Source == nil || s.Source.Kind != "github" {
return ""
}
owner, _, ok := strings.Cut(s.Source.Repo, "/")
if !ok || owner == "" {
return ""
}
return gh(owner)
}
// ---- skills -----------------------------------------------------------------
// MarketSkill is a skill the market offers, from a GitHub repository.
type MarketSkill struct {
ID string `json:"id"` // owner/repo/skill
Source string `json:"source"` // owner/repo
SkillID string `json:"skillId"`
Name string `json:"name"`
Installs int `json:"installs"`
Official bool `json:"official,omitempty"`
// Featured is magpie's own, not one skills.sh lists
Featured bool `json:"featured,omitempty"`
Icon string `json:"icon"`
// Description is known once it has been fetched
Description string `json:"description,omitempty"`
Have string `json:"have,omitempty"`
}
// featuredSkills are magpie's own skills, first in the market: in
// magpie-community/plugins, beside its plugins. magpie-setup has an agent
// set magpie up as the user asks, through its commands (Leyuu_5 on X: so
// many features that an AI could do the setting up); magpie-quota reads
// what magpie quota tells (ttmouse on X).
var featuredSkills = []MarketSkill{{
Source: "magpie-community/plugins", SkillID: "magpie-setup", Name: "magpie-setup", Official: true, Featured: true,
Description: "Set magpie up as you ask, through its own commands: add keys and subscriptions, point agents at models, build routing groups with fallbacks and time-of-day rules, give devices gateway keys with budgets, and share MCP servers and instructions with every agent.",
}, {
Source: "magpie-community/plugins", SkillID: "magpie-quota", Name: "magpie-quota", Official: true, Featured: true,
Description: "Check how much is left of the user's AI subscriptions, coding plans and API key balances through magpie: each usage window's percent used and when it resets, each key's balance, and wait until a subscription has allowance again.",
}}
type skillsShEntry struct {
Source string `json:"source"`
SkillID string `json:"skillId"`
Name string `json:"name"`
Installs int `json:"installs"`
Official bool `json:"isOfficial,omitempty"`
}
// skillsShURL is skills.sh; a variable for tests.
var skillsShURL = "https://www.skills.sh"
//go:embed market_skills.json
var skillsSnapshot []byte
var repoRe = regexp.MustCompile(`^[\w.-]+/[\w.-]+$`)
var popular = struct {
sync.Mutex
list []skillsShEntry
at time.Time
}{}
func marketCache(name string) string {
d, err := appdir.SystemCache()
if err != nil {
return ""
}
return filepath.Join(d, "market", name)
}
// popularSkills are skills.sh's most installed, read off its front page,
// which lists them for itself; kept for six hours, and on disk for when
// it can't be reached, else what magpie shipped with.
func popularSkills() []skillsShEntry {
popular.Lock()
defer popular.Unlock()
if popular.list != nil && time.Since(popular.at) < 6*time.Hour {
return popular.list
}
cache := marketCache("skills.json")
if list, err := fetchPopular(); err == nil && len(list) > 0 {
popular.list, popular.at = list, time.Now()
if b, err := json.Marshal(list); err == nil && cache != "" {
_ = os.MkdirAll(filepath.Dir(cache), 0o755)
_ = os.WriteFile(cache, b, 0o644)
}
return list
}
var list []skillsShEntry
if b, err := os.ReadFile(cache); err != nil || json.Unmarshal(b, &list) != nil || len(list) == 0 {
_ = json.Unmarshal(skillsSnapshot, &list)
}
popular.list, popular.at = list, time.Now().Add(-6*time.Hour+5*time.Minute) // try again in a while
return list
}
func fetchPopular() ([]skillsShEntry, error) {
b, err := get(skillsShURL+"/", "text/html", 8<<20)
if err != nil {
return nil, err
}
// the list is in the page's data as JSON inside a JS string, its quotes escaped
const key = `initialSkills\":`
i := strings.Index(string(b), key)
if i < 0 {
return nil, errors.New("skills.sh's page has no list of skills")
}
s := strings.NewReplacer(`\\`, `\`, `\"`, `"`).Replace(string(b[i+len(key):]))
var list []skillsShEntry
if err := json.NewDecoder(strings.NewReader(s)).Decode(&list); err != nil {
return nil, fmt.Errorf("skills.sh's list: %w", err)
}
return slices.DeleteFunc(list, func(e skillsShEntry) bool { return !repoRe.MatchString(e.Source) || e.SkillID == "" }), nil
}
func searchSkillsSh(q string) ([]skillsShEntry, error) {
var body struct {
Skills []skillsShEntry `json:"skills"`
}
if err := getJSON(skillsShURL+"/api/search?"+url.Values{"q": {q}, "limit": {"40"}}.Encode(), &body); err != nil {
return nil, &SearchError{Source: "skills.sh", Err: err}
}
return slices.DeleteFunc(body.Skills, func(e skillsShEntry) bool { return !repoRe.MatchString(e.Source) || e.SkillID == "" }), nil
}
// MarketSkills is what the market shows for a search: magpie's own skills
// that match it, the most installed skills that do, then what skills.sh
// finds for it.
func MarketSkills(q string) ([]MarketSkill, error) {
q = strings.ToLower(strings.TrimSpace(q))
var list []skillsShEntry
var err error
for _, e := range popularSkills() {
if q == "" || strings.Contains(strings.ToLower(e.Name+" "+e.Source+" "+e.SkillID), q) {
list = append(list, e)
}
}
if q == "" {
// a few each from the repositories with the most, for the first page
// not to be one author's
per := map[string]int{}
list = slices.DeleteFunc(list, func(e skillsShEntry) bool { per[e.Source]++; return per[e.Source] > 4 })
}
if len(q) >= 2 {
var found []skillsShEntry
found, err = searchSkillsSh(q)
for _, f := range found {
if !slices.ContainsFunc(list, func(e skillsShEntry) bool { return e.Source == f.Source && e.SkillID == f.SkillID }) {
list = append(list, f)
}
}
}
if len(list) > 60 {
list = list[:60]
}
about := cachedAbout()
l, _ := loadLocked()
out := []MarketSkill{}
for _, f := range featuredSkills {
if q != "" && !strings.Contains(strings.ToLower(f.Name+" "+f.Source+" "+f.Description), q) {
continue
}
owner, _, _ := strings.Cut(f.Source, "/")
f.ID, f.Icon = f.Source+"/"+f.SkillID, gh(owner)
if l != nil {
f.Have = l.haveSkill(f.Source, f.SkillID, f.Name)
}
// once skills.sh lists it too, its count of installs is told, and
// it isn't shown twice
list = slices.DeleteFunc(list, func(e skillsShEntry) bool {
if e.Source == f.Source && e.SkillID == f.SkillID {
f.Installs = e.Installs
return true
}
return false
})
out = append(out, f)
}
for _, e := range list {
owner, _, _ := strings.Cut(e.Source, "/")
ms := MarketSkill{ID: e.Source + "/" + e.SkillID, Source: e.Source, SkillID: e.SkillID, Name: e.Name, Installs: e.Installs,
Official: e.Official, Icon: gh(owner)}
ms.Description = about[ms.ID]
if l != nil {
ms.Have = l.haveSkill(e.Source, e.SkillID, e.Name)
}
out = append(out, ms)
}
return out, err
}
func (l *Library) haveSkill(source, id, name string) string {
for _, s := range l.Skills {
if s.Source != nil && s.Source.Kind == "github" && strings.EqualFold(s.Source.Repo, source) &&
(s.Name == id || s.Name == name || lastPart(s.Source.Path) == id) {
return s.Name
}
}
// one taken in from an agent's folder (npx skills puts them in
// ~/.agents/skills) has no source to match, and adding is turned away
// by name all the same (InstallMarketSkill): it's had by its name
for _, n := range []string{id, name} {
if s := l.skill(n); s != nil {
return s.Name
}
}
return ""
}
var abouts = struct {
sync.Mutex
m map[string]string
loaded bool
}{m: map[string]string{}}
func cachedAbout() map[string]string {
abouts.Lock()
defer abouts.Unlock()
if !abouts.loaded {
abouts.loaded = true
if b, err := os.ReadFile(marketCache("about.json")); err == nil {
_ = json.Unmarshal(b, &abouts.m)
}
}
return maps.Clone(abouts.m)
}
// SkillsAbout is what each skill says of itself, by id (owner/repo/skill),
// from its SKILL.md; kept on disk once known.
func SkillsAbout(ids []string) map[string]string {
have := cachedAbout()
out := map[string]string{}
var wg sync.WaitGroup
var mu sync.Mutex
sem := make(chan struct{}, 6)
for _, id := range ids {
if d, ok := have[id]; ok {
out[id] = d
continue
}
if strings.Count(id, "/") != 2 || !repoRe.MatchString(id[:strings.LastIndex(id, "/")]) {
continue
}
wg.Add(1)
go func() {
defer wg.Done()
sem <- struct{}{}
defer func() { <-sem }()
// the skill's page on skills.sh says what it's for — its
// download API gives the SKILL.md itself, but only 60 an hour
b, err := get(skillsShURL+"/"+id[:strings.LastIndex(id, "/")+1]+url.PathEscape(id[strings.LastIndex(id, "/")+1:]), "text/html", 4<<20)
if err != nil {
return
}
if d := metaDescription(b); d != "" {
mu.Lock()
out[id] = d
mu.Unlock()
}
}()
}
wg.Wait()
abouts.Lock()
changed := false
for id, d := range out {
if _, ok := abouts.m[id]; !ok {
abouts.m[id], changed = d, true
}
}
if changed {
if b, err := json.Marshal(abouts.m); err == nil {
p := marketCache("about.json")
_ = os.MkdirAll(filepath.Dir(p), 0o755)
_ = os.WriteFile(p, b, 0o644)
}
}
abouts.Unlock()
return out
}
// InstallMarketSkill adds a skill from the market, found in its repository,
// and gives it to the agents named — to every agent that can have skills
// when none are.
func InstallMarketSkill(source, id string, agents []string) (*Result, error) {
if !repoRe.MatchString(source) {
return nil, fmt.Errorf("%s isn't a GitHub repository", source)
}
// only this skill's folder, when the repository is too big to
// download whole (#2794)
p, err := cachedProbeFor(source, id)
if err != nil {
return nil, err
}
i := slices.IndexFunc(p.Candidates, func(c Candidate) bool { return c.Name == id })
if i < 0 {
i = slices.IndexFunc(p.Candidates, func(c Candidate) bool { return lastPart(c.Path) == id })
}
if i < 0 {
return nil, fmt.Errorf("%s has no skill %s any more", source, id)
}
if agents == nil {
for _, t := range Targets() {
if t.Skills != "" {
agents = append(agents, t.Agent.ID)
}
}
}
// the market showed this one skill: the repository's others are
// offered as new by a check, beside it
path := p.Candidates[i].Path
return installChange(func(l *Library, in *installed) error { return installFrom(l, p, []string{path}, agents, false, in) })
}
// ---- icons ------------------------------------------------------------------
// Icon is a market icon, fetched once and kept on disk: only an icon the
// market gave out is fetched, so the page can't have magpie fetch anything else.
func Icon(u string) ([]byte, string, error) {
sum := sha256.Sum256([]byte(u))
file := marketCache(filepath.Join("icons", hex.EncodeToString(sum[:16])))
if b, err := os.ReadFile(file); err == nil {
if ct, err := os.ReadFile(file + ".type"); err == nil {
return b, string(ct), nil
}
}
if !offered(u) {
return nil, "", errors.New("not an icon of the market's")
}
b, ct, err := fetchImage(u)
if err != nil {
return nil, "", err
}
if file != "" {
_ = os.MkdirAll(filepath.Dir(file), 0o755)
if os.WriteFile(file, b, 0o644) == nil {
_ = os.WriteFile(file+".type", []byte(ct), 0o644)
}
}
return b, ct, nil
}
func offered(u string) bool {
if !strings.HasPrefix(u, "https://") {
return false
}
if slices.ContainsFunc(featured, func(m MarketServer) bool { return m.Icon == u }) {
return true
}
if rest, ok := strings.CutPrefix(u, "https://github.com/"); ok && strings.HasSuffix(rest, ".png?size=96") && !strings.Contains(strings.TrimSuffix(rest, ".png?size=96"), "/") {
return true // an owner's avatar
}
if l, err := loadLocked(); err == nil && slices.Contains(slices.Collect(maps.Values(l.Icons)), u) {
return true // one a server was added with
}
seenServers.Lock()
defer seenServers.Unlock()
for _, m := range seenServers.m {
if m.Icon == u {
return true
}
}
return false
}
// fetchImage downloads an icon through usemagpie.ai's icon proxy, so the
// host a server's author picked never sees the user's IP.
func fetchImage(u string) ([]byte, string, error) {
b, err := get(iconproxy.URL(u), "image/*", 1<<20)
if err != nil {
return nil, "", err
}
ct := http.DetectContentType(b)
if strings.Contains(string(b[:min(len(b), 512)]), "<svg") {
ct = "image/svg+xml"
}
if !strings.HasPrefix(ct, "image/") {
return nil, "", fmt.Errorf("%s isn't an image", u)
}
return b, ct, nil
}
// ---- fetching ---------------------------------------------------------------
var client = &http.Client{Timeout: 20 * time.Second}
// registryTimeout is the MCP Registry's own budget: its search answers in
// 13-37 s often enough (9 of 33 tries past 13 s, 6 past 20 s, the slowest
// 37.3 s - measured in #3043) that the market's 20 s client emptied the
// search with "context deadline exceeded". Its asks go with a longer one;
// skills.sh and the icons keep the short one.
var registryTimeout = 60 * time.Second
// registryClient is the client the registry is asked with, a variable for
// tests.
var registryClient = &http.Client{Timeout: registryTimeout}
// SearchError is a source the market searches that didn't answer; what
// the market has of its own is still listed beside it (#1589).
type SearchError struct {
Source string // skills.sh, the MCP Registry
Err error
}
func (e *SearchError) Error() string { return "couldn't search " + e.Source + ": " + e.Err.Error() }
func (e *SearchError) Unwrap() error { return e.Err }
// retryPause is how long a 503 is waited out before the one more try; a
// variable for tests.
var retryPause = time.Second
var metaDescRe = regexp.MustCompile(`<meta\s+name="description"\s+content="([^"]*)"`)
// metaDescription is what a page's <meta name="description"> says.
func metaDescription(page []byte) string {
m := metaDescRe.FindSubmatch(page)
if m == nil {
return ""
}
return strings.TrimSpace(html.UnescapeString(string(m[1])))
}
func get(u, accept string, limit int64) ([]byte, error) {
return getBy(client, u, accept, limit)
}
// getBy is get with the client: the registry's own budget (#3043).
func getBy(c *http.Client, u, accept string, limit int64) ([]byte, error) {
req, err := http.NewRequest("GET", u, nil)
if err != nil {
return nil, err
}
// skills.sh answers a client it doesn't know with a page of its own
req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36 magpie")
req.Header.Set("Accept", accept)
resp, err := c.Do(req)
if err != nil {
return nil, err
}
// a 503 is a server starting cold or busy for a moment (skills.sh's
// search answered one, and 200 right after, #1589): it's asked once more
if resp.StatusCode == http.StatusServiceUnavailable {
resp.Body.Close()
time.Sleep(retryPause)
if resp, err = c.Do(req); err != nil {
return nil, err
}
}
defer resp.Body.Close()
if resp.StatusCode != 200 {
return nil, fmt.Errorf("%s answered %s", resp.Request.URL.Host, resp.Status)
}
b, err := io.ReadAll(io.LimitReader(resp.Body, limit+1))
if err != nil {
return nil, err
}
if int64(len(b)) > limit {
return nil, fmt.Errorf("%s is too large", u)
}
return b, nil
}
func getJSON(u string, v any) error {
b, err := get(u, "application/json", 8<<20)
if err != nil {
return err
}
return json.Unmarshal(b, v)
}
// getJSONBy is getJSON with the client (the registry's own budget, #3043).
func getJSONBy(c *http.Client, u string, v any) error {
b, err := getBy(c, u, "application/json", 8<<20)
if err != nil {
return err
}
return json.Unmarshal(b, v)
}
// loadLocked reads the library, waiting for a change being made.
func loadLocked() (*Library, error) {
mu.Lock()
defer mu.Unlock()
return load()
}