Skip to content

Every agent's model. One place. Codex on DeepSeek, Claude Code on Kimi, from the menu bar.

13.0 KB
Raw Blame
package gateway

import (
	"bytes"
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"mime"
	"net/http"
	"net/url"
	"strings"
	"time"
	"unicode/utf8"

	"golang.org/x/net/html"
	"golang.org/x/net/html/charset"

	"github.com/yetone/magpie/internal/access"
	"github.com/yetone/magpie/internal/budget"
	"github.com/yetone/magpie/internal/netproxy"
)

// The web search MCP server's second tool, web_fetch (#2286, blackrion):
// an agent that found a page with web_search, and has no fetch tool of its
// own, reads it. The URL is the agent's, so the page is downloaded the way
// a caller's image is (publicClient): from public addresses only, through
// every redirect, so a gateway key's holder can't have magpie read a
// service only this computer, or the user's network, reaches. A page is
// read for at most webFetchTimeout and webFetchRead bytes; an HTML page is
// given as Markdown-like text, other text as it is, anything else refused.
// A long page comes in parts of max_chars characters, offset naming where
// the next starts.

const fetchMCPTool = "web_fetch"

const (
	webFetchTimeout  = 30 * time.Second
	webFetchRead     = 5 << 20
	webFetchChars    = 20000
	webFetchMaxChars = 100000
)

// webFetchClient downloads the page; a test swaps it.
var webFetchClient = publicClient

func fetchTool() map[string]any {
	return map[string]any{
		"name":        fetchMCPTool,
		"title":       "Web fetch",
		"description": "Fetch a web page by its http(s) URL and return its text (an HTML page as Markdown, with its links). A long page comes in parts: the answer says the offset to read on from.",
		"inputSchema": json.RawMessage(`{"type":"object","properties":{"url":{"type":"string","description":"The page's http:// or https:// URL"},"max_chars":{"type":"integer","description":"At most this many characters (default 20000, at most 100000)"},"offset":{"type":"integer","description":"The character to start from, to read on in a long page (default 0)"}},"required":["url"]}`),
		"annotations": map[string]any{"readOnlyHint": true, "openWorldHint": true},
	}
}

type fetchArgs struct {
	URL      string `json:"url"`
	MaxChars int    `json:"max_chars"`
	Offset   int    `json:"offset"`
}

// fetchMCPRun reads the page a web_fetch names, held to the calling gateway
// key's limit as a search is.
func (s *Server) fetchMCPRun(r *http.Request, a fetchArgs) (string, error) {
	raw := strings.TrimSpace(a.URL)
	if raw == "" {
		return "", searchMCPError("web_fetch needs a url")
	}
	u, err := url.Parse(raw)
	if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" {
		return "", searchMCPError("web_fetch reads only http:// and https:// URLs, not " + raw)
	}
	if a.Offset < 0 {
		return "", searchMCPError("offset can't be negative")
	}
	limit := a.MaxChars
	if limit <= 0 {
		limit = webFetchChars
	}
	limit = min(limit, webFetchMaxChars)
	release, refused := budget.Reserve(access.Caller(r.Context()), int64(len(raw)), "", limitClock())
	if refused != nil {
		return "", refused
	}
	defer release()
	ctx, cancel := context.WithTimeout(netproxy.With(r.Context(), ""), webFetchTimeout)
	defer cancel()
	title, text, final, err := fetchPage(ctx, u)
	if err != nil {
		return "", searchMCPError("couldn't fetch " + u.Redacted() + ": " + err.Error())
	}
	return fetchPart(final, title, text, a.Offset, limit), nil
}

// fetchPart is what web_fetch answers: where the page was read from, its
// title, and the part of its text from offset, at most limit characters,
// saying where the rest starts.
func fetchPart(final, title, text string, offset, limit int) string {
	var b strings.Builder
	b.WriteString("URL: " + final + "\n")
	if title != "" {
		b.WriteString("Title: " + title + "\n")
	}
	b.WriteString("\n")
	runes := []rune(text)
	if offset >= len(runes) {
		if len(runes) == 0 {
			b.WriteString("(the page has no text)")
		} else {
			fmt.Fprintf(&b, "(the page has %d characters, none from offset %d)", len(runes), offset)
		}
		return b.String()
	}
	end := min(offset+limit, len(runes))
	b.WriteString(string(runes[offset:end]))
	if end < len(runes) {
		fmt.Fprintf(&b, "\n\n[%d more characters: call web_fetch with offset=%d to read on]", len(runes)-end, end)
	}
	return b.String()
}

// fetchPage downloads u and gives its title, its text, and the URL it was
// read from after redirects.
func fetchPage(ctx context.Context, u *url.URL) (title, text, final string, err error) {
	req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil)
	if err != nil {
		return "", "", "", err
	}
	req.Header.Set("Accept", "text/html,application/xhtml+xml,text/plain;q=0.9,*/*;q=0.5")
	req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; web-fetch)")
	// the URL each redirect leads to, by name: the response's own request
	// is the one sent, to the address the name was pinned to
	last := req.URL
	c := *webFetchClient
	c.CheckRedirect = func(next *http.Request, via []*http.Request) error {
		if webFetchClient.CheckRedirect != nil {
			if err := webFetchClient.CheckRedirect(next, via); err != nil {
				return err
			}
		} else if len(via) >= 10 {
			return fmt.Errorf("it redirects more than 10 times")
		}
		last = next.URL
		return nil
	}
	resp, err := c.Do(req)
	if err != nil {
		if ctx.Err() == context.DeadlineExceeded {
			return "", "", "", fmt.Errorf("it took longer than %s", webFetchTimeout)
		}
		if errors.Is(err, errPrivateAddress) {
			return "", "", "", errors.New("it is at a private address (this computer, its network, or a name that resolves to one), which web_fetch doesn't read")
		}
		return "", "", "", err
	}
	defer resp.Body.Close()
	final = last.Redacted()
	body, err := io.ReadAll(io.LimitReader(resp.Body, webFetchRead))
	if err != nil {
		if ctx.Err() == context.DeadlineExceeded {
			return "", "", "", fmt.Errorf("it took longer than %s", webFetchTimeout)
		}
		return "", "", "", err
	}
	ct := resp.Header.Get("Content-Type")
	if ct == "" {
		ct = http.DetectContentType(body)
	}
	mt, _, _ := mime.ParseMediaType(ct)
	kind := fetchKind(mt)
	if resp.StatusCode >= 400 {
		msg := fmt.Sprintf("the site answered %s", resp.Status)
		if kind != "" {
			if t := fetchText(body, ct, kind, last)[1]; strings.TrimSpace(t) != "" {
				msg += ": " + fetchClip(t, 300)
			}
		}
		return "", "", "", fmt.Errorf("%s", msg)
	}
	if kind == "" {
		return "", "", "", fmt.Errorf("it is %s, not a page of text", mt)
	}
	got := fetchText(body, ct, kind, last)
	return got[0], got[1], final, nil
}

// fetchKind is "html" for a page, "text" for other text, "" for neither.
func fetchKind(mt string) string {
	switch {
	case mt == "text/html" || mt == "application/xhtml+xml":
		return "html"
	case strings.HasPrefix(mt, "text/"), mt == "application/json", mt == "application/xml",
		strings.HasSuffix(mt, "+json"), strings.HasSuffix(mt, "+xml"),
		mt == "application/javascript", mt == "application/x-ndjson":
		return "text"
	}
	return ""
}

// fetchText is a body's title and text, decoded from its charset.
func fetchText(body []byte, ct, kind string, base *url.URL) [2]string {
	r, err := charset.NewReader(strings.NewReader(string(body)), ct)
	if err != nil {
		r = strings.NewReader(string(body))
	}
	if kind == "text" {
		b, _ := io.ReadAll(r)
		return [2]string{"", strings.ToValidUTF8(string(b), "�")}
	}
	doc, err := html.Parse(r)
	if err != nil {
		return [2]string{"", ""}
	}
	return htmlText(doc, base)
}

func fetchClip(s string, n int) string {
	s = strings.Join(strings.Fields(s), " ")
	if utf8.RuneCountInString(s) <= n {
		return s
	}
	return string([]rune(s)[:n]) + "…"
}

// htmlText is an HTML page's title and its text as Markdown: headings,
// lists, links (made absolute), code and paragraphs kept; scripts, styles
// and other markup that isn't read dropped.
func htmlText(doc *html.Node, base *url.URL) [2]string {
	m := &mdWriter{base: base}
	m.walk(doc)
	return [2]string{strings.Join(strings.Fields(m.title), " "), m.done()}
}

type mdWriter struct {
	b     bytes.Buffer
	base  *url.URL
	title string
	pre   int
	space bool // a space is owed before the next word
	tight bool // just after a link's "[", a list's "- " or a heading's "#": no space before the first word
	lists []int
}

var mdSkip = map[string]bool{"script": true, "style": true, "noscript": true, "template": true, "svg": true, "iframe": true, "object": true, "canvas": true, "head": true, "select": true, "button": true, "input": true, "textarea": true}

var mdBlock = map[string]bool{"p": true, "div": true, "section": true, "article": true, "header": true, "footer": true, "main": true, "aside": true, "nav": true, "table": true, "tr": true, "blockquote": true, "form": true, "figure": true, "figcaption": true, "dl": true, "dt": true, "dd": true, "address": true, "details": true, "summary": true, "body": true, "ul": true, "ol": true}

// newline ends the current line, at most n blank-ish breaks in a row.
func (m *mdWriter) newline(n int) {
	m.tight = false
	s := m.b.Bytes()
	have := len(s) - len(bytes.TrimRight(s, "\n"))
	if len(s) == 0 {
		have = n
	}
	for ; have < n; have++ {
		m.b.WriteByte('\n')
	}
	m.space = false
}

func (m *mdWriter) atLineStart() bool {
	s := m.b.Bytes()
	return len(s) == 0 || s[len(s)-1] == '\n'

}

func (m *mdWriter) words(t string) {
	if m.pre > 0 {
		m.b.WriteString(t)
		return
	}
	lead := len(t) > 0 && strings.TrimLeft(t[:1], " \t\r\n\f") == ""
	trail := len(t) > 0 && strings.TrimRight(t[len(t)-1:], " \t\r\n\f") == ""
	f := strings.Fields(t)
	if len(f) == 0 {
		if lead {
			m.space = true
		}
		return
	}
	if (lead || m.space) && !m.atLineStart() && !m.tight {
		m.b.WriteByte(' ')
	}
	m.tight = false
	m.b.WriteString(strings.Join(f, " "))
	m.space = trail
}

func attr(n *html.Node, k string) string {
	for _, a := range n.Attr {
		if a.Key == k {
			return a.Val
		}
	}
	return ""
}

func hasAttr(n *html.Node, k string) bool {
	for _, a := range n.Attr {
		if a.Key == k {
			return true
		}
	}
	return false
}

func (m *mdWriter) kids(n *html.Node) {
	for c := n.FirstChild; c != nil; c = c.NextSibling {
		m.walk(c)
	}
}

func (m *mdWriter) walk(n *html.Node) {
	switch n.Type {
	case html.TextNode:
		m.words(n.Data)
		return
	case html.DocumentNode:
		m.kids(n)
		return
	case html.ElementNode:
	default:
		return
	}
	tag := n.Data
	if tag == "title" && m.title == "" {
		for c := n.FirstChild; c != nil; c = c.NextSibling {
			if c.Type == html.TextNode {
				m.title += c.Data
			}
		}
		return
	}
	if tag == "head" {
		// only its title is read
		for c := n.FirstChild; c != nil; c = c.NextSibling {
			if c.Type == html.ElementNode && c.Data == "title" {
				m.walk(c)
			}
		}
		return
	}
	if mdSkip[tag] || hasAttr(n, "hidden") || attr(n, "aria-hidden") == "true" {
		return
	}
	switch tag {
	case "br":
		m.b.WriteByte('\n')
		m.space = false
	case "hr":
		m.newline(2)
		m.b.WriteString("---")
		m.newline(2)
	case "h1", "h2", "h3", "h4", "h5", "h6":
		m.newline(2)
		m.b.WriteString(strings.Repeat("#", int(tag[1]-'0')) + " ")
		m.tight = true
		m.kids(n)
		m.newline(2)
	case "li":
		m.newline(1)
		depth := max(len(m.lists)-1, 0)
		m.b.WriteString(strings.Repeat("  ", depth))
		if len(m.lists) > 0 && m.lists[len(m.lists)-1] > 0 {
			fmt.Fprintf(&m.b, "%d. ", m.lists[len(m.lists)-1])
			m.lists[len(m.lists)-1]++
		} else {
			m.b.WriteString("- ")
		}
		m.tight = true
		m.kids(n)
		m.newline(1)
	case "ul", "ol":
		start := 0
		if tag == "ol" {
			start = 1
		}
		m.lists = append(m.lists, start)
		m.newline(1)
		m.kids(n)
		m.lists = m.lists[:len(m.lists)-1]
		m.newline(1)
	case "pre":
		m.newline(2)
		m.b.WriteString("```\n")
		m.pre++
		m.kids(n)
		m.pre--
		m.newline(1)
		m.b.WriteString("```")
		m.newline(2)
	case "code":
		if m.pre > 0 {
			m.kids(n)
			return
		}
		if m.space && !m.atLineStart() {
			m.b.WriteByte(' ')
		}
		m.space = false
		m.b.WriteByte('`')
		m.kids(n)
		m.b.WriteByte('`')
	case "td", "th":
		if !m.atLineStart() {
			m.b.WriteString(" | ")
		}
		m.space = false
		m.kids(n)
	case "a":
		href := strings.TrimSpace(attr(n, "href"))
		link := ""
		if href != "" && !strings.HasPrefix(href, "#") {
			if h, err := m.base.Parse(href); err == nil && (h.Scheme == "http" || h.Scheme == "https") {
				link = h.String()
			}
		}
		if link == "" || m.pre > 0 {
			m.kids(n)
			return
		}
		if m.space && !m.atLineStart() {
			m.b.WriteByte(' ')
		}
		m.space = false
		at := m.b.Len()
		m.b.WriteByte('[')
		m.tight = true
		m.kids(n)
		m.tight = false
		if m.b.Len() == at+1 {
			// a link with no text (an icon's): its URL alone says nothing
			m.b.Truncate(at)
			return
		}
		m.b.WriteString("](" + link + ")")
	case "img":
		if alt := strings.TrimSpace(attr(n, "alt")); alt != "" {
			m.words(" " + alt + " ")
		}
	default:
		if mdBlock[tag] {
			m.newline(2)
			if tag == "blockquote" {
				m.b.WriteString("> ")
			}
			m.kids(n)
			m.newline(2)
			return
		}
		m.kids(n)
	}
}

func (m *mdWriter) done() string {
	lines := strings.Split(m.b.String(), "\n")
	out := make([]string, 0, len(lines))
	blank := 0
	for _, l := range lines {
		l = strings.TrimRight(l, " \t")
		if t := strings.TrimSpace(l); t == "-" || t == ">" {
			// a list item or quote whose text was all dropped
			l = ""
		}
		if l == "" {
			blank++
			if blank > 1 {
				continue
			}
		} else {
			blank = 0
		}
		out = append(out, l)
	}
	return strings.TrimSpace(strings.Join(out, "\n"))
}