Skip to content

Every agent's model. One place. Codex on DeepSeek, Claude Code on Kimi, from the menu bar.

gateway: a search magpie runs for a model cites its pages in Anthropic text blocks (H20)

yetone1d6b72ada8c9
diff --git a/docs/subsystems/gateway-routing.md b/docs/subsystems/gateway-routing.md
index ae13b6434..9be80040d 100644
--- a/docs/subsystems/gateway-routing.md
+++ b/docs/subsystems/gateway-routing.md
@@ -382,6 +382,30 @@ the `TestResponsesSearch` tests in
and the Responses cases of `TestWebSearchForAModelThatCannot` in
[`search_test.go`](../../internal/gateway/search_test.go).
+## Citations in translated Anthropic replies
+
+After a `KSearch` event (or a `Search` part), the Anthropic encoder and
+`renderAnthropic` cut the text that follows into lines (`citer` in
+[`search_cite.go`](../../internal/gateway/search_cite.go)). magpie's
+`web_search` tool asks the model to cite the URLs it uses, so a line that
+names the URL of a page the reply's searches found, whole (`urlIn`: not a
+longer path, a query or another host), is a text block of its own whose
+`citations` are `web_search_result_location` entries for those pages
+(`citesOf`), as Anthropic's own search cites them; the lines between run
+together in one block without citations, and a whitespace-only line is
+held for the next, so no block is whitespace alone. Streamed, each cited
+block's `citations_delta` events come before its `text_delta`, and the
+stream buffers only to the end of a line, only after a search. The text
+itself is the model's, unchanged. `encrypted_index` is `""`, like the
+results' `encrypted_content`: it is Anthropic's sealed pointer, which
+magpie can't make. `cited_text` is what the search said of the page,
+cut to 150 characters (`Hit.Cited`: a search API's snippet, or the text an
+Anthropic searcher's own citation quoted), else `""`. A reply with no
+search is said as before. Covered by `TestSearchCitedAsAnthropics`
+(streamed and whole), `TestNoSearchNoCitations` and
+`TestCitesOnlyTheWholeURL`. Responses and Chat replies carry no
+`url_citation` annotations yet.
+
While a provider asked by `searchReply` is queued or thinking, sending only
keepalives or `response.in_progress`, `askTranslated` reads it with
`readSSEAlive` and hands on a `KAlive` event after `keepaliveGap` with
diff --git a/internal/gateway/anthropic.go b/internal/gateway/anthropic.go
index 1135c9b0d..a5d7e17cb 100644
--- a/internal/gateway/anthropic.go
+++ b/internal/gateway/anthropic.go
@@ -1371,6 +1371,11 @@ type anthropicEncoder struct {
fresh bool
started bool
col collector
+ // cite cuts the text said after a search into lines, each naming a
+ // page found its own block with citations (search_cite.go); cited:
+ // the open text block is such a line's
+ cite citer
+ cited bool
}
// anthropicID is a reply's id as Anthropic's API gives one, msg_…: an
@@ -1487,17 +1492,52 @@ func (e *anthropicEncoder) delta(d map[string]any) {
e.w.event("content_block_delta", map[string]any{"type": "content_block_delta", "index": e.index, "delta": d})
}
+// sayCited says a line of text after a search (citer): a line that cites
+// a page is a text block of its own, its citations first, as Anthropic's
+// stream gives them (citations_delta); one that cites none goes on in the
+// open text block unless that block cites; the whitespace left at the end
+// (tail) goes on whatever text block is open.
+func (e *anthropicEncoder) sayCited(text string, cites []map[string]any, tail bool) {
+ switch {
+ case len(cites) > 0:
+ e.close()
+ e.openBlock(Text, map[string]any{"text": ""})
+ e.cited = true
+ for _, c := range cites {
+ e.delta(map[string]any{"type": "citations_delta", "citation": c})
+ }
+ case tail && e.open == Text:
+ default:
+ if e.cited {
+ e.close()
+ }
+ e.openBlock(Text, map[string]any{"text": ""})
+ e.cited = false
+ }
+ e.delta(map[string]any{"type": "text_delta", "text": text})
+}
+
func (e *anthropicEncoder) event(ev Event) {
if ev.Kind != KStart && !e.started {
e.start(Event{})
}
switch ev.Kind {
+ case KText, KUsage, KStart, KTextSig, KStop, KAlive:
+ default:
+ // anything else ends the text: its last line is said
+ e.cite.flush(e.sayCited)
+ }
+ switch ev.Kind {
case KStart:
e.start(ev)
case KText:
if ev.Text == "" {
return
}
+ if e.cite.on() {
+ e.cite.feed(ev.Text, e.sayCited)
+ break
+ }
e.openBlock(Text, map[string]any{"text": ""})
e.delta(map[string]any{"type": "text_delta", "text": ev.Text})
case KThinkStart:
@@ -1550,6 +1590,7 @@ func (e *anthropicEncoder) event(ev Event) {
e.index++
e.w.event("content_block_start", map[string]any{"type": "content_block_start", "index": e.index,
"content_block": searchResultBlock(id, ev.Hits)})
+ e.cite.hits = append(e.cite.hits, ev.Hits...)
e.w.event("content_block_stop", map[string]any{"type": "content_block_stop", "index": e.index})
e.index++
case KError:
@@ -1579,6 +1620,7 @@ func (e *anthropicEncoder) finish() {
if !e.started {
e.start(Event{})
}
+ e.cite.flush(e.sayCited)
e.close()
res := e.col.finish()
delta := map[string]any{"stop_reason": stopToAnthropic(res.Stop), "stop_sequence": nil}
@@ -1597,12 +1639,17 @@ func (e *anthropicEncoder) finish() {
// renderAnthropic is the non-streaming reply.
func renderAnthropic(res Result, model string) []byte {
content := []any{}
+ var cite citer
for _, p := range res.Parts {
switch p.Kind {
case Text:
if p.Text == "" && p.Signature != "" {
continue // only Gemini's signature, which is Gemini's
}
+ if cite.on() {
+ content = append(content, citedBlocks(&cite, p.Text)...)
+ continue
+ }
content = append(content, map[string]any{"type": "text", "text": p.Text})
case Thinking:
switch {
@@ -1624,6 +1671,7 @@ func renderAnthropic(res Result, model string) []byte {
content = append(content,
map[string]any{"type": "server_tool_use", "id": id, "name": "web_search", "input": map[string]any{"query": p.Text}},
searchResultBlock(id, p.Hits))
+ cite.hits = append(cite.hits, p.Hits...)
}
}
id := anthropicID(res.ID)
@@ -1642,6 +1690,32 @@ func renderAnthropic(res Result, model string) []byte {
return b
}
+// citedBlocks are the text blocks of text said after a search, as the
+// stream's sayCited makes them.
+func citedBlocks(c *citer, text string) []any {
+ var blocks []any
+ var open map[string]any // the last text block, while it may go on
+ cited := false
+ say := func(t string, cites []map[string]any, tail bool) {
+ switch {
+ case len(cites) > 0:
+ open = map[string]any{"type": "text", "text": t, "citations": cites}
+ blocks = append(blocks, open)
+ cited = true
+ return
+ case tail && open != nil, open != nil && !cited:
+ open["text"] = open["text"].(string) + t
+ return
+ }
+ open = map[string]any{"type": "text", "text": t}
+ blocks = append(blocks, open)
+ cited = false
+ }
+ c.feed(text, say)
+ c.flush(say)
+ return blocks
+}
+
// searchResultBlock is a web_search_tool_result block of the pages a
// search found.
func searchResultBlock(id string, hits []Hit) map[string]any {
diff --git a/internal/gateway/ir.go b/internal/gateway/ir.go
index 2426c2ed1..0cc785894 100644
--- a/internal/gateway/ir.go
+++ b/internal/gateway/ir.go
@@ -92,6 +92,10 @@ type Hit struct {
URL string `json:"url"`
// PageAge is how old the search said the page is, "" when it didn't
PageAge string `json:"page_age,omitempty"`
+ // Cited is what the search said of the page (a search API's snippet,
+ // the text a searcher's own citation quotes), which a citation of it
+ // quotes (citesOf); "" when it said nothing of it alone
+ Cited string `json:"cited_text,omitempty"`
}
// attachmentText is the fallback when a protocol cannot carry a Gemini file.
diff --git a/internal/gateway/search.go b/internal/gateway/search.go
index 744823455..8eeae5458 100644
--- a/internal/gateway/search.go
+++ b/internal/gateway/search.go
@@ -926,9 +926,13 @@ func (s *Server) searchWith(ctx context.Context, p provider.Provider, model, que
s.serve(w, r, provider.Anthropic, body)
var out struct {
Content []struct {
- Type string `json:"type"`
- Text string `json:"text"`
- Content json.RawMessage `json:"content"`
+ Type string `json:"type"`
+ Text string `json:"text"`
+ Content json.RawMessage `json:"content"`
+ Citations []struct {
+ URL string `json:"url"`
+ CitedText string `json:"cited_text"`
+ } `json:"citations"`
} `json:"content"`
Error *struct {
Message string `json:"message"`
@@ -947,11 +951,17 @@ func (s *Server) searchWith(ctx context.Context, p provider.Provider, model, que
var text strings.Builder
var sources []string
var found []Hit
+ quoted := map[string]string{} // what the searcher's citations quote of each page
searched := false
for _, b := range out.Content {
switch b.Type {
case "text":
text.WriteString(b.Text)
+ for _, c := range b.Citations {
+ if c.URL != "" && c.CitedText != "" && quoted[c.URL] == "" {
+ quoted[c.URL] = c.CitedText
+ }
+ }
case "server_tool_use":
searched = true
case "web_search_tool_result":
@@ -976,6 +986,9 @@ func (s *Server) searchWith(ctx context.Context, p provider.Provider, model, que
}
}
}
+ for i := range found {
+ found[i].Cited = quoted[found[i].URL]
+ }
said := strings.TrimSpace(text.String())
if said == "" || (len(found) == 0 && !strings.Contains(said, "://")) {
if !searched {
diff --git a/internal/gateway/search_api.go b/internal/gateway/search_api.go
index b58cfcd0d..1425681c9 100644
--- a/internal/gateway/search_api.go
+++ b/internal/gateway/search_api.go
@@ -195,7 +195,7 @@ func pagesSaid(pages []foundPage) (string, []Hit) {
b.WriteString(p.Text + "\n")
}
b.WriteString("\n")
- hits = append(hits, Hit{Title: p.Title, URL: p.URL, PageAge: p.Age})
+ hits = append(hits, Hit{Title: p.Title, URL: p.URL, PageAge: p.Age, Cited: p.Text})
}
return strings.TrimSpace(b.String()), hits
}
diff --git a/internal/gateway/search_cite.go b/internal/gateway/search_cite.go
new file mode 100644
index 000000000..3aa9406b5
--- /dev/null
+++ b/internal/gateway/search_cite.go
@@ -0,0 +1,133 @@
+package gateway
+
+import "strings"
+
+// After a search magpie ran for a model (searchReply), or one a provider
+// ran and reported (KSearch), the model's answer names the pages it used
+// by their addresses: magpie's web_search tool asks it to cite the URLs.
+// An Anthropic client is told which pages a line cites as Anthropic's own
+// search tells it, a text block whose citations are
+// web_search_result_location entries for the pages of the reply's
+// web_search_tool_result blocks. A line naming a page found is its own
+// text block with those citations; the lines between them run together in
+// one block as before. The text is the model's, unchanged.
+//
+// encrypted_index is Anthropic's sealed pointer into its own result, which
+// magpie can't make: it is "", as the results' encrypted_content is
+// (searchResultBlock). cited_text is what the search said of the page, when
+// it said something (Hit.Cited), else "".
+
+// citedTextMost is how much of a page's text a citation quotes, as
+// Anthropic's does (up to 150 characters).
+const citedTextMost = 150
+
+// citer cuts a reply's text into lines once a search has found pages, and
+// says which of the pages each line cites. A line that is only whitespace
+// is held for the next, so no block is whitespace alone (Anthropic turns
+// such a block away when the client sends it back).
+type citer struct {
+ hits []Hit
+ line string // the line being said, not yet ended
+ held string // whitespace lines held for the next line
+}
+
+// on is whether there are pages a line may cite.
+func (c *citer) on() bool { return len(c.hits) > 0 }
+
+// feed takes text and says each line it ends: its text and citations.
+func (c *citer) feed(text string, say func(text string, cites []map[string]any, tail bool)) {
+ c.line += text
+ for {
+ i := strings.IndexByte(c.line, '\n')
+ if i < 0 {
+ return
+ }
+ line := c.line[:i+1]
+ c.line = c.line[i+1:]
+ c.end(line, say)
+ }
+}
+
+func (c *citer) end(line string, say func(string, []map[string]any, bool)) {
+ if strings.TrimSpace(line) == "" {
+ c.held += line
+ return
+ }
+ line, c.held = c.held+line, ""
+ say(line, citesOf(line, c.hits), false)
+}
+
+// flush says what is left: a line not ended, and the whitespace held,
+// which goes on the block said last (tail).
+func (c *citer) flush(say func(string, []map[string]any, bool)) {
+ line := c.line
+ c.line = ""
+ if strings.TrimSpace(line) != "" {
+ c.end(line, say)
+ return
+ }
+ if rest := c.held + line; rest != "" {
+ c.held = ""
+ say(rest, nil, true)
+ }
+}
+
+// citesOf are the citations of the pages line names, in the order it
+// names them, each once.
+func citesOf(line string, hits []Hit) []map[string]any {
+ type at struct {
+ i int
+ h Hit
+ }
+ var found []at
+ seen := map[string]bool{}
+ for _, h := range hits {
+ u := strings.TrimSuffix(h.URL, "/")
+ if u == "" || seen[u] {
+ continue
+ }
+ if i := urlIn(line, u); i >= 0 {
+ seen[u] = true
+ found = append(found, at{i, h})
+ }
+ }
+ if len(found) == 0 {
+ return nil
+ }
+ // in the order the line names them
+ for i := 1; i < len(found); i++ {
+ for j := i; j > 0 && found[j].i < found[j-1].i; j-- {
+ found[j], found[j-1] = found[j-1], found[j]
+ }
+ }
+ cites := make([]map[string]any, 0, len(found))
+ for _, f := range found {
+ cites = append(cites, map[string]any{"type": "web_search_result_location", "url": f.h.URL, "title": f.h.Title,
+ "encrypted_index": "", "cited_text": firstRunes(strings.TrimSpace(f.h.Cited), citedTextMost)})
+ }
+ return cites
+}
+
+// urlIn is where line names the address u (its trailing slash cut), the
+// whole of it: what follows, past a slash and the punctuation that ends a
+// sentence or markdown's emphasis, is the end, a space, a bracket, a quote
+// or a character outside ASCII (Chinese punctuation). -1 when it doesn't.
+func urlIn(line, u string) int {
+ for from := 0; ; {
+ i := strings.Index(line[from:], u)
+ if i < 0 {
+ return -1
+ }
+ i += from
+ rest := line[i+len(u):]
+ if urlEnds(strings.TrimPrefix(rest, "/")) {
+ return i
+ }
+ from = i + 1
+ }
+}
+
+func urlEnds(rest string) bool {
+ rest = strings.TrimLeft(rest, ".,;:!?*_")
+ return rest == "" || rest[0] >= 0x80 || strings.IndexByte(" \t\r\n)]}>\"'`|", rest[0]) >= 0
+}
diff --git a/internal/gateway/search_cite_test.go b/internal/gateway/search_cite_test.go
new file mode 100644
index 000000000..43266ab3c
--- /dev/null
+++ b/internal/gateway/search_cite_test.go
@@ -0,0 +1,226 @@
+package gateway
+
+import (
+ "context"
+ "encoding/json"
+ "io"
+ "net/http"
+ "net/http/httptest"
+ "reflect"
+ "strings"
+ "sync"
+ "testing"
+
+ "github.com/yetone/magpie/internal/provider"
+)
+
+// citedBlock is a text block of an Anthropic reply, with its citations.
+type citedBlock struct {
+ Text string
+ Citations []map[string]any
+}
+
+// textBlocksOf are an Anthropic reply's text blocks, from its stream or
+// its body, and the types of all its blocks in order.
+func textBlocksOf(t *testing.T, body string, stream bool) ([]citedBlock, []string) {
+ t.Helper()
+ var blocks []citedBlock
+ var types []string
+ if !stream {
+ var out struct {
+ Content []struct {
+ Type string `json:"type"`
+ Text string `json:"text"`
+ Citations []map[string]any `json:"citations"`
+ } `json:"content"`
+ }
+ if err := json.Unmarshal([]byte(body), &out); err != nil {
+ t.Fatalf("%v: %s", err, body)
+ }
+ for _, c := range out.Content {
+ types = append(types, c.Type)
+ if c.Type == "text" {
+ blocks = append(blocks, citedBlock{c.Text, c.Citations})
+ }
+ }
+ return blocks, types
+ }
+ at := map[float64]int{} // a stream's block index, to its text block
+ for _, ev := range events(body) {
+ switch ev["type"] {
+ case "content_block_start":
+ b := ev["content_block"].(map[string]any)
+ types = append(types, b["type"].(string))
+ if b["type"] == "text" {
+ at[ev["index"].(float64)] = len(blocks)
+ blocks = append(blocks, citedBlock{})
+ }
+ case "content_block_delta":
+ i, ok := at[ev["index"].(float64)]
+ if !ok {
+ continue
+ }
+ d := ev["delta"].(map[string]any)
+ switch d["type"] {
+ case "text_delta":
+ blocks[i].Text += d["text"].(string)
+ case "citations_delta":
+ // a citation comes before the text it is for, as
+ // Anthropic's stream sends it
+ if blocks[i].Text != "" {
+ t.Fatalf("citation after the block's text: %s", body)
+ }
+ blocks[i].Citations = append(blocks[i].Citations, d["citation"].(map[string]any))
+ }
+ }
+ }
+ return blocks, types
+}
+
+// The pages a search magpie ran found are cited in the answer's text as
+// Anthropic's own web search cites them (H20 on Discord: text blocks came
+// back with no citations): each line naming a page's URL is a text block
+// whose citations are web_search_result_location entries, the lines
+// between them a block with none, streamed or not.
+func TestSearchCitedAsAnthropics(t *testing.T) {
+ var mu sync.Mutex
+ rounds := 0
+ search := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ switch r.URL.Path {
+ case "/v1/models":
+ io.WriteString(w, `{"data":[{"id":"claude-haiku-4-5"}]}`)
+ case "/v1/messages":
+ // Anthropic's reply, its text citing a page as it does
+ w.Header().Set("Content-Type", "application/json")
+ io.WriteString(w, `{"id":"s","type":"message","role":"assistant","model":"claude-haiku-4-5","content":[
+ {"type":"server_tool_use","id":"srv","name":"web_search","input":{"query":"go"}},
+ {"type":"web_search_tool_result","tool_use_id":"srv","content":[
+ {"type":"web_search_result","title":"Go downloads","url":"https://go.dev/dl/","encrypted_content":"EqoBCkYIBxgC","page_age":"1 day ago"},
+ {"type":"web_search_result","title":"Release History","url":"https://go.dev/doc/devel/release","encrypted_content":"EqoBCkYIBxgD"}]},
+ {"type":"text","text":"The latest Go is 1.27.1.","citations":[{"type":"web_search_result_location","url":"https://go.dev/dl/","title":"Go downloads","encrypted_index":"Eo8BCioIBxgC","cited_text":"go1.27.1 (released 2026-10-01) includes security fixes."}]}],
+ "stop_reason":"end_turn","usage":{"input_tokens":5,"output_tokens":5}}`)
+ default:
+ http.NotFound(w, r)
+ }
+ }))
+ defer search.Close()
+ model := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ io.ReadAll(r.Body)
+ mu.Lock()
+ rounds++
+ n := rounds
+ mu.Unlock()
+ w.Header().Set("Content-Type", "text/event-stream")
+ if n%2 == 1 {
+ io.WriteString(w, sse(`data: {"id":"c","choices":[{"index":0,"delta":{"role":"assistant","content":"Let me check."}}]}`,
+ `data: {"id":"c","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"id":"call_1","type":"function","function":{"name":"web_search","arguments":"{\"query\":\"latest go\"}"}}]}}]}`,
+ `data: {"id":"c","choices":[{"index":0,"delta":{},"finish_reason":"tool_calls"}],"usage":{"prompt_tokens":30,"completion_tokens":7}}`,
+ `data: [DONE]`))
+ return
+ }
+ // the answer, an address cut across chunks, one line naming a
+ // page the search didn't find
+ io.WriteString(w, sse(`data: {"id":"d","choices":[{"index":0,"delta":{"role":"assistant","content":"Go 1.27.1 is out ([Go downloads](https://go.d"}}]}`,
+ `data: {"id":"d","choices":[{"index":0,"delta":{"content":"ev/dl/)).\nIt fixes bugs; see https://example.com/notes.\n\n- History: https://go.dev/doc/devel/release"}}]}`,
+ `data: {"id":"d","choices":[{"index":0,"delta":{"content":"。"}}]}`,
+ `data: {"id":"d","choices":[{"index":0,"delta":{},"finish_reason":"stop"}],"usage":{"prompt_tokens":10,"completion_tokens":5}}`,
+ `data: [DONE]`))
+ }))
+ defer model.Close()
+
+ t.Setenv("XDG_CONFIG_HOME", t.TempDir())
+ t.Setenv("XDG_CACHE_HOME", t.TempDir())
+ setHome(t, t.TempDir())
+ hosts := searchHosts[provider.Anthropic]
+ searchHosts[provider.Anthropic] = append(hosts, provider.HostOf(search.URL))
+ defer func() { searchHosts[provider.Anthropic] = hosts }()
+ for _, p := range []provider.Provider{
+ {ID: "srch", Name: "Search", Key: "k", Anthropic: search.URL},
+ {ID: "deep", Name: "Deep", Key: "k", Chat: model.URL, Models: []string{"deepseek-v4.1-flash"}},
+ } {
+ if err := provider.Save(p); err != nil {
+ t.Fatal(err)
+ }
+ }
+ p, _ := provider.Find("srch")
+ if _, err := p.Fetch(context.Background()); err != nil {
+ t.Fatal(err)
+ }
+
+ dl := map[string]any{"type": "web_search_result_location", "url": "https://go.dev/dl/", "title": "Go downloads",
+ "encrypted_index": "", "cited_text": "go1.27.1 (released 2026-10-01) includes security fixes."}
+ release := map[string]any{"type": "web_search_result_location", "url": "https://go.dev/doc/devel/release", "title": "Release History",
+ "encrypted_index": "", "cited_text": ""}
+ want := []citedBlock{
+ {Text: "Let me check."},
+ {Text: "\n\nGo 1.27.1 is out ([Go downloads](https://go.dev/dl/)).\n", Citations: []map[string]any{dl}},
+ {Text: "It fixes bugs; see https://example.com/notes.\n"},
+ {Text: "\n- History: https://go.dev/doc/devel/release。", Citations: []map[string]any{release}},
+ }
+ for _, stream := range []bool{true, false} {
+ t.Run(map[bool]string{true: "stream", false: "body"}[stream], func(t *testing.T) {
+ body := `{"model":"deep/deepseek-v4.1-flash","max_tokens":1000,"stream":` + map[bool]string{true: "true", false: "false"}[stream] + `,
+ "messages":[{"role":"user","content":"What is the latest Go?"}],
+ "tools":[{"type":"web_search_20250305","name":"web_search","max_uses":8}]}`
+ rec := httptest.NewRecorder()
+ New().Handler().ServeHTTP(rec, httptest.NewRequest(http.MethodPost, "/v1/messages", strings.NewReader(body)))
+ if rec.Code != 200 {
+ t.Fatalf("%d %s", rec.Code, rec.Body.String())
+ }
+ blocks, types := textBlocksOf(t, rec.Body.String(), stream)
+ if !reflect.DeepEqual(types, []string{"text", "server_tool_use", "web_search_tool_result", "text", "text", "text"}) {
+ t.Fatalf("blocks %v\n%s", types, rec.Body.String())
+ }
+ if !reflect.DeepEqual(blocks, want) {
+ t.Fatalf("text blocks\n got %#v\nwant %#v\n%s", blocks, want, rec.Body.String())
+ }
+ })
+ }
+}
+
+// A reply with no search in it is said as before: one text block, no
+// citations, though it names an address.
+func TestNoSearchNoCitations(t *testing.T) {
+ text := "See https://go.dev/dl/.\nAnd more.\n"
+ var out struct {
+ Content []map[string]any `json:"content"`
+ }
+ json.Unmarshal(renderAnthropic(Result{Parts: []Part{{Kind: Text, Text: text}}}, "m"), &out)
+ if !reflect.DeepEqual(out.Content, []map[string]any{{"type": "text", "text": text}}) {
+ t.Fatalf("%v", out.Content)
+ }
+ rec := httptest.NewRecorder()
+ e := &anthropicEncoder{w: newSSEWriter(rec), model: "m"}
+ e.event(Event{Kind: KText, Text: "See https://go.dev/dl/."})
+ e.event(Event{Kind: KText, Text: "\nAnd more.\n"})
+ e.finish()
+ blocks, _ := textBlocksOf(t, rec.Body.String(), true)
+ if !reflect.DeepEqual(blocks, []citedBlock{{Text: text}}) || strings.Contains(rec.Body.String(), "citation") {
+ t.Fatalf("%#v\n%s", blocks, rec.Body.String())
+ }
+}
+
+// A URL is cited only whole: a longer path, a query or another host is
+// not the page found.
+func TestCitesOnlyTheWholeURL(t *testing.T) {
+ hits := []Hit{{Title: "A", URL: "https://x.dev/a/"}, {Title: "B", URL: "https://x.dev/b"}}
+ for line, want := range map[string][]string{
+ "https://x.dev/a": {"https://x.dev/a/"},
+ "(https://x.dev/a/), https://x.dev/b.": {"https://x.dev/a/", "https://x.dev/b"},
+ "**https://x.dev/b**": {"https://x.dev/b"},
+ "https://x.dev/b then https://x.dev/a": {"https://x.dev/b", "https://x.dev/a/"},
+ "https://x.dev/a.html": nil,
+ "https://x.dev/bb": nil,
+ "https://x.dev/b?q=1": nil,
+ "https://x.dev/b_c": nil,
+ "no address": nil,
+ } {
+ var got []string
+ for _, c := range citesOf(line, hits) {
+ got = append(got, c["url"].(string))
+ }
+ if !reflect.DeepEqual(got, want) {
+ t.Errorf("%q cites %v, want %v", line, got, want)
+ }
+ }
+}