Skip to content

Every agent's model. One place. Codex on DeepSeek, Claude Code on Kimi, from the menu bar.

12.0 KB
Raw Blame
package gateway

import (
	"encoding/json"
	"io"
	"net/http"
	"net/http/httptest"
	"strings"
	"testing"
	"time"

	"github.com/yetone/magpie/internal/provider"
)

// The message is what #2400's agent showed, word for word; no vendor's
// reply was captured, so the envelopes around it below are each protocol's
// error shapes, not a vendor's bytes.
const highRiskMessage = "The request was rejected because it was considered high risk"

func TestHighRiskErrorMessage(t *testing.T) {
	for _, msg := range []string{highRiskMessage, " \n" + highRiskMessage + "\n"} {
		b, _ := json.Marshal(map[string]any{"error": map[string]any{"message": msg}})
		if note, ok := policyRefusal(b); !ok || !strings.Contains(note, highRiskMessage) {
			t.Errorf("not classified as a refusal: %s: %q", b, note)
		}
	}
	for _, msg := range []string{
		"high risk", "not " + highRiskMessage, highRiskMessage + "; please explain this message",
		`The server quoted "` + highRiskMessage + `"`, "invalid API key",
	} {
		b, _ := json.Marshal(map[string]any{"error": map[string]any{"message": msg}})
		if note, ok := policyRefusal(b); ok {
			t.Errorf("unrelated error classified as a refusal: %s: %q", b, note)
		}
	}
	for _, event := range []string{
		`data: {"error":{"message":"` + highRiskMessage + `"}}`,
		`event: error` + "\n" + `data: {"message":"` + highRiskMessage + `"}`,
		`data: {"type":"error","error":{"type":"invalid_request_error","message":"` + highRiskMessage + `"}}`,
		`data: {"type":"response.failed","response":{"error":{"message":"` + highRiskMessage + `"}}}`,
	} {
		if kind, status, msg := streamEvent([]byte(event)); kind == eventError || status != refusedStatus || !strings.Contains(msg, highRiskMessage) {
			t.Errorf("error event was not a refusal: %s: kind %d, status %d, message %q", event, kind, status, msg)
		}
	}
}

func TestHighRiskErrorFallsBack(t *testing.T) {
	for _, from := range []provider.Protocol{provider.Chat, provider.Responses, provider.Anthropic} {
		for _, wire := range []struct {
			name string
			code int
			body string
		}{
			{"http error", 400, `{"error":{"message":"` + highRiskMessage + `"}}`},
			{"stream error", 200, sse(`data: {"error":{"message":"` + highRiskMessage + `"}}`)},
		} {
			t.Run(string(from)+"/"+wire.name, func(t *testing.T) {
				fresh(t)
				ctype := "application/json"
				if wire.code == 200 {
					ctype = "text/event-stream"
				}
				a := &scripted{replies: []reply{{wire.code, ctype, wire.body}}}
				b := &scripted{replies: []reply{{200, "text/event-stream", anthropicAnswer}}}
				scriptedOn(t, "a", provider.Chat, a)
				scriptedOn(t, "b", provider.Anthropic, b)
				refusalGroup(t, "a/m", "b/m")
				path, ask := refusalErrorAsk(from)
				s := New()
				code, body := sendRefusalError(t, s, path, ask)
				if code != 200 || !strings.Contains(body, "from b") || strings.Contains(body, highRiskMessage) || a.n != 1 || b.n != 1 {
					t.Fatalf("status %d: %s (a %d, b %d)", code, body, a.n, b.n)
				}
				r := lastRoute(s)
				if len(r.Tries) != 2 || r.Tries[0].Fail != failRefused || r.Tries[0].Rest != nil || !strings.Contains(r.Tries[0].Error, highRiskMessage) {
					t.Fatalf("refusal was not recorded, or rested the account: %+v", r.Tries)
				}
			})
		}
	}
}

func refusalErrorAsk(from provider.Protocol) (string, string) {
	switch from {
	case provider.Responses:
		return "/v1/responses", codexAsk
	case provider.Anthropic:
		return "/v1/messages", `{"model":"group/g","max_tokens":10,"stream":true,"messages":[{"role":"user","content":"hi"}]}`
	default:
		return "/v1/chat/completions", `{"model":"group/g","stream":true,"messages":[{"role":"user","content":"hi"}]}`
	}
}

func sendRefusalError(t *testing.T, s *Server, path, body string) (int, string) {
	t.Helper()
	gw := httptest.NewServer(lanGuard(s.Handler()))
	defer gw.Close()
	client := gw.Client()
	client.Timeout = 10 * time.Second // deadlock guard for the held-open upstream
	res, err := client.Post(gw.URL+path, "application/json", strings.NewReader(body))
	if err != nil {
		t.Fatalf("request did not finish without upstream EOF: %v", err)
	}
	defer res.Body.Close()
	b, err := io.ReadAll(res.Body)
	if err != nil {
		t.Fatal(err)
	}
	return res.StatusCode, string(b)
}

// A provider's error ends this try even if its HTTP connection stays open.
// The server only ends when canceled or released by cleanup: the deadline
// is a deadlock guard, not an assertion about how fast a loaded runner is.
func TestRefusalErrorDoesNotWaitForEOF(t *testing.T) {
	for _, from := range []provider.Protocol{provider.Chat, provider.Responses, provider.Anthropic} {
		t.Run(string(from), func(t *testing.T) {
			fresh(t)
			release, canceled := make(chan struct{}), make(chan struct{})
			up := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
				io.ReadAll(r.Body)
				w.Header().Set("Content-Type", "text/event-stream")
				io.WriteString(w, sse(`data: {"choices":[],"usage":{"prompt_tokens":3,"completion_tokens":0}}`,
					`data: {"error":{"code":"content_filter","message":"The response was filtered"}}`))
				w.(http.Flusher).Flush()
				select {
				case <-r.Context().Done():
					close(canceled)
				case <-release:
				}
			}))
			defer up.Close()
			defer close(release)
			if err := provider.Save(provider.Provider{ID: "a", Name: "A", Key: "k", Models: []string{"m"}, Chat: up.URL + "/v1"}); err != nil {
				t.Fatal(err)
			}
			b := &scripted{replies: []reply{{200, "text/event-stream", anthropicAnswer}}}
			scriptedOn(t, "b", provider.Anthropic, b)
			refusalGroup(t, "a/m", "b/m")
			path, ask := refusalErrorAsk(from)
			s := New()
			code, body := sendRefusalError(t, s, path, ask)
			select {
			case <-canceled:
			case <-time.After(10 * time.Second):
				t.Fatal("a held refusal error waited for upstream EOF instead of canceling the try")
			}
			if code != 200 || !strings.Contains(body, "from b") || b.n != 1 {
				t.Fatalf("status %d: %s (b %d)", code, body, b.n)
			}
			if r := lastRoute(s); len(r.Tries) != 2 || r.Tries[0].Fail != failRefused || r.Tries[0].Rest != nil || r.Tries[0].Status != 400 || !strings.Contains(r.Tries[0].Error, "The response was filtered") || strings.Contains(r.Tries[0].Error, "context canceled") {
				t.Fatalf("cancellation changed refusal accounting: %+v", r.Tries)
			}
			if r := lastRoute(s); len(r.Usage) != 2 || r.Usage[0].Input != 3 {
				t.Fatalf("usage received before the error was lost: %+v", r.Usage)
			}
		})
	}
}

func TestHighRiskQuotationIsAnAnswer(t *testing.T) {
	for _, body := range []string{
		`{"id":"c1","choices":[{"message":{"role":"assistant","content":"` + highRiskMessage + `"},"finish_reason":"stop"}]}`,
		sse(`data: {"id":"c1","choices":[{"index":0,"delta":{"content":"`+highRiskMessage+`"}}]}`,
			`data: {"id":"c1","choices":[{"index":0,"delta":{},"finish_reason":"stop"}]}`, `data: [DONE]`),
	} {
		fresh(t)
		ctype := "application/json"
		if strings.HasPrefix(body, "data:") {
			ctype = "text/event-stream"
		}
		a := &scripted{replies: []reply{{200, ctype, body}}}
		b := &scripted{replies: []reply{{200, "", chatOK}}}
		scriptedOn(t, "a", provider.Chat, a)
		scriptedOn(t, "b", provider.Chat, b)
		refusalGroup(t, "a/m", "b/m")
		s := New()
		path, ask := refusalErrorAsk(provider.Chat)
		code, got := sendRefusalError(t, s, path, ask)
		if code != 200 || strings.TrimSpace(got) != strings.TrimSpace(body) || a.n != 1 || b.n != 0 {
			t.Fatalf("quoted message was treated as a refusal: %d %s (a %d, b %d)", code, got, a.n, b.n)
		}
		if r := lastRoute(s); len(r.Tries) != 1 || r.Tries[0].Fail != "" {
			t.Fatalf("quoted message recorded as a failure: %+v", r.Tries)
		}
	}
}

func TestHighRiskErrorWhenRetryStops(t *testing.T) {
	for _, scenario := range []string{"no retries", "last candidate", "every candidate refuses"} {
		t.Run(scenario, func(t *testing.T) {
			fresh(t)
			if scenario == "no retries" {
				refusalRetries(t, 0)
			}
			errReply := reply{200, "text/event-stream", sse(`data: {"error":{"message":"` + highRiskMessage + `"}}`)}
			a, b := &scripted{replies: []reply{errReply}}, &scripted{replies: []reply{errReply}}
			scriptedOn(t, "a", provider.Chat, a)
			scriptedOn(t, "b", provider.Chat, b)
			members := []string{"a/m", "b/m"}
			if scenario == "last candidate" {
				members = members[:1]
			}
			refusalGroup(t, members...)
			s := New()
			path, ask := refusalErrorAsk(provider.Responses)
			code, body := sendRefusalError(t, s, path, ask)
			wantB := 0
			if scenario == "every candidate refuses" {
				wantB = 1
			}
			if code != 400 || !strings.Contains(body, highRiskMessage) || a.n != 1 || b.n != wantB {
				t.Fatalf("status %d: %s (a %d, b %d)", code, body, a.n, b.n)
			}
			for _, tr := range lastRoute(s).Tries {
				if tr.Fail != failRefused || tr.Rest != nil {
					t.Fatalf("refused candidate rested or changed error kind: %+v", tr)
				}
			}
		})
	}
}

func TestHighRiskTextAndStartedReplyAreNotReplayed(t *testing.T) {
	for _, content := range []string{
		`"content":"` + highRiskMessage + `"`,
		`"content":"The server says: ` + highRiskMessage + `"`,
		`"content":"half an answer"`,
		`"tool_calls":[{"index":0,"id":"call_1","type":"function","function":{"name":"lookup","arguments":"{}"}}]`,
	} {
		t.Run(content, func(t *testing.T) {
			fresh(t)
			body := sse(`data: {"id":"c1","choices":[{"index":0,"delta":{`+content+`}}]}`,
				`data: {"error":{"message":"`+highRiskMessage+`"}}`, `data: [DONE]`)
			a := &scripted{replies: []reply{{200, "text/event-stream", body}}}
			b := &scripted{replies: []reply{{200, "", chatOK}}}
			scriptedOn(t, "a", provider.Chat, a)
			scriptedOn(t, "b", provider.Chat, b)
			refusalGroup(t, "a/m", "b/m")
			s := New()
			path, ask := refusalErrorAsk(provider.Chat)
			code, got := sendRefusalError(t, s, path, ask)
			if code != 200 || got != body || a.n != 1 || b.n != 0 {
				t.Fatalf("started reply changed or replayed: %d %s (a %d, b %d)", code, got, a.n, b.n)
			}
			if r := lastRoute(s); len(r.Tries) != 1 || r.Tries[0].Fail != failRefused || r.Tries[0].Rest != nil {
				t.Fatalf("late refusal rested the account: %+v", r.Tries)
			}
		})
	}
}

func TestHighRiskNamedErrorAfterContentRestsNobody(t *testing.T) {
	fresh(t)
	body := sse(`data: {"id":"c1","choices":[{"index":0,"delta":{"content":"half"}}]}`,
		"event: error\n"+`data: {"message":"`+highRiskMessage+`"}`, `data: [DONE]`)
	a := &scripted{replies: []reply{{200, "text/event-stream", body}}}
	b := &scripted{replies: []reply{{200, "", chatOK}}}
	scriptedOn(t, "a", provider.Chat, a)
	scriptedOn(t, "b", provider.Chat, b)
	refusalGroup(t, "a/m", "b/m")
	s := New()
	path, ask := refusalErrorAsk(provider.Chat)
	code, got := sendRefusalError(t, s, path, ask)
	if code != 200 || got != body || a.n != 1 || b.n != 0 {
		t.Fatalf("started reply changed or replayed: %d %s (a %d, b %d)", code, got, a.n, b.n)
	}
	if r := lastRoute(s); len(r.Tries) != 1 || r.Tries[0].Fail != failRefused || r.Tries[0].Rest != nil {
		t.Fatalf("named refusal was not recognized: %+v", r.Tries)
	}
}

func TestHighRiskErrorSplitAfterContent(t *testing.T) {
	const start = "data: {\"choices\":[{\"delta\":{\"content\":\"half\"}}]}\n\n"
	for _, x := range []struct {
		name    string
		frame   string
		refused bool
	}{
		{"named error", sse("event: error\n" + `data: {"message":"` + highRiskMessage + `"}`), true},
		{"named coded error", sse("event: error\n" + `data: {"code":"content_filter","message":"filtered"}`), true},
		{"nested error", sse(`data: {"error":{"message":"` + highRiskMessage + `"}}`), true},
		{"quoted error", sse(`data: {"choices":[{"delta":{"content":"The error says: ` + highRiskMessage + `"}}]}`), false},
	} {
		t.Run(x.name, func(t *testing.T) {
			for split := 0; split <= len(x.frame); split++ {
				rec := httptest.NewRecorder()
				h := newHoldWriter(rec, true)
				h.Header().Set("Content-Type", "text/event-stream")
				stopped := false
				h.stop = func() { stopped = true }
				io.WriteString(h, start)
				io.WriteString(h, x.frame[:split])
				io.WriteString(h, x.frame[split:])
				if h.refusedAfter != x.refused || h.refused || stopped || !h.passing || rec.Body.String() != start+x.frame {
					t.Fatalf("split %d of %q: late refusal %v, held refusal %v, stopped %v, passing %v, body %q", split, x.frame, h.refusedAfter, h.refused, stopped, h.passing, rec.Body.String())
				}
			}
		})
	}
}