Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
fix(cli): preserve raw MCP document download bytes (#4256)
Rudy Mizrahi Celeklic573d727d8e2
diff --git a/cli/internal/mcp/download_bytes_test.go b/cli/internal/mcp/download_bytes_test.gonew file mode 100644index 000000000..187ee1357--- /dev/null+++ b/cli/internal/mcp/download_bytes_test.go@@ -0,0 +1,75 @@+package mcp++import (+ "bytes"+ "encoding/base64"+ "encoding/json"+ "io"+ "testing"++ mcpsdk "github.com/modelcontextprotocol/go-sdk/mcp"+)++func TestTool_DocDownload_PreservesRawBytes(t *testing.T) {+ cases := []struct {+ name string+ data []byte+ base64 bool+ }{+ {name: "ASCII text", data: []byte("plain text\n")},+ {name: "Unicode text", data: []byte("中文 café 🐄\n")},+ {name: "empty file", data: []byte{}},+ {name: "early NUL", data: []byte{'a', 0, 'b'}, base64: true},+ {name: "invalid first byte", data: []byte{0xff, 'a', 'b'}, base64: true},+ {name: "invalid middle byte", data: []byte{'a', 0xff, 'b'}, base64: true},+ {name: "truncated UTF-8", data: []byte{'a', 0xe2, 0x82}, base64: true},+ {name: "invalid after probe", data: append(bytes.Repeat([]byte{'a'}, 512), 0xff), base64: true},+ }+ for _, tc := range cases {+ t.Run(tc.name, func(t *testing.T) {+ svc := &fakeSvc{+ openDocName: "fixture.bin",+ openDocBody: io.NopCloser(bytes.NewReader(tc.data)),+ }+ c, _ := newTestServer(t, svc)+ var structured docDownloadOutput+ result := callTool(t, c, "doc_download", map[string]any{"doc_id": "doc-1"}, &structured)+ if result.StructuredContent == nil {+ t.Fatal("missing structured content")+ }+ if len(result.Content) != 1 {+ t.Fatalf("content blocks=%d, want 1", len(result.Content))+ }+ text, ok := result.Content[0].(*mcpsdk.TextContent)+ if !ok {+ t.Fatalf("content type=%T, want text", result.Content[0])+ }+ var fallback docDownloadOutput+ if err := json.Unmarshal([]byte(text.Text), &fallback); err != nil {+ t.Fatal(err)+ }+ if structured != fallback {+ t.Errorf("structured and text content disagree: %+v vs %+v", structured, fallback)+ }+ for name, output := range map[string]docDownloadOutput{"structured": structured, "text": fallback} {+ if output.DocID != "doc-1" || output.FileName != "fixture.bin" || output.Bytes != len(tc.data) {+ t.Errorf("%s metadata=%+v, want fixture identity and bytes=%d", name, output, len(tc.data))+ }+ if output.IsBase64 != tc.base64 {+ t.Errorf("%s is_base64=%v, want %v", name, output.IsBase64, tc.base64)+ }+ decoded := []byte(output.Content)+ if output.IsBase64 {+ var err error+ decoded, err = base64.StdEncoding.DecodeString(output.Content)+ if err != nil {+ t.Fatalf("%s base64: %v", name, err)+ }+ }+ if !bytes.Equal(decoded, tc.data) {+ t.Errorf("%s byte round-trip=%x, want %x", name, decoded, tc.data)+ }+ }+ })+ }+}diff --git a/cli/internal/mcp/tools.go b/cli/internal/mcp/tools.goindex 72f2a49a8..fba57953b 100644--- a/cli/internal/mcp/tools.go+++ b/cli/internal/mcp/tools.go@@ -8,6 +8,7 @@ import ("io""strings""time"+ "unicode/utf8"mcpsdk "github.com/modelcontextprotocol/go-sdk/mcp"@@ -326,8 +327,8 @@ type docDownloadOutput struct {DocID string `json:"doc_id"`FileName string `json:"file_name"`Bytes int `json:"bytes"`- // Content is the file contents (UTF-8 if text, base64 if the SDK- // reports a binary-looking blob). For binary, agents should decode+ // Content is the file contents (UTF-8 if text, base64 for invalid+ // UTF-8 or a binary-looking blob). For base64, agents should decode// before consuming.Content string `json:"content"`IsBase64 bool `json:"is_base64"`@@ -341,8 +342,10 @@ const maxDocDownloadBytes = 1 << 20func addDocDownload(server *mcpsdk.Server, svc knowledgeService) {mcpsdk.AddTool(server, &mcpsdk.Tool{- Name: "doc_download",- Description: "Download a document's raw bytes by ID. Capped at 1 MiB per call - for larger documents, use search_chunks to find the relevant excerpts. is_base64 reports whether content was base64-encoded (heuristic: presence of NUL byte in the first 512 bytes).",+ Name: "doc_download",+ Description: "Download a document's raw bytes by ID. Capped at 1 MiB per call - for larger documents, " ++ "use search_chunks to find the relevant excerpts. is_base64 reports whether content was " ++ "base64-encoded (invalid UTF-8 or presence of NUL byte in the first 512 bytes).",Annotations: &mcpsdk.ToolAnnotations{Title: "Download Document",DestructiveHint: bptr(false),@@ -689,11 +692,13 @@ func addChunkList(server *mcpsdk.Server, svc chunkListService) {})}-// encodeDownload returns (content, isBase64). Heuristic: if the first 512-// bytes contain a NUL, treat as binary. Otherwise it's UTF-8-ish text.-// Matches what /usr/bin/file's "binary" heuristic does at a coarse level --// good enough to spare an agent from base64-decoding obvious text.+// encodeDownload returns (content, isBase64). Invalid UTF-8 must be encoded+// to preserve bytes through JSON serialization. Valid UTF-8 containing a+// NUL in the first 512 bytes also uses base64 for binary-looking content.func encodeDownload(buf []byte) (string, bool) {+ if !utf8.Valid(buf) {+ return base64.StdEncoding.EncodeToString(buf), true+ }probe := bufif len(probe) > 512 {probe = probe[:512]