Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

fix(cli): preserve raw MCP document download bytes (#4256)

Rudy Mizrahi Celeklic573d727d8e2
diff --git a/cli/internal/mcp/download_bytes_test.go b/cli/internal/mcp/download_bytes_test.go
new file mode 100644
index 000000000..187ee1357
--- /dev/null
+++ b/cli/internal/mcp/download_bytes_test.go
@@ -0,0 +1,75 @@
+package mcp
+
+import (
+ "bytes"
+ "encoding/base64"
+ "encoding/json"
+ "io"
+ "testing"
+
+ mcpsdk "github.com/modelcontextprotocol/go-sdk/mcp"
+)
+
+func TestTool_DocDownload_PreservesRawBytes(t *testing.T) {
+ cases := []struct {
+ name string
+ data []byte
+ base64 bool
+ }{
+ {name: "ASCII text", data: []byte("plain text\n")},
+ {name: "Unicode text", data: []byte("中文 café 🐄\n")},
+ {name: "empty file", data: []byte{}},
+ {name: "early NUL", data: []byte{'a', 0, 'b'}, base64: true},
+ {name: "invalid first byte", data: []byte{0xff, 'a', 'b'}, base64: true},
+ {name: "invalid middle byte", data: []byte{'a', 0xff, 'b'}, base64: true},
+ {name: "truncated UTF-8", data: []byte{'a', 0xe2, 0x82}, base64: true},
+ {name: "invalid after probe", data: append(bytes.Repeat([]byte{'a'}, 512), 0xff), base64: true},
+ }
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ svc := &fakeSvc{
+ openDocName: "fixture.bin",
+ openDocBody: io.NopCloser(bytes.NewReader(tc.data)),
+ }
+ c, _ := newTestServer(t, svc)
+ var structured docDownloadOutput
+ result := callTool(t, c, "doc_download", map[string]any{"doc_id": "doc-1"}, &structured)
+ if result.StructuredContent == nil {
+ t.Fatal("missing structured content")
+ }
+ if len(result.Content) != 1 {
+ t.Fatalf("content blocks=%d, want 1", len(result.Content))
+ }
+ text, ok := result.Content[0].(*mcpsdk.TextContent)
+ if !ok {
+ t.Fatalf("content type=%T, want text", result.Content[0])
+ }
+ var fallback docDownloadOutput
+ if err := json.Unmarshal([]byte(text.Text), &fallback); err != nil {
+ t.Fatal(err)
+ }
+ if structured != fallback {
+ t.Errorf("structured and text content disagree: %+v vs %+v", structured, fallback)
+ }
+ for name, output := range map[string]docDownloadOutput{"structured": structured, "text": fallback} {
+ if output.DocID != "doc-1" || output.FileName != "fixture.bin" || output.Bytes != len(tc.data) {
+ t.Errorf("%s metadata=%+v, want fixture identity and bytes=%d", name, output, len(tc.data))
+ }
+ if output.IsBase64 != tc.base64 {
+ t.Errorf("%s is_base64=%v, want %v", name, output.IsBase64, tc.base64)
+ }
+ decoded := []byte(output.Content)
+ if output.IsBase64 {
+ var err error
+ decoded, err = base64.StdEncoding.DecodeString(output.Content)
+ if err != nil {
+ t.Fatalf("%s base64: %v", name, err)
+ }
+ }
+ if !bytes.Equal(decoded, tc.data) {
+ t.Errorf("%s byte round-trip=%x, want %x", name, decoded, tc.data)
+ }
+ }
+ })
+ }
+}
diff --git a/cli/internal/mcp/tools.go b/cli/internal/mcp/tools.go
index 72f2a49a8..fba57953b 100644
--- a/cli/internal/mcp/tools.go
+++ b/cli/internal/mcp/tools.go
@@ -8,6 +8,7 @@ import (
"io"
"strings"
"time"
+ "unicode/utf8"
mcpsdk "github.com/modelcontextprotocol/go-sdk/mcp"
@@ -326,8 +327,8 @@ type docDownloadOutput struct {
DocID string `json:"doc_id"`
FileName string `json:"file_name"`
Bytes int `json:"bytes"`
- // Content is the file contents (UTF-8 if text, base64 if the SDK
- // reports a binary-looking blob). For binary, agents should decode
+ // Content is the file contents (UTF-8 if text, base64 for invalid
+ // UTF-8 or a binary-looking blob). For base64, agents should decode
// before consuming.
Content string `json:"content"`
IsBase64 bool `json:"is_base64"`
@@ -341,8 +342,10 @@ const maxDocDownloadBytes = 1 << 20
func addDocDownload(server *mcpsdk.Server, svc knowledgeService) {
mcpsdk.AddTool(server, &mcpsdk.Tool{
- Name: "doc_download",
- Description: "Download a document's raw bytes by ID. Capped at 1 MiB per call - for larger documents, use search_chunks to find the relevant excerpts. is_base64 reports whether content was base64-encoded (heuristic: presence of NUL byte in the first 512 bytes).",
+ Name: "doc_download",
+ Description: "Download a document's raw bytes by ID. Capped at 1 MiB per call - for larger documents, " +
+ "use search_chunks to find the relevant excerpts. is_base64 reports whether content was " +
+ "base64-encoded (invalid UTF-8 or presence of NUL byte in the first 512 bytes).",
Annotations: &mcpsdk.ToolAnnotations{
Title: "Download Document",
DestructiveHint: bptr(false),
@@ -689,11 +692,13 @@ func addChunkList(server *mcpsdk.Server, svc chunkListService) {
})
}
-// encodeDownload returns (content, isBase64). Heuristic: if the first 512
-// bytes contain a NUL, treat as binary. Otherwise it's UTF-8-ish text.
-// Matches what /usr/bin/file's "binary" heuristic does at a coarse level -
-// good enough to spare an agent from base64-decoding obvious text.
+// encodeDownload returns (content, isBase64). Invalid UTF-8 must be encoded
+// to preserve bytes through JSON serialization. Valid UTF-8 containing a
+// NUL in the first 512 bytes also uses base64 for binary-looking content.
func encodeDownload(buf []byte) (string, bool) {
+ if !utf8.Valid(buf) {
+ return base64.StdEncoding.EncodeToString(buf), true
+ }
probe := buf
if len(probe) > 512 {
probe = probe[:512]