Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(gmail): retain complete MIME payload when loading threads (#21811)

Xiang1cc1eaed3232
diff --git a/internal/syncer/connector/gmail.go b/internal/syncer/connector/gmail.go
index 7044cf5785..d9f29c7ae8 100644
--- a/internal/syncer/connector/gmail.go
+++ b/internal/syncer/connector/gmail.go
@@ -210,7 +210,9 @@ func (c *GmailConnector) loadThread(ctx context.Context, userEmail, threadID str
if err != nil {
return gmailThread{}, err
}
- query := url.Values{"fields": {"id,messages(id,labelIds,payload(headers,parts(body(data),mimeType)))"}}
+ // Keep the complete MIME subtree: bodies may live at the root or at any
+ // nesting depth, and a shallow fields projection silently removes them.
+ query := url.Values{"fields": {"id,messages(id,labelIds,payload)"}}
var thread gmailThread
err = c.getJSON(ctx, client, "https://gmail.googleapis.com/gmail/v1/users/"+url.PathEscape(userEmail)+"/threads/"+url.PathEscape(threadID)+"?"+query.Encode(), &thread)
return thread, err
diff --git a/internal/syncer/connector/gmail_test.go b/internal/syncer/connector/gmail_test.go
index eb521592ee..c53b7829d5 100644
--- a/internal/syncer/connector/gmail_test.go
+++ b/internal/syncer/connector/gmail_test.go
@@ -1,9 +1,12 @@
package connector
import (
+ "bytes"
"context"
"encoding/base64"
+ "encoding/json"
"errors"
+ "fmt"
"io"
"net/http"
"strings"
@@ -332,3 +335,62 @@ func TestGmailTimeRangeQuery(t *testing.T) {
t.Fatalf("query = %q", got)
}
}
+
+// Exercise the HTTP path: the regular fixtures bypass loadThread and therefore
+// cannot catch fields projections that remove body data before decoding.
+func TestGmailSyncRequestsCompleteMimePayload(t *testing.T) {
+ for _, nested := range []bool{false, true} {
+ t.Run(fmt.Sprintf("nested=%t", nested), func(t *testing.T) {
+ c, err := NewGmailConnector(map[string]any{"batch_size": 1})
+ if err != nil {
+ t.Fatal(err)
+ }
+ c.listUsers = func(context.Context) ([]string, error) { return []string{"[email protected]"}, nil }
+ c.httpClientForUser = func(context.Context, string) (*http.Client, error) {
+ return &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
+ var result any
+ if strings.HasSuffix(req.URL.Path, "/threads") {
+ result = map[string]any{"threads": []map[string]string{{"id": "thread-1"}}}
+ } else {
+ payload := map[string]any{"mimeType": "text/plain", "headers": []map[string]string{{"name": "Subject", "value": "Body projection regression"}}, "body": map[string]string{"data": base64.RawURLEncoding.EncodeToString([]byte("invoice amount 125"))}}
+ if nested {
+ payload = map[string]any{"mimeType": "multipart/mixed", "headers": payload["headers"], "parts": []any{map[string]any{"mimeType": "multipart/alternative", "parts": []any{payload}}}}
+ }
+ fields := req.URL.Query().Get("fields")
+ // Model Google's documented partial-response behavior for
+ // the old narrow projection versus a complete payload.
+ switch fields {
+ case "id,messages(id,labelIds,payload(headers,parts(body(data),mimeType)))":
+ delete(payload, "body")
+ delete(payload, "mimeType")
+ if nested {
+ payload["parts"] = []any{map[string]string{"mimeType": "multipart/alternative"}}
+ }
+ case "", "id,messages(id,labelIds,payload)":
+ default:
+ t.Fatalf("fixture does not model fields %q", fields)
+ }
+ result = map[string]any{"id": "thread-1", "messages": []any{map[string]any{"id": "message-1", "payload": payload}}}
+ }
+ data, err := json.Marshal(result)
+ if err != nil {
+ return nil, err
+ }
+ return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(bytes.NewReader(data))}, nil
+ })}, nil
+ }
+ session, err := c.OpenSync(t.Context(), SyncRequest{FromBeginning: true})
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer session.Close()
+ batch, err := session.NextBatch(t.Context())
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(batch.Documents) != 1 || !strings.Contains(string(batch.Documents[0].Blob), "invoice amount 125") {
+ t.Fatalf("mail body missing from synced document: %+v", batch.Documents)
+ }
+ })
+ }
+}