RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(gmail): retain complete MIME payload when loading threads (#21811)
Xiang1cc1eaed3232
diff --git a/internal/syncer/connector/gmail.go b/internal/syncer/connector/gmail.goindex 7044cf5785..d9f29c7ae8 100644--- a/internal/syncer/connector/gmail.go+++ b/internal/syncer/connector/gmail.go@@ -210,7 +210,9 @@ func (c *GmailConnector) loadThread(ctx context.Context, userEmail, threadID strif err != nil {return gmailThread{}, err}- query := url.Values{"fields": {"id,messages(id,labelIds,payload(headers,parts(body(data),mimeType)))"}}+ // Keep the complete MIME subtree: bodies may live at the root or at any+ // nesting depth, and a shallow fields projection silently removes them.+ query := url.Values{"fields": {"id,messages(id,labelIds,payload)"}}var thread gmailThreaderr = c.getJSON(ctx, client, "https://gmail.googleapis.com/gmail/v1/users/"+url.PathEscape(userEmail)+"/threads/"+url.PathEscape(threadID)+"?"+query.Encode(), &thread)return thread, errdiff --git a/internal/syncer/connector/gmail_test.go b/internal/syncer/connector/gmail_test.goindex eb521592ee..c53b7829d5 100644--- a/internal/syncer/connector/gmail_test.go+++ b/internal/syncer/connector/gmail_test.go@@ -1,9 +1,12 @@package connectorimport (+ "bytes""context""encoding/base64"+ "encoding/json""errors"+ "fmt""io""net/http""strings"@@ -332,3 +335,62 @@ func TestGmailTimeRangeQuery(t *testing.T) {t.Fatalf("query = %q", got)}}++// Exercise the HTTP path: the regular fixtures bypass loadThread and therefore+// cannot catch fields projections that remove body data before decoding.+func TestGmailSyncRequestsCompleteMimePayload(t *testing.T) {+ for _, nested := range []bool{false, true} {+ t.Run(fmt.Sprintf("nested=%t", nested), func(t *testing.T) {+ c, err := NewGmailConnector(map[string]any{"batch_size": 1})+ if err != nil {+ t.Fatal(err)+ }+ c.listUsers = func(context.Context) ([]string, error) { return []string{"[email protected]"}, nil }+ c.httpClientForUser = func(context.Context, string) (*http.Client, error) {+ return &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {+ var result any+ if strings.HasSuffix(req.URL.Path, "/threads") {+ result = map[string]any{"threads": []map[string]string{{"id": "thread-1"}}}+ } else {+ payload := map[string]any{"mimeType": "text/plain", "headers": []map[string]string{{"name": "Subject", "value": "Body projection regression"}}, "body": map[string]string{"data": base64.RawURLEncoding.EncodeToString([]byte("invoice amount 125"))}}+ if nested {+ payload = map[string]any{"mimeType": "multipart/mixed", "headers": payload["headers"], "parts": []any{map[string]any{"mimeType": "multipart/alternative", "parts": []any{payload}}}}+ }+ fields := req.URL.Query().Get("fields")+ // Model Google's documented partial-response behavior for+ // the old narrow projection versus a complete payload.+ switch fields {+ case "id,messages(id,labelIds,payload(headers,parts(body(data),mimeType)))":+ delete(payload, "body")+ delete(payload, "mimeType")+ if nested {+ payload["parts"] = []any{map[string]string{"mimeType": "multipart/alternative"}}+ }+ case "", "id,messages(id,labelIds,payload)":+ default:+ t.Fatalf("fixture does not model fields %q", fields)+ }+ result = map[string]any{"id": "thread-1", "messages": []any{map[string]any{"id": "message-1", "payload": payload}}}+ }+ data, err := json.Marshal(result)+ if err != nil {+ return nil, err+ }+ return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(bytes.NewReader(data))}, nil+ })}, nil+ }+ session, err := c.OpenSync(t.Context(), SyncRequest{FromBeginning: true})+ if err != nil {+ t.Fatal(err)+ }+ defer session.Close()+ batch, err := session.NextBatch(t.Context())+ if err != nil {+ t.Fatal(err)+ }+ if len(batch.Documents) != 1 || !strings.Contains(string(batch.Documents[0].Blob), "invoice amount 125") {+ t.Fatalf("mail body missing from synced document: %+v", batch.Documents)+ }+ })+ }+}