RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(parser): support .msg parsing and make email attachments retrievable (#18198)
Jackc23d5fc81994
diff --git a/.gitattributes b/.gitattributesindex 199ea9a55b..74d8a60212 100644--- a/.gitattributes+++ b/.gitattributes@@ -1,2 +1,9 @@*.sh text eol=lfdocker/entrypoint.sh text eol=lf executable++# Binary fixtures must be stored byte-for-byte. Mark them binary so git+# never applies text/CRLF normalization, and so the pre-commit text fixers+# (see tools/hooks/check_files.py) skip them. OLE2 .msg files in particular+# were previously corrupted by the mixed-line-ending / end-of-file fixers.+*.msg binary+diff --git a/go.mod b/go.modindex d460ccd1e3..81a245e863 100644--- a/go.mod+++ b/go.mod@@ -4,6 +4,7 @@ go 1.26.4require (cloud.google.com/go/storage v1.63.0+ github.com/AkmalOt/gomsg v0.0.0-20260407083308-985c3a1a76b7github.com/DATA-DOG/go-sqlmock v1.5.2github.com/LuxorLabs/tenki-sdk-go/sandbox v0.7.0github.com/alibabacloud-go/agentrun-20250910/v5 v5.8.4@@ -232,3 +233,5 @@ require ()replace github.com/infiniflow/infinity-go-sdk => github.com/infiniflow/infinity/go v0.0.0-20260806040857-d755c5ad25d9++replace github.com/AkmalOt/gomsg => github.com/xugangqiang/gomsg v0.0.0-20260407083308-985c3a1a76b7diff --git a/go.sum b/go.sumindex 25e50f4434..10b93787e2 100644--- a/go.sum+++ b/go.sum@@ -526,6 +526,8 @@ github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/github.com/wk8/go-ordered-map/v2 v2.1.8/go.mod h1:5nJHM5DyteebpVlHnWMV0rPz6Zp7+xBAnxjb1X5vnTw=github.com/x-cray/logrus-prefixed-formatter v0.5.2 h1:00txxvfBM9muc0jiLIEAkAcIMJzfthRT6usrui8uGmg=github.com/x-cray/logrus-prefixed-formatter v0.5.2/go.mod h1:2duySbKsL6M18s5GU7VPsoEPHyzalCE06qoARUCeBBE=+github.com/xugangqiang/gomsg v0.0.0-20260407083308-985c3a1a76b7 h1:iX9RUmlLw+Hoe78WV5vlm+JjIZUCzdRn2y6tWNxZTeY=+github.com/xugangqiang/gomsg v0.0.0-20260407083308-985c3a1a76b7/go.mod h1:TCjWm+lo7de/zYHb8rHfVSaKxOP4tZNrnRUzkqV4r28=github.com/xuri/efp v0.0.1 h1:fws5Rv3myXyYni8uwj2qKjVaRP30PdjeYe2Y6FDsCL8=github.com/xuri/efp v0.0.1/go.mod h1:ybY/Jr0T0GTCnYjKqmdwxyxn2BQf2RcQIIvex5QldPI=github.com/xuri/excelize/v2 v2.11.0 h1:HxaEFl6sRN2+8J5a8HaKq+0M4FsjBGMnWWtjOCPSG88=diff --git a/internal/parser/parser/email_parser.go b/internal/parser/parser/email_parser.goindex 3fe86f5cf7..14bede0c39 100644--- a/internal/parser/parser/email_parser.go+++ b/internal/parser/parser/email_parser.go@@ -22,24 +22,26 @@ import ("encoding/base64""fmt""io"+ "log""mime""mime/multipart""mime/quotedprintable""net/mail""path/filepath""strings"+ "time"+ "github.com/AkmalOt/gomsg""golang.org/x/text/encoding""golang.org/x/text/encoding/simplifiedchinese""golang.org/x/text/transform"++ "ragflow/internal/utility")-// EmailParser parses .eml (RFC 5322 email) files into structured-// JSON or plain-text output. Mirrors Python's _email() method in-// rag/flow/parser/parser.py.-//-// .msg (Outlook) files are not supported in the Go path; callers-// receive a clear error.+// EmailParser parses .eml (RFC 5322 email) and .msg (Outlook OLE2) files+// into structured JSON or plain-text output. Mirrors Python's _email()+// method in rag/flow/parser/parser.py, whose .msg branch uses extract_msg.type EmailParser struct {fields []stringoutputFormat string@@ -75,32 +77,78 @@ func (p *EmailParser) ConfigureFromSetup(setup map[string]any) {}func (p *EmailParser) ParseWithResult(ctx context.Context, filename string, data []byte) ParseResult {+ return p.parseEmail(ctx, filename, data, 0)+}++// parseEmail is ParseWithResult with a re-chunk depth so nested email+// attachments are parsed (and their body made retrievable) without unbounded+// recursion into attachments-of-attachments.+func (p *EmailParser) parseEmail(ctx context.Context, filename string, data []byte, depth int) ParseResult {ext := strings.ToLower(filepath.Ext(filename))++ var content map[string]anyif ext == ".msg" {- return ParseResult{- Err: fmt.Errorf("email: .msg (Outlook) files are not supported in the Go parser; use .eml format"),+ var (+ msg map[string]any+ err error+ )+ // gomsg.Decode parses untrusted OLE2/CFB input; guard against a+ // panic from a malformed .msg so one bad email can't take down the+ // ingestion worker.+ func() {+ defer func() {+ if r := recover(); r != nil {+ // Log so a genuine bug (e.g. a nil deref in parseMSG)+ // is not silently masked as a "decode panicked" error.+ log.Printf("email: .msg decode panicked for %q; skipping: %v", filename, r)+ err = fmt.Errorf("email: .msg decode panicked: %v", r)+ }+ }()+ msg, err = parseMSG(data, p.fields)+ }()+ if err != nil {+ return ParseResult{Err: fmt.Errorf("email: .msg: %w", err)}}+ content = msg+ } else {+ content = parseEML(bytes.NewReader(data), p.fields)}- emailContent := parseEML(bytes.NewReader(data), p.fields)-outputFormat := p.outputFormatif outputFormat == "" {outputFormat = "text"}+ // Re-chunk attachments so their content becomes retrievable within the+ // same document (user-oriented; mirrors Python legacy rag/app/email.py+ // naive_chunk). Each attachment is re-parsed by its file extension via+ // the shared parser registry; a single unparseable/skipped attachment+ // never breaks the whole email.+ extraItems, attachmentText := p.rechunkEmailAttachments(ctx, content, depth)++ // attachments has been consumed by rechunkEmailAttachments (which+ // re-parses each attachment by extension to make its content+ // retrievable). It is otherwise dead weight: jsonItemsToPages copies+ // every key into a schema.Page, but buildPagesFromBytes keeps only+ // text+doc_type_kwd, so carrying the full attachment payloads through to+ // the chunker would only bloat the intermediate pages before being+ // discarded. Drop it from the result content.+ delete(content, "attachments")+if outputFormat == "json" {- emailContent["doc_type_kwd"] = "text"+ content["doc_type_kwd"] = "text"+ items := []map[string]any{content}+ items = append(items, extraItems...)return ParseResult{OutputFormat: "json",File: map[string]any{"name": filename},- JSON: []map[string]any{emailContent},+ JSON: items,}}// Text output: flatten fields into a single string.var sb strings.Builder- for k, v := range emailContent {+ for k, v := range content {switch val := v.(type) {case string:sb.WriteString(k)@@ -119,19 +167,20 @@ func (p *EmailParser) ParseWithResult(ctx context.Context, filename string, data}}sb.WriteString("}\n")- case []map[string]any:- for _, att := range val {- fn, _ := att["filename"].(string)- pl, _ := att["payload"].(string)- sb.WriteString(fn)- sb.WriteString(":")- sb.WriteString(pl)- sb.WriteString("\n")- }case []string:sb.WriteString(strings.Join(val, "\n"))}}+ // Attachment text (re-parsed by extension) replaces the old crude+ // "filename:payload" flatten, so binary attachments no longer leak+ // mojibake into the searchable (indexed) text. The raw attachment+ // payloads themselves are dropped after rechunk (see parseEmail), so the+ // JSON output path carries only the re-parsed attachment text, never the+ // original payload bytes.+ if attachmentText != "" {+ sb.WriteString(attachmentText)+ sb.WriteString("\n")+ }return ParseResult{OutputFormat: "text",File: map[string]any{"name": filename},@@ -270,6 +319,12 @@ func readMailBody(body io.Reader, contentType string, collectAttachments bool) (attachments = append(attachments, map[string]any{"filename": attachmentFilename(part),"payload": decodeMailPayload(raw, partParams["charset"]),+ // raw preserves the byte-exact decoded-CTE bytes so the+ // re-chunk step re-parses the original attachment instead of+ // the charset-decoded string (which can differ when the+ // attachment declares a CJK charset). rechunkEmailAttachments+ // prefers "raw" and falls back to "payload".+ "raw": string(raw),})continue}@@ -417,3 +472,324 @@ func decodeTransform(payload []byte, decoder *encoding.Decoder) (string, error)}return "", fmt.Errorf("decode produced replacement characters")}++// parseMSG parses an Outlook .msg (OLE2 compound document) file using the+// gomsg library, mirroring the Python flow parser's _email() .msg branch+// (rag/flow/parser/parser.py), which parses via extract_msg. The output map+// shares the same field shape as parseEML so the downstream json/text+// assembly in ParseWithResult is reused unchanged.+func parseMSG(data []byte, fields []string) (map[string]any, error) {+ target := targetFieldsSet(fields)++ msg, err := gomsg.Decode(bytes.NewReader(data))+ if err != nil {+ return nil, err+ }++ content := map[string]any{}++ if target["from"] {+ content["from"] = formatSender(msg)+ }+ if target["to"] {+ content["to"] = formatRecipient(msg.DisplayTo)+ }+ if target["cc"] {+ content["cc"] = formatRecipient(msg.DisplayCC)+ }+ if target["bcc"] {+ content["bcc"] = formatRecipient(msg.DisplayBCC)+ }+ if target["date"] {+ content["date"] = formatMsgDate(msg.Date)+ }+ if target["subject"] {+ content["subject"] = msg.Subject+ }++ // Always emit metadata to match the Python flow parser contract, which+ // unconditionally builds a {message_id, in_reply_to} metadata dict for+ // .msg files regardless of whether "metadata" is in the configured fields.+ // Empty values are emitted as nil (JSON null) to match extract_msg's None.+ content["metadata"] = map[string]any{+ "message_id": orNil(msg.MessageID),+ "in_reply_to": orNil(msg.InReplyTo),+ }++ if target["body"] {+ // Mirror Python: prefer the plain body, fall back to the HTML body+ // when the plain body is empty. The .msg branch emits only "text"+ // (never "text_html"), matching the Python _email .msg contract exactly.+ text := msg.Body+ if strings.TrimSpace(text) == "" && len(msg.BodyHTML) > 0 {+ text = string(msg.BodyHTML)+ }+ content["text"] = text+ }++ if target["attachments"] {+ // Flatten attachments, recursing into embedded .msg files. gomsg+ // parses an embedded Message via Attachment.EmbeddedMessage even+ // though it exposes no raw bytes for it; the embedded message body+ // is surfaced as a retrievable text attachment (see msgAttachments).+ content["attachments"] = msgAttachments(msg)+ }++ return content, nil+}++// msgAttachments flattens a gomsg.Message's attachments into the+// {filename, payload} shape rechunkEmailAttachments consumes. Embedded .msg+// attachments expose no raw bytes via gomsg, but gomsg does parse the embedded+// Message; we surface its body as a retrievable text attachment (named .txt so+// the re-chunk step re-parses it as plain text) and recurse into its own+// attachments, so an embedded email is no longer silently dropped.+func msgAttachments(msg *gomsg.Message) []map[string]any {+ out := make([]map[string]any, 0, len(msg.Attachments))+ for _, a := range msg.Attachments {+ if a.IsEmbeddedMessage() {+ if em := a.EmbeddedMessage(); em != nil {+ body := em.Body+ if strings.TrimSpace(body) == "" {+ body = string(em.BodyHTML)+ }+ if body != "" {+ out = append(out, map[string]any{+ "filename": a.DisplayName() + ".txt",+ "payload": body,+ })+ }+ out = append(out, msgAttachments(em)...)+ continue+ }+ }+ out = append(out, map[string]any{+ "filename": a.DisplayName(),+ "payload": string(a.Data()),+ })+ }+ return out+}++// primarySenderEmail prefers the SMTP address, falling back to the raw email+// address when SMTP is unavailable (e.g. an EX address type).+func primarySenderEmail(msg *gomsg.Message) string {+ if msg.SenderSMTP != "" {+ return msg.SenderSMTP+ }+ return msg.SenderEmail+}++// formatSender renders the .msg sender the way extract_msg's "sender" string+// is displayed: "Display Name <email>" when a distinct display name is present,+// otherwise "<email>" (matching extract_msg's angular-bracket form for an+// address with no display name).+func formatSender(msg *gomsg.Message) string {+ email := primarySenderEmail(msg)+ if msg.SenderName != "" && msg.SenderName != email {+ return msg.SenderName + " <" + email + ">"+ }+ if email != "" {+ return "<" + email + ">"+ }+ return msg.SenderName+}++// formatRecipient renders a display recipient string the way extract_msg does:+// a bare single email address is wrapped in angle brackets, while a string+// that already contains an address form (display name, or multiple recipients)+// is returned unchanged.+func formatRecipient(display string) string {+ if display == "" {+ return ""+ }+ if strings.Contains(display, "<") {+ return display+ }+ if !strings.ContainsAny(display, ",;") && strings.Contains(display, "@") {+ return "<" + display + ">"+ }+ return display+}++// orNil maps an empty string to nil so it serializes as JSON null, matching+// extract_msg's None for absent properties.+func orNil(s string) any {+ if s == "" {+ return nil+ }+ return s+}++// formatMsgDate renders an Outlook .msg date the way extract_msg does:+// strftime("%Y-%m-%d %H:%M:%S%z"), which emits the zone without a colon+// (e.g. "2018-03-24 00:06:29+0800"). Go's -0700 layout reproduces that+// (the -07:00 layout would wrongly insert a colon). A zero time (date+// missing from the .msg) maps to nil so it serializes as JSON null,+// matching extract_msg's None, instead of a bogus sentinel such as+// "0001-01-01 00:00:00+0000".+func formatMsgDate(t time.Time) any {+ if t.IsZero() {+ return nil+ }+ return t.Format("2006-01-02 15:04:05-0700")+}++// recoverParse runs fn, converting a panic from an untrusted attachment+// parser (e.g. a corrupt PDF/DOCX hitting a native CGO backend) into a zero+// ParseResult with panicked=true, so the caller can skip that one attachment+// instead of failing the whole email. This mirrors the recover around the+// .msg parseMSG call in parseEmail.+func recoverParse(fn func() ParseResult) (res ParseResult, panicked bool) {+ defer func() {+ if r := recover(); r != nil {+ panicked = true+ }+ }()+ return fn(), false+}++// rechunkEmailAttachments re-parses each email attachment by its file+// extension and folds the extracted text back into the same document so+// attachment content becomes retrievable. This mirrors the user-oriented+// behaviour of Python's legacy rag/app/email.py, which re-chunks every+// attachment via naive_chunk and merges the resulting chunks into the same+// document.+//+// KNOWN LIMITATION: re-chunk uses the default-config parser returned by+// GetParser(ft) (and, for a nested email, only the top-level p.fields — no+// tenant/setup language, parse_method, or OCR/VLM model). The extracted+// attachment text may therefore differ from ingesting the same file as a+// standalone document through the pipeline's tenant/setup-configured+// parser. This is a deliberate, best-effort approximation for making+// attachment content retrievable (the same simplification Python's+// email.py naive_chunk makes), not a correctness bug; threading the full+// tenant/setup config into re-chunk is intentionally out of scope here.+//+// Attachments whose extension has no text-oriented parser (images, audio,+// video) are skipped: they carry no plain text in this pipeline and would+// require vision/speech models that are out of scope here. A single+// unparseable, empty, unsupported, or panicking attachment is skipped+// without failing the whole email (mirrors email.py's per-attachment+// try/except).+//+// The function returns both forms the caller needs:+// - extraItems: structured JSON items (one per non-empty text segment),+// each carrying only "text" and "doc_type_kwd", for the JSON output path.+// - text: the concatenated attachment text, for the text output path.+//+// Re-chunking stops at one level of nesting: a nested email is parsed (so its+// body becomes retrievable) but its own attachments are not re-chunked, to+// avoid unbounded recursion through attachments-of-attachments.+const maxRechunkPayloadBytes = 32 << 20 // 32 MiB++func (p *EmailParser) rechunkEmailAttachments(ctx context.Context, content map[string]any, depth int) ([]map[string]any, string) {+ if depth > 0 {+ return nil, ""+ }+ // Honor task cancellation so a long attachment list does not keep+ // re-parsing after the ingestion task has been stopped/aborted.+ if ctx.Err() != nil {+ return nil, ""+ }++ raw, ok := content["attachments"].([]map[string]any)+ if !ok || len(raw) == 0 {+ return nil, ""+ }++ var extra []map[string]any+ var sb strings.Builder+ for _, att := range raw {+ fn, _ := att["filename"].(string)+ payload, _ := att["payload"].(string)+ // Prefer the byte-exact raw bytes (when present) so a re-parsed+ // attachment is not silently corrupted by a prior charset decode.+ if rawPayload, ok := att["raw"].(string); ok && rawPayload != "" {+ payload = rawPayload+ }+ if fn == "" || payload == "" {+ // Empty payload (e.g. an embedded .msg with no exposed bytes) or+ // a missing filename — nothing to re-parse.+ continue+ }+ if len(payload) > maxRechunkPayloadBytes {+ // Too large to re-parse inline; re-chunking re-runs the heavy+ // parsers (PDF/OCR/...), which would otherwise dominate or stall+ // ingestion on a single huge attachment. The attachment remains+ // referenced in metadata.+ continue+ }+ ft := utility.GetFileType(fn)+ switch ft {+ case utility.FileTypeOTHER, utility.FileTypeVISUAL,+ utility.FileTypeAURAL, utility.FileTypeVIDEO, utility.FileTypeFOLDER:+ // No plain-text parser in this pipeline; skip rather than call+ // a vision/speech model that is out of scope.+ continue+ }+ var res ParseResult+ var panicked bool+ if ft == utility.FileTypeEMAIL {+ // Reuse the top-level field configuration (including "body") so a+ // nested .eml/.msg is parsed for its body instead of with a fresh,+ // unconfigured parser that would only emit metadata and index+ // garbage. We always request JSON output for the nested parse so+ // the extracted "text" field is returned clean (the text output+ // path would otherwise also flatten metadata into the result).+ ep := NewEmailParser()+ ep.ConfigureFromSetup(map[string]any{+ "fields": p.fields,+ "output_format": "json",+ })+ res, panicked = recoverParse(func() ParseResult {+ return ep.parseEmail(ctx, fn, []byte(payload), depth+1)+ })+ } else {+ np, err := GetParser(ft)+ if err != nil {+ continue+ }+ res, panicked = recoverParse(func() ParseResult {+ return np.ParseWithResult(ctx, fn, []byte(payload))+ })+ }+ if panicked {+ // An untrusted attachment parser (e.g. a corrupt PDF/DOCX hitting+ // a native CGO backend) panicked. Skip just this attachment so one+ // bad file can't fail the whole email — mirrors the .msg+ // parseMSG recover.+ log.Printf("email: attachment %q re-parse panicked; skipping", fn)+ }+ if panicked || res.Err != nil {+ continue+ }++ var texts []string+ if res.OutputFormat == "json" && len(res.JSON) > 0 {+ for _, it := range res.JSON {+ if t, ok := it["text"].(string); ok {+ if t = strings.TrimSpace(t); t != "" {+ texts = append(texts, t)+ }+ }+ }+ } else if res.Text != "" {+ if t := strings.TrimSpace(res.Text); t != "" {+ texts = append(texts, t)+ }+ }++ for _, t := range texts {+ extra = append(extra, map[string]any{+ "text": t,+ "doc_type_kwd": "text",+ })+ if sb.Len() > 0 {+ sb.WriteString("\n")+ }+ sb.WriteString(t)+ }+ }+ return extra, sb.String()+}diff --git a/internal/parser/parser/email_parser_test.go b/internal/parser/parser/email_parser_test.goindex aeeb83095c..dd9436aa8a 100644--- a/internal/parser/parser/email_parser_test.go+++ b/internal/parser/parser/email_parser_test.go@@ -17,9 +17,15 @@package parserimport (+ "bytes"+ "context""encoding/base64"+ "mime/multipart"+ "net/textproto"+ "os""strings""testing"+ "time")func TestEmailParser_EmlJSON(t *testing.T) {@@ -110,15 +116,122 @@ func TestEmailParser_EmlText(t *testing.T) {}}-func TestEmailParser_MsgNotSupported(t *testing.T) {+// TestEmailParser_MsgSupported parses a real Outlook .msg fixture and verifies+// the Go output aligns with the Python flow parser _email() .msg branch+// (rag/flow/parser/parser.py). Replaces the old "MsgNotSupported" test now that+// .msg is supported via gomsg.+func TestEmailParser_MsgSupported(t *testing.T) {ctx := t.Context()+ data, err := os.ReadFile("testdata/sample.msg")+ if err != nil {+ t.Fatalf("read fixture: %v", err)+ }++ p := NewEmailParser()+ p.ConfigureFromSetup(map[string]any{+ "output_format": "json",+ "fields": []string{"from", "to", "cc", "bcc", "date", "subject", "body", "attachments", "metadata"},+ })++ result := p.ParseWithResult(ctx, "sample.msg", data)+ if result.Err != nil {+ t.Fatalf("unexpected error: %v", result.Err)+ }+ if len(result.JSON) != 1 {+ t.Fatalf("expected 1 JSON item, got %d", len(result.JSON))+ }+ item := result.JSON[0]++ if v, ok := item["from"].(string); !ok || v != "<[email protected]>" {+ t.Errorf("from: got %q", v)+ }+ if v, ok := item["to"].(string); !ok || v != "<[email protected]>" {+ t.Errorf("to: got %q", v)+ }+ if v, ok := item["subject"].(string); !ok || v != "asdf" {+ t.Errorf("subject: got %q", v)+ }+ if v, ok := item["date"].(string); !ok || v != "2018-03-24 00:06:29+0800" {+ t.Errorf("date: got %q, want 2018-03-24 00:06:29+0800", v)+ }+ if v, ok := item["text"].(string); !ok || v != " \r\n\r\n" {+ t.Errorf("text: got %q", v)+ }+ // The .msg branch must NOT emit text_html (matches Python _email .msg branch).+ if _, ok := item["text_html"]; ok {+ t.Error("text_html must be absent for .msg")+ }+ meta, ok := item["metadata"].(map[string]any)+ if !ok {+ t.Fatalf("metadata missing or wrong type: %T", item["metadata"])+ }+ if v, ok := meta["message_id"].(string); !ok || v == "" {+ t.Errorf("metadata message_id: got %q", v)+ }+ // Empty in_reply_to mirrors extract_msg's None -> JSON null.+ if v, ok := meta["in_reply_to"]; ok && v != nil {+ t.Errorf("metadata in_reply_to: got %v, want nil", v)+ }+ if _, ok := meta["in_reply_to"]; !ok {+ t.Error("metadata in_reply_to key must be present")+ }+ // attachments are extracted by the .msg branch but deliberately dropped+ // from the final ParseResult (consumed by rechunkEmailAttachments;+ // buildPagesFromBytes keeps only text+doc_type_kwd). Verify the+ // high-level result no longer carries the heavy payload...+ if _, ok := item["attachments"]; ok {+ t.Error("attachments must be dropped from the final ParseResult")+ }+ // ...and verify the .msg branch still extracts them at the parse level.+ msgContent, err := parseMSG(data, []string{"from", "to", "cc", "bcc", "date", "subject", "body", "attachments", "metadata"})+ if err != nil {+ t.Fatalf("parseMSG: %v", err)+ }+ atts, ok := msgContent["attachments"].([]map[string]any)+ if !ok {+ t.Fatalf("parseMSG attachments missing or wrong type: %T", msgContent["attachments"])+ }+ if len(atts) != 1 {+ t.Fatalf("expected 1 attachment, got %d", len(atts))+ }+ if fn, _ := atts[0]["filename"].(string); fn != "5AAoPFgV-nJ965R7o-98C38840-4454-4750-9AEF-F53DB3E37548.jpg" {+ t.Errorf("filename = %q", fn)+ }+ if pl, _ := atts[0]["payload"].(string); len(pl) != 122784 {+ t.Errorf("payload length = %d, want 122784", len(pl))+ }+}++// TestEmailParser_MsgMetadataAlwaysPresent verifies the .msg branch emits+// metadata unconditionally (matching the Python contract) even when "metadata"+// is omitted from the configured fields.+func TestEmailParser_MsgMetadataAlwaysPresent(t *testing.T) {+ ctx := t.Context()+ data, err := os.ReadFile("testdata/sample.msg")+ if err != nil {+ t.Fatalf("read fixture: %v", err)+ }+p := NewEmailParser()- result := p.ParseWithResult(ctx, "test.msg", []byte{})- if result.Err == nil {- t.Fatal("expected error for .msg file")+ p.ConfigureFromSetup(map[string]any{+ "output_format": "json",+ "fields": []string{"from", "subject"}, // "metadata" intentionally absent+ })++ result := p.ParseWithResult(ctx, "sample.msg", data)+ if result.Err != nil {+ t.Fatalf("unexpected error: %v", result.Err)}- if !strings.Contains(result.Err.Error(), ".msg") {- t.Errorf("error should mention .msg: %v", result.Err)+ item := result.JSON[0]++ if _, ok := item["metadata"].(map[string]any); !ok {+ t.Fatalf("metadata must always be present for .msg, got %T", item["metadata"])+ }+ // Basic fields not in fields are dropped.+ for _, dropped := range []string{"to", "date", "body", "attachments"} {+ if _, ok := item[dropped]; ok {+ t.Errorf("%s should be dropped when not in fields", dropped)+ }}}@@ -163,9 +276,16 @@ func TestEmailParser_Base64Attachment(t *testing.T) {}item := result.JSON[0]- atts, ok := item["attachments"].([]map[string]any)+ // attachments are dropped from the final ParseResult (consumed by+ // rechunk, then deleted); verify the high-level result is clean...+ if _, ok := item["attachments"]; ok {+ t.Error("attachments must be dropped from the final ParseResult")+ }+ // ...and verify the .eml branch still decodes the base64 attachment.+ eml := parseEML(bytes.NewReader([]byte(raw)), []string{"from", "body", "attachments"})+ atts, ok := eml["attachments"].([]map[string]any)if !ok {- t.Fatalf("attachments missing or wrong type: %T", item["attachments"])+ t.Fatalf("attachments missing or wrong type: %T", eml["attachments"])}if len(atts) != 1 {t.Fatalf("expected 1 attachment, got %d", len(atts))@@ -242,8 +362,14 @@ func TestEmailParser_Base64AttachmentInMixedMultipart(t *testing.T) {t.Errorf("text_html: got %q, want to contain 'HTML body'", v)}- // Verify attachment is decoded from base64- atts, ok := item["attachments"].([]map[string]any)+ // attachments are dropped from the final ParseResult; verify the+ // high-level result is clean...+ if _, ok := item["attachments"]; ok {+ t.Error("attachments must be dropped from the final ParseResult")+ }+ // ...and verify the .eml branch still decodes the base64 attachment.+ eml := parseEML(bytes.NewReader([]byte(raw)), []string{"from", "body", "attachments"})+ atts, ok := eml["attachments"].([]map[string]any)if !ok || len(atts) != 1 {t.Fatalf("expected 1 attachment, got %d", len(atts))}@@ -404,6 +530,219 @@ func TestEmailParser_TextHTMLAlwaysPresent(t *testing.T) {// empty list because attachment extraction was coupled to the body branch// (the "else if needAttachments" fallback set an empty slice instead of// walking the message).+// TestEmailParser_AttachmentSearchableJSON verifies the user-oriented+// behaviour: an email attachment is re-parsed by its file extension and its+// content becomes a retrievable chunk in the SAME document (mirrors Python+// legacy rag/app/email.py naive_chunk). The attachment text must appear as a+// separate JSON item, while the email body stays on the main item.+func TestEmailParser_AttachmentSearchableJSON(t *testing.T) {+ ctx := t.Context()+ attachmentContent := "QUOTE: the quick brown fox jumps over the lazy dog."+ encoded := base64.StdEncoding.EncodeToString([]byte(attachmentContent))+ boundary := "attachboundary"+ raw := strings.Join([]string{+ "From: [email protected]",+ "To: [email protected]",+ "Subject: Attachment Test",+ "Content-Type: multipart/mixed; boundary=" + boundary,+ "",+ "--" + boundary,+ "Content-Type: text/plain; charset=utf-8",+ "",+ "Email body text.",+ "--" + boundary,+ "Content-Type: text/plain; charset=utf-8",+ "Content-Disposition: attachment; filename=\"note.txt\"",+ "Content-Transfer-Encoding: base64",+ "",+ encoded,+ "--" + boundary + "--",+ }, "\r\n")++ p := NewEmailParser()+ p.ConfigureFromSetup(map[string]any{+ "output_format": "json",+ "fields": []string{"from", "body", "attachments"},+ })++ result := p.ParseWithResult(ctx, "test.eml", []byte(raw))+ if result.Err != nil {+ t.Fatalf("unexpected error: %v", result.Err)+ }++ // Body must remain on the main item.+ if v, ok := result.JSON[0]["text"].(string); !ok || !strings.Contains(v, "Email body text") {+ t.Errorf("body missing on main item: %v", result.JSON[0]["text"])+ }++ // Attachment text must appear as a separate retrievable JSON item.+ found := false+ for _, it := range result.JSON {+ if txt, ok := it["text"].(string); ok && strings.Contains(txt, "quick brown fox") {+ found = true+ }+ }+ if !found {+ t.Errorf("attachment text not found in JSON output:\n%#v", result.JSON)+ }+}++// TestEmailParser_AttachmentSearchableText is the text-output equivalent:+// the re-parsed attachment text must be present in result.Text.+func TestEmailParser_AttachmentSearchableText(t *testing.T) {+ ctx := t.Context()+ attachmentContent := "QUOTE: the quick brown fox jumps over the lazy dog."+ encoded := base64.StdEncoding.EncodeToString([]byte(attachmentContent))+ boundary := "attachboundary"+ raw := strings.Join([]string{+ "From: [email protected]",+ "To: [email protected]",+ "Subject: Attachment Test",+ "Content-Type: multipart/mixed; boundary=" + boundary,+ "",+ "--" + boundary,+ "Content-Type: text/plain; charset=utf-8",+ "",+ "Email body text.",+ "--" + boundary,+ "Content-Type: text/plain; charset=utf-8",+ "Content-Disposition: attachment; filename=\"note.txt\"",+ "Content-Transfer-Encoding: base64",+ "",+ encoded,+ "--" + boundary + "--",+ }, "\r\n")++ p := NewEmailParser()+ p.ConfigureFromSetup(map[string]any{+ "output_format": "text",+ "fields": []string{"from", "body", "attachments"},+ })++ result := p.ParseWithResult(ctx, "test.eml", []byte(raw))+ if result.Err != nil {+ t.Fatalf("unexpected error: %v", result.Err)+ }+ if !strings.Contains(result.Text, "quick brown fox") {+ t.Errorf("attachment text missing from text output: %q", result.Text)+ }+ if !strings.Contains(result.Text, "Email body text") {+ t.Errorf("email body missing from text output: %q", result.Text)+ }+}++// TestEmailParser_AttachmentEmptyPayloadSkipped verifies error isolation:+// an attachment with an empty payload (the legacy .msg nested-attachment+// case, where gomsg exposes no raw bytes) is skipped without breaking the+// email, and a valid sibling attachment is still re-chunked.+func TestEmailParser_AttachmentEmptyPayloadSkipped(t *testing.T) {+ ctx := t.Context()+ validContent := "VALID attachment payload"+ validEncoded := base64.StdEncoding.EncodeToString([]byte(validContent))+ boundary := "mixedbound"+ raw := strings.Join([]string{+ "From: [email protected]",+ "To: [email protected]",+ "Subject: Empty Payload",+ "Content-Type: multipart/mixed; boundary=" + boundary,+ "",+ "--" + boundary,+ "Content-Type: text/plain; charset=utf-8",+ "",+ "Hello body.",+ "--" + boundary,+ "Content-Type: application/octet-stream; name=\"empty.bin\"",+ "Content-Disposition: attachment; filename=\"empty.bin\"",+ "Content-Transfer-Encoding: base64",+ "",+ "",+ "--" + boundary,+ "Content-Type: text/plain; charset=utf-8",+ "Content-Disposition: attachment; filename=\"valid.txt\"",+ "Content-Transfer-Encoding: base64",+ "",+ validEncoded,+ "--" + boundary + "--",+ }, "\r\n")++ p := NewEmailParser()+ p.ConfigureFromSetup(map[string]any{+ "output_format": "json",+ "fields": []string{"from", "body", "attachments"},+ })++ result := p.ParseWithResult(ctx, "test.eml", []byte(raw))+ if result.Err != nil {+ t.Fatalf("email must not fail on empty-payload attachment: %v", result.Err)+ }++ found := false+ for _, it := range result.JSON {+ if txt, ok := it["text"].(string); ok && strings.Contains(txt, "VALID attachment payload") {+ found = true+ }+ }+ if !found {+ t.Errorf("valid attachment text not found despite empty-payload sibling: %#v", result.JSON)+ }+}++// TestEmailParser_AttachmentUnknownExtSkipped verifies that an attachment+// with an unrecognized extension is skipped (no parser available) while a+// valid sibling attachment is still re-chunked. Mirrors the legacy email.py+// behaviour, which only re-chunks known file types.+func TestEmailParser_AttachmentUnknownExtSkipped(t *testing.T) {+ ctx := t.Context()+ validContent := "KNOWN attachment payload"+ validEncoded := base64.StdEncoding.EncodeToString([]byte(validContent))+ boundary := "mixedbound"+ raw := strings.Join([]string{+ "From: [email protected]",+ "To: [email protected]",+ "Subject: Unknown Ext",+ "Content-Type: multipart/mixed; boundary=" + boundary,+ "",+ "--" + boundary,+ "Content-Type: text/plain; charset=utf-8",+ "",+ "Hello body.",+ "--" + boundary,+ "Content-Type: application/octet-stream; name=\"data.xyz\"",+ "Content-Disposition: attachment; filename=\"data.xyz\"",+ "Content-Transfer-Encoding: base64",+ "",+ base64.StdEncoding.EncodeToString([]byte("opaque bytes")),+ "--" + boundary,+ "Content-Type: text/plain; charset=utf-8",+ "Content-Disposition: attachment; filename=\"known.txt\"",+ "Content-Transfer-Encoding: base64",+ "",+ validEncoded,+ "--" + boundary + "--",+ }, "\r\n")++ p := NewEmailParser()+ p.ConfigureFromSetup(map[string]any{+ "output_format": "json",+ "fields": []string{"from", "body", "attachments"},+ })++ result := p.ParseWithResult(ctx, "test.eml", []byte(raw))+ if result.Err != nil {+ t.Fatalf("email must not fail on unknown-extension attachment: %v", result.Err)+ }++ found := false+ for _, it := range result.JSON {+ if txt, ok := it["text"].(string); ok && strings.Contains(txt, "KNOWN attachment payload") {+ found = true+ }+ }+ if !found {+ t.Errorf("known attachment text not found despite unknown-ext sibling: %#v", result.JSON)+ }+}+func TestEmailParser_AttachmentsWithoutBody(t *testing.T) {ctx := t.Context()attachmentContent := "SECRET attachment payload"@@ -449,10 +788,16 @@ func TestEmailParser_AttachmentsWithoutBody(t *testing.T) {t.Error("text_html should be absent when body not in fields")}- // attachments must be extracted even without body.- atts, ok := item["attachments"].([]map[string]any)+ // attachments are dropped from the final ParseResult (consumed by+ // rechunk, then deleted); verify the high-level result is clean...+ if _, ok := item["attachments"]; ok {+ t.Error("attachments must be dropped from the final ParseResult")+ }+ // ...and verify the .eml branch still extracts them even without body.+ eml := parseEML(bytes.NewReader([]byte(raw)), []string{"from", "attachments"})+ atts, ok := eml["attachments"].([]map[string]any)if !ok {- t.Fatalf("attachments missing or wrong type: %T", item["attachments"])+ t.Fatalf("attachments missing or wrong type: %T", eml["attachments"])}if len(atts) != 1 {t.Fatalf("expected 1 attachment without body, got %d (bug: attachments silently dropped)", len(atts))@@ -464,3 +809,258 @@ func TestEmailParser_AttachmentsWithoutBody(t *testing.T) {t.Errorf("payload = %q, want %q", pl, attachmentContent)}}++// TestEmailParser_NestedEMLAttachmentRechunk locks the regression where a+// nested .eml attachment was re-chunked with an UNCONFIGURED EmailParser+// (fields == nil), so parseEML emitted only metadata and the text path indexed+// "metadata:{...}" garbage. The nested email must instead be parsed with the+// top-level field configuration (including "body") so its body becomes the+// indexed text. The outer email's own metadata flattening is expected; the+// nested email must NOT contribute a second "metadata:{" segment.+func TestEmailParser_NestedEMLAttachmentRechunk(t *testing.T) {+ ctx := t.Context()++ innerRaw := strings.Join([]string{+ "From: [email protected]",+ "To: [email protected]",+ "Subject: inner",+ "Content-Type: text/plain; charset=utf-8",+ "",+ "INNER BODY SECRET",+ }, "\r\n")++ boundary := "outerbound"+ raw := strings.Join([]string{+ "From: [email protected]",+ "To: [email protected]",+ "Subject: outer",+ "MIME-Version: 1.0",+ "Content-Type: multipart/mixed; boundary=" + boundary,+ "",+ "--" + boundary,+ "Content-Type: text/plain; charset=utf-8",+ "",+ "OUTER BODY VISIBLE",+ "--" + boundary,+ "Content-Type: message/rfc822",+ "Content-Disposition: attachment; filename=\"inner.eml\"",+ "",+ innerRaw,+ "--" + boundary + "--",+ }, "\r\n")++ for _, format := range []string{"json", "text"} {+ p := NewEmailParser()+ p.ConfigureFromSetup(map[string]any{+ "output_format": format,+ "fields": []string{"from", "to", "subject", "body", "attachments", "metadata"},+ })++ result := p.ParseWithResult(ctx, "outer.eml", []byte(raw))+ if result.Err != nil {+ t.Fatalf("[%s] unexpected error: %v", format, result.Err)+ }++ // Collect all indexed text for this output mode (JSON populates+ // result.JSON; text populates result.Text).+ var indexed strings.Builder+ if format == "json" {+ for _, it := range result.JSON {+ if txt, ok := it["text"].(string); ok {+ indexed.WriteString(txt)+ indexed.WriteString("\n")+ }+ }+ } else {+ indexed.WriteString(result.Text)+ }++ // The nested email body must be retrievable.+ if !strings.Contains(indexed.String(), "INNER BODY SECRET") {+ t.Errorf("[%s] nested body not indexed: %q", format, indexed.String())+ }+ if !strings.Contains(indexed.String(), "OUTER BODY VISIBLE") {+ t.Errorf("[%s] outer body missing: %q", format, indexed.String())+ }++ if format == "json" {+ // The nested email must be a distinct, clean item, carrying no+ // metadata key (the old bug leaked a "metadata:{...}" string as+ // its text).+ found := false+ for _, it := range result.JSON {+ if txt, ok := it["text"].(string); ok && strings.Contains(txt, "INNER BODY SECRET") {+ found = true+ if _, hasMeta := it["metadata"]; hasMeta {+ t.Errorf("[json] nested item must not carry metadata key: %#v", it)+ }+ }+ }+ if !found {+ t.Errorf("[json] nested body not found as separate item: %#v", result.JSON)+ }+ } else {+ // Text mode: the outer email's own metadata is flattened once;+ // the nested email must NOT add a second "metadata:{" segment.+ if c := strings.Count(result.Text, "metadata:{"); c != 1 {+ t.Errorf("[text] expected exactly 1 metadata:{ segment (outer only), got %d in: %q",+ c, result.Text)+ }+ }+ }+}++// TestEmailParser_RechunkAttachmentTextPath exercises rechunkEmailAttachments+// directly with the {filename, payload} attachment shape that both .eml and+// .msg parsing feed into it. It verifies text attachments are indexed while+// binary (VISUAL) attachments are skipped, and that binary payloads never leak+// into the indexed text.+func TestEmailParser_RechunkAttachmentTextPath(t *testing.T) {+ ctx := t.Context()+ content := map[string]any{+ "attachments": []map[string]any{+ {"filename": "note.txt", "payload": "hello from attachment"},+ {"filename": "pic.jpg", "payload": "<binary bytes>"},+ },+ }++ p := NewEmailParser()+ extra, text := p.rechunkEmailAttachments(ctx, content, 0)++ if len(extra) != 1 {+ t.Fatalf("expected 1 indexed attachment, got %d: %#v", len(extra), extra)+ }+ if v, _ := extra[0]["text"].(string); v != "hello from attachment" {+ t.Errorf("text = %q, want hello from attachment", v)+ }+ if !strings.Contains(text, "hello from attachment") {+ t.Errorf("indexed text missing attachment: %q", text)+ }+ if strings.Contains(text, "<binary bytes>") {+ t.Errorf("binary payload leaked into indexed text: %q", text)+ }+}++// TestFormatMsgDate verifies the .msg date rendering: a zero time (date+// missing from the .msg) maps to nil (JSON null, matching extract_msg's+// None) instead of a bogus sentinel like "0001-01-01 00:00:00+0000".+func TestFormatMsgDate(t *testing.T) {+ if got := formatMsgDate(time.Time{}); got != nil {+ t.Errorf("zero date should map to nil, got %#v", got)+ }+ got := formatMsgDate(time.Date(2018, 3, 24, 0, 6, 29, 0, time.FixedZone("CST", 8*3600)))+ if got != "2018-03-24 00:06:29+0800" {+ t.Errorf("formatted date = %q, want 2018-03-24 00:06:29+0800", got)+ }+}++// TestRechunkEmailAttachments_ContextCancelled verifies that an already+// cancelled context short-circuits re-chunking instead of re-parsing+// attachments (mirrors the cancellation check CodeRabbit flagged).+func TestRechunkEmailAttachments_ContextCancelled(t *testing.T) {+ ctx, cancel := context.WithCancel(t.Context())+ cancel() // task already aborted+ content := map[string]any{+ "attachments": []map[string]any{+ {"filename": "note.txt", "payload": "must not be re-chunked"},+ },+ }+ extra, text := NewEmailParser().rechunkEmailAttachments(ctx, content, 0)+ if len(extra) != 0 || text != "" {+ t.Errorf("cancelled context should short-circuit re-chunk: extra=%#v text=%q", extra, text)+ }+}++// TestRecoverParse_IsolatesPanic verifies that a panic from an untrusted+// attachment parser is converted into a zero ParseResult with panicked=true+// and does NOT propagate out of recoverParse (so a single bad attachment can+// be skipped instead of failing the whole email).+func TestRecoverParse_IsolatesPanic(t *testing.T) {+ res, panicked := recoverParse(func() ParseResult {+ panic("boom")+ })+ if !panicked {+ t.Error("expected panicked=true")+ }+ if res.Err != nil || res.JSON != nil || res.Text != "" {+ t.Errorf("panic should yield a zero ParseResult, got %#v", res)+ }+}++// TestRecoverParse_PassesThrough verifies a normal parser result is returned+// unchanged with panicked=false.+func TestRecoverParse_PassesThrough(t *testing.T) {+ want := ParseResult{OutputFormat: "text", Text: "hello"}+ res, panicked := recoverParse(func() ParseResult {+ return want+ })+ if panicked {+ t.Error("expected panicked=false for a normal result")+ }+ if res.Text != "hello" {+ t.Errorf("result not passed through: %#v", res)+ }+}++// TestEmailParser_RechunkPrefersRaw verifies that rechunkEmailAttachments+// re-parses the byte-exact "raw" bytes when present, rather than the+// charset-decoded "payload". The fallback (no "raw") still uses "payload".+func TestEmailParser_RechunkPrefersRaw(t *testing.T) {+ ctx := t.Context()+ p := NewEmailParser()++ withRaw, textRaw := p.rechunkEmailAttachments(ctx, map[string]any{+ "attachments": []map[string]any{+ {"filename": "note.txt", "payload": "WORLD", "raw": "HELLO"},+ },+ }, 0)+ if len(withRaw) != 1 || withRaw[0]["text"] != "HELLO" {+ t.Fatalf("raw not preferred: extra=%#v text=%q", withRaw, textRaw)+ }+ if strings.Contains(textRaw, "WORLD") {+ t.Errorf("re-chunk used payload instead of raw: %q", textRaw)+ }++ fallback, textFallback := p.rechunkEmailAttachments(ctx, map[string]any{+ "attachments": []map[string]any{+ {"filename": "note.txt", "payload": "WORLD"},+ },+ }, 0)+ if len(fallback) != 1 || fallback[0]["text"] != "WORLD" {+ t.Fatalf("payload fallback broken: extra=%#v text=%q", fallback, textFallback)+ }+}++// TestReadMailBody_AttachmentPreservesRaw verifies that .eml attachment+// collection stores the byte-exact decoded-CTE bytes under "raw" alongside the+// charset-decoded "payload". Declaring charset=gbk makes decodeMailPayload+// decode GBK bytes to a UTF-8 string, so the two differ — re-chunk relies on+// "raw" to avoid silently corrupting the attachment.+func TestReadMailBody_AttachmentPreservesRaw(t *testing.T) {+ // GBK-encoded "中文" (中=0xD6D0, 文=0xCEC4).+ gbk := []byte{0xD6, 0xD0, 0xCE, 0xC4}++ var buf bytes.Buffer+ mw := multipart.NewWriter(&buf)+ bp, _ := mw.CreatePart(textproto.MIMEHeader{"Content-Type": {"text/plain"}})+ bp.Write([]byte("body"))+ ap, _ := mw.CreatePart(textproto.MIMEHeader{+ "Content-Type": {"application/octet-stream; charset=gbk"},+ "Content-Disposition": {`attachment; filename="x.bin"`},+ "Content-Transfer-Encoding": {"base64"},+ })+ ap.Write([]byte(base64.StdEncoding.EncodeToString(gbk)))+ mw.Close()++ _, _, attachments := readMailBody(strings.NewReader(buf.String()), "multipart/mixed; boundary="+mw.Boundary(), true)+ if len(attachments) != 1 {+ t.Fatalf("expected 1 attachment, got %d: %#v", len(attachments), attachments)+ }+ att := attachments[0]+ if raw, _ := att["raw"].(string); raw != string(gbk) {+ t.Errorf("raw = %q, want byte-exact %q", raw, string(gbk))+ }+ if payload, _ := att["payload"].(string); payload != "中文" {+ t.Errorf("payload = %q, want 中文 (charset-decoded)", payload)+ }+}diff --git a/internal/parser/parser/testdata/sample.msg b/internal/parser/parser/testdata/sample.msgnew file mode 100644index 0000000000..00e8d0ce28Binary files /dev/null and b/internal/parser/parser/testdata/sample.msg differdiff --git a/test/test_check_files.py b/test/test_check_files.pynew file mode 100644index 0000000000..ca5acaa712--- /dev/null+++ b/test/test_check_files.py@@ -0,0 +1,92 @@+# Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++"""Tests for tools/hooks/check_files.py.++These lock the strict UTF-8 decode behaviour introduced when the hook stopped+using errors="ignore": a file whose bytes are not valid UTF-8 (and which does+not contain a NUL byte, so the binary guard misses it) must be skipped rather+than silently rewritten with dropped bytes when running in --fix mode.+"""++import sys+from pathlib import Path++# pytest's pythonpath includes "." but be defensive about import location.+_ROOT = Path(__file__).resolve().parents[1]+if str(_ROOT) not in sys.path:+ sys.path.insert(0, str(_ROOT))++from tools.hooks.check_files import (+ check_merge_conflicts,+ check_trailing_whitespace,+)+++def test_trailing_whitespace_fix_skips_invalid_utf8_without_corruption(tmp_path):+ # Invalid UTF-8 (no NUL byte) carrying trailing whitespace.+ f = tmp_path / "blob.bin"+ original = b"data \xff\xfe trailing \n"+ f.write_bytes(original)++ rc = check_trailing_whitespace([f], fix=True)++ # Skipped (no error reported) and, crucially, the bytes are untouched:+ # the old errors="ignore" path would have dropped the invalid bytes and+ # rewritten the file, corrupting it.+ assert rc == 0+ assert f.read_bytes() == original+++def test_trailing_whitespace_fix_strips_valid_utf8(tmp_path):+ f = tmp_path / "ok.txt"+ f.write_bytes(b"hello \n")++ rc = check_trailing_whitespace([f], fix=True)++ assert rc == 0+ assert f.read_bytes() == b"hello\n"+++def test_trailing_whitespace_fix_is_noop_on_clean_file(tmp_path):+ f = tmp_path / "ok.txt"+ f.write_bytes(b"clean\n")++ check_trailing_whitespace([f], fix=True)++ assert f.read_bytes() == b"clean\n"+++def test_merge_conflicts_skips_binary_file_with_markers(tmp_path):+ # A binary file whose bytes happen to contain ASCII conflict markers.+ f = tmp_path / "blob.bin"+ f.write_bytes(b"\x00<<<<<<< HEAD\n>>>>>>> branch\n")++ rc = check_merge_conflicts([f])++ # Skipped (binary guard), so no false-positive conflict report.+ assert rc == 0+++def test_merge_conflicts_detects_real_conflict(tmp_path):+ f = tmp_path / "conflicted.txt"+ f.write_text(+ "<<<<<<< HEAD\nlocal\n=======\nincoming\n>>>>>>> branch\n",+ encoding="utf-8",+ )++ rc = check_merge_conflicts([f])++ # Real conflict markers are still detected.+ assert rc == 1diff --git a/tools/hooks/check_files.py b/tools/hooks/check_files.pyold mode 100644new mode 100755index ac85253c35..7427c00483--- a/tools/hooks/check_files.py+++ b/tools/hooks/check_files.py@@ -12,7 +12,6 @@ from pathlib import Pathimport yaml-MERGE_PATTERNS = ("<<<<<<< ", "=======\n", ">>>>>>> ")# Printable ASCII (0x20-0x7E) plus newline — matches the regex used by the@@ -24,6 +23,15 @@ def _read_bytes(path: Path) -> bytes:return path.read_bytes()+# Binary fixtures (OLE2/.msg, PDF, images, …) contain NUL bytes. The+# text-fixing hooks below must never rewrite them: line-ending or+# trailing-newline normalization corrupts binary files byte-for-byte+# (e.g. a .msg fixture was previously mangled by the mixed-line-ending+# and end-of-file fixers). Skip any file that contains a NUL byte.+def _is_binary(data: bytes) -> bool:+ return b"\x00" in data++def _git_paths(*args: str) -> list[Path]:proc = subprocess.run(["git", *args, "-z"],@@ -52,7 +60,7 @@ def check_json(paths: list[Path], fix: bool = False) -> int:continuetry:json.loads(path.read_text(encoding="utf-8"))- except Exception as exc:+ except (json.JSONDecodeError, OSError, UnicodeDecodeError) as exc:errors.append(f"invalid json: {path}: {exc}")return _report(errors)@@ -64,7 +72,7 @@ def check_yaml(paths: list[Path], fix: bool = False) -> int:continuetry:yaml.safe_load(path.read_text(encoding="utf-8"))- except Exception as exc:+ except (yaml.YAMLError, OSError, UnicodeDecodeError) as exc:errors.append(f"invalid yaml: {path}: {exc}")return _report(errors)@@ -75,6 +83,8 @@ def check_eof(paths: list[Path], fix: bool = False) -> int:if not path.is_file():continuedata = _read_bytes(path)+ if _is_binary(data):+ continueif data and not data.endswith(b"\n"):if fix:with path.open("ab") as f:@@ -94,8 +104,17 @@ def check_trailing_whitespace(paths: list[Path], fix: bool = False) -> int:if not path.is_file():continuetry:- text = path.read_text(encoding="utf-8", errors="ignore")- except Exception:+ data = _read_bytes(path)+ except OSError:+ continue+ if _is_binary(data):+ continue+ try:+ text = data.decode("utf-8")+ except UnicodeDecodeError:+ # Not valid UTF-8 (and not NUL-binary): skip rather than silently+ # dropping bytes with errors="ignore", which would corrupt the+ # file when run with fix=True.continueif not text:continue@@ -120,6 +139,8 @@ def check_mixed_line_endings(paths: list[Path], fix: bool = False) -> int:if not path.is_file():continuedata = _read_bytes(path)+ if _is_binary(data):+ continuehas_crlf = b"\r\n" in datahas_lf = b"\n" in data.replace(b"\r\n", b"")if has_crlf and has_lf:@@ -136,7 +157,18 @@ def check_merge_conflicts(paths: list[Path], fix: bool = False) -> int:for path in paths:if not path.is_file():continue- text = path.read_text(encoding="utf-8", errors="ignore")+ try:+ data = _read_bytes(path)+ except OSError:+ continue+ if _is_binary(data):+ continue+ try:+ text = data.decode("utf-8")+ except UnicodeDecodeError:+ # Not valid UTF-8: skip rather than risking a false positive from+ # bytes mangled by errors="ignore".+ continueif all(pattern in text for pattern in MERGE_PATTERNS):errors.append(f"merge conflict markers: {path}")return _report(errors)