BillionMail gives you open-source MailServer, NewsLetter, Email Marketing — fully self-hosted, dev-friendly, and free from monthly fees. Join the discord: https://discord.gg/asfXzBUhZr
Add batch upsert functionality for abnormal recipients with detailed error reasons and improve MX record validation
lotk3fd42d48bc7a
diff --git a/core/api/abnormal_recipient/v1/abnormal_recipient.go b/core/api/abnormal_recipient/v1/abnormal_recipient.goindex 01a06f63..8b243e45 100644--- a/core/api/abnormal_recipient/v1/abnormal_recipient.go+++ b/core/api/abnormal_recipient/v1/abnormal_recipient.go@@ -73,7 +73,7 @@ type AbnormalSwitchRes struct {}type SetAbnormalSwitchReq struct {- g.Meta `path:"/abnormal_recipient/set_check_switch" method:"post" tags:"Abnormal Recipient" summary:"Abnormal Switch"`+ g.Meta `path:"/abnormal_recipient/set_check_switch" method:"post" tags:"Abnormal Recipient" summary:"Set Check Abnormal Switch"`Authorization string `json:"authorization" dc:"Authorization" in:"header"`Oper int `json:"oper" v:"required|in:0,1" dc:"Operation type (1 for on, 0 for off)"`}diff --git a/core/internal/controller/abnormal_recipient/abnormal_recipient_v1_check_group.go b/core/internal/controller/abnormal_recipient/abnormal_recipient_v1_check_group.goindex 456c3809..ca739000 100644--- a/core/internal/controller/abnormal_recipient/abnormal_recipient_v1_check_group.go+++ b/core/internal/controller/abnormal_recipient/abnormal_recipient_v1_check_group.go@@ -15,6 +15,7 @@ import ("net""os""strings"+ "sync""time")@@ -22,7 +23,7 @@ type DNSRecord struct {Type string `json:"type"`Host string `json:"host"`Value string `json:"value"`- Valid bool `json:"valid"`+ Valid bool `json:"valid"`}func (c *ControllerV1) CheckGroup(ctx context.Context, req *v1.CheckGroupReq) (res *v1.CheckGroupRes, err error) {@@ -85,34 +86,58 @@ func (c *ControllerV1) CheckGroup(ctx context.Context, req *v1.CheckGroupReq) (rtotalCount = len(emails)validCount = 0invalidCount = 0+ wg sync.WaitGroup+ mutex sync.Mutex++ invalidEmailDetails []abnormal_recipient.RecipientDetail)+ // Use a channel to control concurrency, limiting to 20 concurrent DNS lookups+ const maxConcurrency = 20+ semaphore := make(chan struct{}, maxConcurrency)+for domain, domainEmails := range domainMap {- date := time.Now().Format("2006-01-02 15:04:05")- record := DNSRecord{- Type: "MX",- Host: "@",- Value: domain,- }- //g.Log().Info(ctx, "Checking domain:", domain, "emails:", domainEmails)- status := ValidateMXRecord(record, domain)- //g.Log().Info(ctx, "ValidateMXRecord result for domain:", domain, "status:", status)- if status {- validEmails = append(validEmails, domainEmails...)- validCount += len(domainEmails)- appendLogLine(logDir, fmt.Sprintf("%s: %s ----------------------------- √", date, domain))-- } else {- invalidEmails = append(invalidEmails, domainEmails...)- invalidCount += len(domainEmails)- appendLogLine(logDir, fmt.Sprintf("%s: %s ----------------------------- x mx logging error", date, domain))- }+ wg.Add(1)+ go func(domain string, domainEmails []string) {+ defer wg.Done()+ semaphore <- struct{}{} // Acquire a token+ defer func() { <-semaphore }() // Release the token++ date := time.Now().Format("2006-01-02 15:04:05")+ record := DNSRecord{+ Type: "MX",+ Host: "@",+ Value: domain,+ }+ status, reason := ValidateMXRecord(record, domain)++ mutex.Lock()+ defer mutex.Unlock()++ if status {+ validEmails = append(validEmails, domainEmails...)+ validCount += len(domainEmails)+ appendLogLine(logDir, fmt.Sprintf("%s: %s ----------------------------- √", date, domain))+ } else {+ invalidEmails = append(invalidEmails, domainEmails...)+ for _, email := range domainEmails {+ invalidEmailDetails = append(invalidEmailDetails, abnormal_recipient.RecipientDetail{+ Email: email,+ ErrorReason: reason,+ })+ }+ invalidCount += len(domainEmails)+ appendLogLine(logDir, fmt.Sprintf("%s: %s ----------------------------- x %s", date, domain, reason))+ }+ }(domain, domainEmails)}+ wg.Wait() // Wait for all goroutines to finish+switch req.Oper {case 2: // Add abnormal table- _ = abnormal_recipient.BatchUpsertAbnormalRecipients(ctx, invalidEmails, 3, "Manual scanning group")- appendLogLine(logDir, "Adding invalid emails to blocklist...")+ _ = abnormal_recipient.BatchUpsertAbnormalRecipientsWithDetails(ctx, invalidEmailDetails, 3, "Manual scanning group (MX check)")+ appendLogLine(logDir, "Adding invalid emails to blocklist with details...")case 3: // Remove from the group_, _ = g.DB().Model("bm_contacts").Where("group_id", req.GroupId).WhereIn("email", invalidEmails).Delete()appendLogLine(logDir, "Removing invalid emails from group...")@@ -138,10 +163,10 @@ func (c *ControllerV1) CheckGroup(ctx context.Context, req *v1.CheckGroupReq) (rreturn res, nil}-func ValidateMXRecord(record DNSRecord, domain string, aRecordHosts ...string) bool {+func ValidateMXRecord(record DNSRecord, domain string, aRecordHosts ...string) (bool, string) {g.Log().Info(context.Background(), "ValidateMXRecord called with:", record, domain)if strings.ToUpper(record.Type) != "MX" {- return false+ return false, "Invalid record type"}if record.Host != "@" {@@ -163,18 +188,16 @@ func ValidateMXRecord(record DNSRecord, domain string, aRecordHosts ...string) bif err != nil {g.Log().Error(context.Background(), "Failed to query MX records after retries:", err)- return false+ return false, "DNS lookup failed: " + err.Error()}g.Log().Debug(context.Background(), "Query MX records success", mxRecords)- aRecordHosts = append([]string{record.Value}, aRecordHosts...)-if len(mxRecords) > 0 {- return true+ return true, ""}- return false+ return false, "no MX record"}func appendLogLine(filePath, line string) {diff --git a/core/internal/service/abnormal_recipient/abnormal_recipient.go b/core/internal/service/abnormal_recipient/abnormal_recipient.goindex f58f6ff7..0916d8e7 100644--- a/core/internal/service/abnormal_recipient/abnormal_recipient.go+++ b/core/internal/service/abnormal_recipient/abnormal_recipient.go@@ -140,6 +140,76 @@ func BatchUpsertAbnormalRecipients(ctx context.Context, recipients []string, addreturn nil}+// BatchUpsertAbnormalRecipientsWithDetails+func BatchUpsertAbnormalRecipientsWithDetails(ctx context.Context, recipientDetails []RecipientDetail, addType int, baseDescription string) error {+ now := time.Now().Unix()++ if len(recipientDetails) == 0 {+ return nil+ }++ recipients := make([]string, len(recipientDetails))+ detailsMap := make(map[string]RecipientDetail)+ for i, detail := range recipientDetails {+ recipients[i] = detail.Email+ detailsMap[detail.Email] = detail+ }++ var existList []entity.AbnormalRecipient+ err := g.DB().Model("abnormal_recipient").WhereIn("recipient", recipients).Scan(&existList)+ if err != nil {+ return fmt.Errorf("Failed to query existing abnormal recipients: %w", err)+ }+ existMap := make(map[string]*entity.AbnormalRecipient)+ for _, r := range existList {+ existMap[r.Recipient] = &r+ }++ // 1. Update the existing records+ for _, r := range existList {+ detail := detailsMap[r.Recipient]+ description := fmt.Sprintf("%s - %s", baseDescription, detail.ErrorReason)++ _, err := g.DB().Model("abnormal_recipient").Where("id", r.Id).Data(g.Map{+ "count": r.Count + 1,+ "description": description,+ "add_type": addType,+ }).Update()+ if err != nil {+ return fmt.Errorf("Failed to update abnormal recipient: %w", err)+ }+ }++ // 2. Inserting a non-existent record+ var insertList []g.Map+ for _, detail := range recipientDetails {+ if _, ok := existMap[detail.Email]; !ok {+ description := fmt.Sprintf("%s - %s", baseDescription, detail.ErrorReason)+ insertList = append(insertList, g.Map{+ "recipient": detail.Email,+ "count": 1,+ "add_type": addType,+ "description": description,+ "create_time": now,+ })+ }+ }++ if len(insertList) > 0 {+ _, err := g.DB().Model("abnormal_recipient").Data(insertList).InsertIgnore()+ if err != nil {+ return fmt.Errorf("Failed to insert abnormal recipients: %w", err)+ }+ }++ return nil+}++type RecipientDetail struct {+ Email string `json:"email"`+ ErrorReason string `json:"error_reason"`+}+func AbnormalRecipientAutoStat(ctx context.Context) {var abnormalSwitch string