Skip to content

BillionMail gives you open-source MailServer, NewsLetter, Email Marketing — fully self-hosted, dev-friendly, and free from monthly fees. Join the discord: https://discord.gg/asfXzBUhZr

Add batch upsert functionality for abnormal recipients with detailed error reasons and improve MX record validation

lotk3fd42d48bc7a
diff --git a/core/api/abnormal_recipient/v1/abnormal_recipient.go b/core/api/abnormal_recipient/v1/abnormal_recipient.go
index 01a06f63..8b243e45 100644
--- a/core/api/abnormal_recipient/v1/abnormal_recipient.go
+++ b/core/api/abnormal_recipient/v1/abnormal_recipient.go
@@ -73,7 +73,7 @@ type AbnormalSwitchRes struct {
}
type SetAbnormalSwitchReq struct {
- g.Meta `path:"/abnormal_recipient/set_check_switch" method:"post" tags:"Abnormal Recipient" summary:"Abnormal Switch"`
+ g.Meta `path:"/abnormal_recipient/set_check_switch" method:"post" tags:"Abnormal Recipient" summary:"Set Check Abnormal Switch"`
Authorization string `json:"authorization" dc:"Authorization" in:"header"`
Oper int `json:"oper" v:"required|in:0,1" dc:"Operation type (1 for on, 0 for off)"`
}
diff --git a/core/internal/controller/abnormal_recipient/abnormal_recipient_v1_check_group.go b/core/internal/controller/abnormal_recipient/abnormal_recipient_v1_check_group.go
index 456c3809..ca739000 100644
--- a/core/internal/controller/abnormal_recipient/abnormal_recipient_v1_check_group.go
+++ b/core/internal/controller/abnormal_recipient/abnormal_recipient_v1_check_group.go
@@ -15,6 +15,7 @@ import (
"net"
"os"
"strings"
+ "sync"
"time"
)
@@ -22,7 +23,7 @@ type DNSRecord struct {
Type string `json:"type"`
Host string `json:"host"`
Value string `json:"value"`
- Valid bool `json:"valid"`
+ Valid bool `json:"valid"`
}
func (c *ControllerV1) CheckGroup(ctx context.Context, req *v1.CheckGroupReq) (res *v1.CheckGroupRes, err error) {
@@ -85,34 +86,58 @@ func (c *ControllerV1) CheckGroup(ctx context.Context, req *v1.CheckGroupReq) (r
totalCount = len(emails)
validCount = 0
invalidCount = 0
+ wg sync.WaitGroup
+ mutex sync.Mutex
+
+ invalidEmailDetails []abnormal_recipient.RecipientDetail
)
+ // Use a channel to control concurrency, limiting to 20 concurrent DNS lookups
+ const maxConcurrency = 20
+ semaphore := make(chan struct{}, maxConcurrency)
+
for domain, domainEmails := range domainMap {
- date := time.Now().Format("2006-01-02 15:04:05")
- record := DNSRecord{
- Type: "MX",
- Host: "@",
- Value: domain,
- }
- //g.Log().Info(ctx, "Checking domain:", domain, "emails:", domainEmails)
- status := ValidateMXRecord(record, domain)
- //g.Log().Info(ctx, "ValidateMXRecord result for domain:", domain, "status:", status)
- if status {
- validEmails = append(validEmails, domainEmails...)
- validCount += len(domainEmails)
- appendLogLine(logDir, fmt.Sprintf("%s: %s ----------------------------- √", date, domain))
-
- } else {
- invalidEmails = append(invalidEmails, domainEmails...)
- invalidCount += len(domainEmails)
- appendLogLine(logDir, fmt.Sprintf("%s: %s ----------------------------- x mx logging error", date, domain))
- }
+ wg.Add(1)
+ go func(domain string, domainEmails []string) {
+ defer wg.Done()
+ semaphore <- struct{}{} // Acquire a token
+ defer func() { <-semaphore }() // Release the token
+
+ date := time.Now().Format("2006-01-02 15:04:05")
+ record := DNSRecord{
+ Type: "MX",
+ Host: "@",
+ Value: domain,
+ }
+ status, reason := ValidateMXRecord(record, domain)
+
+ mutex.Lock()
+ defer mutex.Unlock()
+
+ if status {
+ validEmails = append(validEmails, domainEmails...)
+ validCount += len(domainEmails)
+ appendLogLine(logDir, fmt.Sprintf("%s: %s ----------------------------- √", date, domain))
+ } else {
+ invalidEmails = append(invalidEmails, domainEmails...)
+ for _, email := range domainEmails {
+ invalidEmailDetails = append(invalidEmailDetails, abnormal_recipient.RecipientDetail{
+ Email: email,
+ ErrorReason: reason,
+ })
+ }
+ invalidCount += len(domainEmails)
+ appendLogLine(logDir, fmt.Sprintf("%s: %s ----------------------------- x %s", date, domain, reason))
+ }
+ }(domain, domainEmails)
}
+ wg.Wait() // Wait for all goroutines to finish
+
switch req.Oper {
case 2: // Add abnormal table
- _ = abnormal_recipient.BatchUpsertAbnormalRecipients(ctx, invalidEmails, 3, "Manual scanning group")
- appendLogLine(logDir, "Adding invalid emails to blocklist...")
+ _ = abnormal_recipient.BatchUpsertAbnormalRecipientsWithDetails(ctx, invalidEmailDetails, 3, "Manual scanning group (MX check)")
+ appendLogLine(logDir, "Adding invalid emails to blocklist with details...")
case 3: // Remove from the group
_, _ = g.DB().Model("bm_contacts").Where("group_id", req.GroupId).WhereIn("email", invalidEmails).Delete()
appendLogLine(logDir, "Removing invalid emails from group...")
@@ -138,10 +163,10 @@ func (c *ControllerV1) CheckGroup(ctx context.Context, req *v1.CheckGroupReq) (r
return res, nil
}
-func ValidateMXRecord(record DNSRecord, domain string, aRecordHosts ...string) bool {
+func ValidateMXRecord(record DNSRecord, domain string, aRecordHosts ...string) (bool, string) {
g.Log().Info(context.Background(), "ValidateMXRecord called with:", record, domain)
if strings.ToUpper(record.Type) != "MX" {
- return false
+ return false, "Invalid record type"
}
if record.Host != "@" {
@@ -163,18 +188,16 @@ func ValidateMXRecord(record DNSRecord, domain string, aRecordHosts ...string) b
if err != nil {
g.Log().Error(context.Background(), "Failed to query MX records after retries:", err)
- return false
+ return false, "DNS lookup failed: " + err.Error()
}
g.Log().Debug(context.Background(), "Query MX records success", mxRecords)
- aRecordHosts = append([]string{record.Value}, aRecordHosts...)
-
if len(mxRecords) > 0 {
- return true
+ return true, ""
}
- return false
+ return false, "no MX record"
}
func appendLogLine(filePath, line string) {
diff --git a/core/internal/service/abnormal_recipient/abnormal_recipient.go b/core/internal/service/abnormal_recipient/abnormal_recipient.go
index f58f6ff7..0916d8e7 100644
--- a/core/internal/service/abnormal_recipient/abnormal_recipient.go
+++ b/core/internal/service/abnormal_recipient/abnormal_recipient.go
@@ -140,6 +140,76 @@ func BatchUpsertAbnormalRecipients(ctx context.Context, recipients []string, add
return nil
}
+// BatchUpsertAbnormalRecipientsWithDetails
+func BatchUpsertAbnormalRecipientsWithDetails(ctx context.Context, recipientDetails []RecipientDetail, addType int, baseDescription string) error {
+ now := time.Now().Unix()
+
+ if len(recipientDetails) == 0 {
+ return nil
+ }
+
+ recipients := make([]string, len(recipientDetails))
+ detailsMap := make(map[string]RecipientDetail)
+ for i, detail := range recipientDetails {
+ recipients[i] = detail.Email
+ detailsMap[detail.Email] = detail
+ }
+
+ var existList []entity.AbnormalRecipient
+ err := g.DB().Model("abnormal_recipient").WhereIn("recipient", recipients).Scan(&existList)
+ if err != nil {
+ return fmt.Errorf("Failed to query existing abnormal recipients: %w", err)
+ }
+ existMap := make(map[string]*entity.AbnormalRecipient)
+ for _, r := range existList {
+ existMap[r.Recipient] = &r
+ }
+
+ // 1. Update the existing records
+ for _, r := range existList {
+ detail := detailsMap[r.Recipient]
+ description := fmt.Sprintf("%s - %s", baseDescription, detail.ErrorReason)
+
+ _, err := g.DB().Model("abnormal_recipient").Where("id", r.Id).Data(g.Map{
+ "count": r.Count + 1,
+ "description": description,
+ "add_type": addType,
+ }).Update()
+ if err != nil {
+ return fmt.Errorf("Failed to update abnormal recipient: %w", err)
+ }
+ }
+
+ // 2. Inserting a non-existent record
+ var insertList []g.Map
+ for _, detail := range recipientDetails {
+ if _, ok := existMap[detail.Email]; !ok {
+ description := fmt.Sprintf("%s - %s", baseDescription, detail.ErrorReason)
+ insertList = append(insertList, g.Map{
+ "recipient": detail.Email,
+ "count": 1,
+ "add_type": addType,
+ "description": description,
+ "create_time": now,
+ })
+ }
+ }
+
+ if len(insertList) > 0 {
+ _, err := g.DB().Model("abnormal_recipient").Data(insertList).InsertIgnore()
+ if err != nil {
+ return fmt.Errorf("Failed to insert abnormal recipients: %w", err)
+ }
+ }
+
+ return nil
+}
+
+type RecipientDetail struct {
+ Email string `json:"email"`
+ ErrorReason string `json:"error_reason"`
+}
+
func AbnormalRecipientAutoStat(ctx context.Context) {
var abnormalSwitch string