BillionMail gives you open-source MailServer, NewsLetter, Email Marketing — fully self-hosted, dev-friendly, and free from monthly fees. Join the discord: https://discord.gg/asfXzBUhZr
Abnormal recipient scan
lotk6ec60453f426
diff --git a/core/api/abnormal_recipient/abnormal_recipient.go b/core/api/abnormal_recipient/abnormal_recipient.goindex b42e6666..0ad8d296 100644--- a/core/api/abnormal_recipient/abnormal_recipient.go+++ b/core/api/abnormal_recipient/abnormal_recipient.go@@ -14,4 +14,9 @@ type IAbnormalRecipientV1 interface {ListAbnormalRecipient(ctx context.Context, req *v1.ListAbnormalRecipientReq) (res *v1.ListAbnormalRecipientRes, err error)AddAbnormalRecipient(ctx context.Context, req *v1.AddAbnormalRecipientReq) (res *v1.AddAbnormalRecipientRes, err error)DeleteAbnormalRecipient(ctx context.Context, req *v1.DeleteAbnormalRecipientReq) (res *v1.DeleteAbnormalRecipientRes, err error)+ CheckGroup(ctx context.Context, req *v1.CheckGroupReq) (res *v1.CheckGroupRes, err error)+ AbnormalSwitch(ctx context.Context, req *v1.AbnormalSwitchReq) (res *v1.AbnormalSwitchRes, err error)+ SetAbnormalSwitch(ctx context.Context, req *v1.SetAbnormalSwitchReq) (res *v1.SetAbnormalSwitchRes, err error)+ ClearabnormalRecipient(ctx context.Context, req *v1.ClearabnormalRecipientReq) (res *v1.ClearabnormalRecipientRes, err error)+ GetScanLog(ctx context.Context, req *v1.GetScanLogReq) (res *v1.GetScanLogRes, err error)}diff --git a/core/api/abnormal_recipient/v1/abnormal_recipient.go b/core/api/abnormal_recipient/v1/abnormal_recipient.goindex 6b910b26..01a06f63 100644--- a/core/api/abnormal_recipient/v1/abnormal_recipient.go+++ b/core/api/abnormal_recipient/v1/abnormal_recipient.go@@ -51,3 +51,51 @@ type DeleteAbnormalRecipientReq struct {type DeleteAbnormalRecipientRes struct {api_v1.StandardRes}++type CheckGroupReq struct {+ g.Meta `path:"/abnormal_recipient/check_group" method:"post" tags:"Abnormal Recipient" summary:"Check group emails and handle abnormal"`+ Authorization string `json:"authorization" dc:"Authorization" in:"header"`+ GroupId int `json:"group_id" v:"required|min:1" dc:"Contact Group ID"`+ Oper int `json:"oper" v:"required|in:1,2,3" dc:"Operation type (1: only check, 2: add to exception table, 3: delete within group)"`+}++type CheckGroupRes struct {+ api_v1.StandardRes+}++type AbnormalSwitchReq struct {+ g.Meta `path:"/abnormal_recipient/check_switch" method:"get" tags:"Abnormal Recipient" summary:"Abnormal Switch"`+ Authorization string `json:"authorization" dc:"Authorization" in:"header"`+}++type AbnormalSwitchRes struct {+ api_v1.StandardRes+}++type SetAbnormalSwitchReq struct {+ g.Meta `path:"/abnormal_recipient/set_check_switch" method:"post" tags:"Abnormal Recipient" summary:"Abnormal Switch"`+ Authorization string `json:"authorization" dc:"Authorization" in:"header"`+ Oper int `json:"oper" v:"required|in:0,1" dc:"Operation type (1 for on, 0 for off)"`+}++type SetAbnormalSwitchRes struct {+ api_v1.StandardRes+}++type ClearabnormalRecipientReq struct {+ g.Meta `path:"/abnormal_recipient/clear_abnormal" method:"post" tags:"Abnormal Recipient" summary:"Check group emails and handle abnormal"`+ Authorization string `json:"authorization" dc:"Authorization" in:"header"`+}++type ClearabnormalRecipientRes struct {+ api_v1.StandardRes+}++type GetScanLogReq struct {+ g.Meta `path:"/abnormal_recipient/get_scan_log" method:"post" tags:"Abnormal Recipient" summary:"Get Scan Log"`+ Authorization string `json:"authorization" dc:"Authorization" in:"header"`+}++type GetScanLogRes struct {+ api_v1.StandardRes+}diff --git a/core/internal/controller/abnormal_recipient/abnormal_recipient.go b/core/internal/controller/abnormal_recipient/abnormal_recipient.goindex b299fc15..2e462b11 100644--- a/core/internal/controller/abnormal_recipient/abnormal_recipient.go+++ b/core/internal/controller/abnormal_recipient/abnormal_recipient.go@@ -3,3 +3,33 @@// =================================================================================package abnormal_recipient++import "github.com/gogf/gf/v2/frame/g"++func GetAbnormalSwitch() string {+ var abnormalSwitch string+ val, err := g.DB().Model("bm_options").Where("name", "abnormal_mail_check_switch").Value("value")+ if err == nil && val != nil && val.String() != "" {+ abnormalSwitch = val.String()+ } else {+ abnormalSwitch = "1"+ _ = SetAbnormalSwitch("1")++ }+ return abnormalSwitch+}++func SetAbnormalSwitch(status string) error {+ _, err := g.DB().Model("bm_options").+ Data(g.Map{+ "name": "abnormal_mail_check_switch",+ "value": status,+ }).+ OnConflict("name").+ Save()++ if err != nil {+ return err+ }+ return nil+}diff --git a/core/internal/controller/abnormal_recipient/abnormal_recipient_v1_abnormal_switch.go b/core/internal/controller/abnormal_recipient/abnormal_recipient_v1_abnormal_switch.gonew file mode 100644index 00000000..d2212c06--- /dev/null+++ b/core/internal/controller/abnormal_recipient/abnormal_recipient_v1_abnormal_switch.go@@ -0,0 +1,18 @@+package abnormal_recipient++import (+ "billionmail-core/api/abnormal_recipient/v1"+ "billionmail-core/internal/service/public"+ "context"+ "strconv"+)++func (c *ControllerV1) AbnormalSwitch(ctx context.Context, req *v1.AbnormalSwitchReq) (res *v1.AbnormalSwitchRes, err error) {+ res = &v1.AbnormalSwitchRes{}++ abnormalSwitch := GetAbnormalSwitch()++ res.Data, _ = strconv.Atoi(abnormalSwitch)+ res.SetSuccess(public.LangCtx(ctx, "Success"))+ return+}diff --git a/core/internal/controller/abnormal_recipient/abnormal_recipient_v1_check_group.go b/core/internal/controller/abnormal_recipient/abnormal_recipient_v1_check_group.gonew file mode 100644index 00000000..3964df11--- /dev/null+++ b/core/internal/controller/abnormal_recipient/abnormal_recipient_v1_check_group.go@@ -0,0 +1,193 @@+package abnormal_recipient++import (+ "billionmail-core/internal/consts"+ "billionmail-core/internal/model/entity"+ "billionmail-core/internal/service/abnormal_recipient"+ "billionmail-core/internal/service/public"+ "context"+ "fmt"+ "github.com/gogf/gf/os/gtimer"+ "github.com/gogf/gf/v2/errors/gerror"+ "github.com/gogf/gf/v2/frame/g"+ "net"+ "os"+ "strings"+ "time"++ "billionmail-core/api/abnormal_recipient/v1"+)++type DNSRecord struct {+ Type string `json:"type"`+ Host string `json:"host"`+ Value string `json:"value"`+ Valid bool `json:"valid"`+}++func (c *ControllerV1) CheckGroup(ctx context.Context, req *v1.CheckGroupReq) (res *v1.CheckGroupRes, err error) {+ g.Log().Info(ctx, "CheckGroup called with req:", req)+ res = &v1.CheckGroupRes{}+ groupNameValue, err := g.DB().Model("bm_contact_groups").Ctx(ctx).Where("id", req.GroupId).Fields("name").Value()+ if err != nil {+ g.Log().Error(ctx, "Group name query error:", err)+ res.SetError(gerror.New(public.LangCtx(ctx, "The group does not exist.")))+ return+ }++ var contacts []entity.Contact+ err = g.DB().Model("bm_contacts").Where("group_id", req.GroupId).Where("active", 1).Scan(&contacts)+ //g.Log().Info(ctx, "bm_contacts query result, contacts:", contacts, "err:", err)+ if err != nil || len(contacts) == 0 {+ g.Log().Error(ctx, "No contacts found or db error:", err)+ res.SetError(gerror.New(public.LangCtx(ctx, "The contact person within the group is unavailable.")))+ return+ }++ var logDir = public.HostWorkDir + "/logs/core/check_email_valid.txt"++ gtimer.AddOnce(1000*time.Millisecond, func() {++ f, err := os.OpenFile(logDir, os.O_TRUNC|os.O_CREATE|os.O_WRONLY, 0644)+ if err != nil {+ g.Log().Error(ctx, "Failed to truncate log file:", err)+ } else {+ f.Close()+ }++ groupNameInfo := groupNameValue.String()+ emails := make([]string, 0, len(contacts))+ for _, c := range contacts {+ emails = append(emails, c.Email)+ }++ domainMap := make(map[string][]string)+ for _, email := range emails {+ parts := strings.SplitN(email, "@", 2)+ if len(parts) == 2 {+ domainMap[parts[1]] = append(domainMap[parts[1]], email)+ }+ }++ g.Log().Info(ctx, "Start async check group, groupNameInfo:", groupNameInfo)+ nowStr := time.Now().Format("2006-01-02 15:04:05")+ groupName := fmt.Sprintf("Group[%s]", groupNameInfo)+ appendLogLine(logDir, fmt.Sprintf("%s: %s start checking...", nowStr, groupName))+ appendLogLine(logDir, fmt.Sprintf("Total emails to check: %d", len(emails)))+ appendLogLine(logDir, fmt.Sprintf("Total domains to check: %d", len(domainMap)))++ var (+ validEmails []string+ invalidEmails []string+ totalCount = len(emails)+ validCount = 0+ invalidCount = 0+ )++ for domain, domainEmails := range domainMap {+ date := time.Now().Format("2006-01-02 15:04:05")+ record := DNSRecord{+ Type: "MX",+ Host: "@",+ Value: domain,+ }+ g.Log().Info(ctx, "Checking domain:", domain, "emails:", domainEmails)+ status := ValidateMXRecord(record, domain)+ g.Log().Info(ctx, "ValidateMXRecord result for domain:", domain, "status:", status)+ if status {+ validEmails = append(validEmails, domainEmails...)+ validCount += len(domainEmails)+ appendLogLine(logDir, fmt.Sprintf("%s: %s ----------------------------- √", date, domain))++ } else {+ invalidEmails = append(invalidEmails, domainEmails...)+ invalidCount += len(domainEmails)+ appendLogLine(logDir, fmt.Sprintf("%s: %s ----------------------------- x mx logging error", date, domain))+ }+ }++ switch req.Oper {+ case 2: // Add abnormal table+ _ = abnormal_recipient.BatchUpsertAbnormalRecipients(ctx, invalidEmails, 3, "Manual scanning group exception")+ appendLogLine(logDir, "Adding invalid emails to blocklist...")+ case 3: // Remove from the group+ _, _ = g.DB().Model("bm_contacts").Where("group_id", req.GroupId).WhereIn("email", invalidEmails).Delete()+ appendLogLine(logDir, "Removing invalid emails from group...")+ }+ nowStr = time.Now().Format("2006-01-02 15:04:05")+ appendLogLine(logDir, "---------------------------------------------------------------------------------------")+ appendLogLine(logDir, fmt.Sprintf("Results for [%s]:", groupName))+ appendLogLine(logDir, fmt.Sprintf("Tested: %d", totalCount))+ appendLogLine(logDir, fmt.Sprintf("Normal: %d", validCount))+ appendLogLine(logDir, fmt.Sprintf("Abnormal: %d", invalidCount))+ appendLogLine(logDir, fmt.Sprintf("Abnormal emails: %s", strings.Join(invalidEmails, ", ")))+ appendLogLine(logDir, "---------------------------------------------------------------------------------------")+ appendLogLine(logDir, fmt.Sprintf("%s: Check finished", nowStr))+ })++ _ = public.WriteLog(ctx, public.LogParams{+ Type: consts.LOGTYPE.AbnormalRecipient,+ Log: "Manually scan the abnormal email addresses in group :" + groupNameValue.String() + " successfully",+ })++ res.SetSuccess(public.LangCtx(ctx, "The process is currently running in the background. Please check the log."))++ return res, nil+}++func ValidateMXRecord(record DNSRecord, domain string, aRecordHosts ...string) bool {+ g.Log().Info(context.Background(), "ValidateMXRecord called with:", record, domain)+ if strings.ToUpper(record.Type) != "MX" {+ return false+ }++ if record.Host != "@" {+ domain = strings.TrimSuffix(record.Host, ".") + "." + domain+ }++ var mxRecords []*net.MX+ var err error++ for i := 0; i < 3; i++ {+ mxRecords, err = net.LookupMX(domain)+ if err == nil {+ break+ }+ if i < 2 { // 非最后一次,等待后再试+ time.Sleep(500 * time.Millisecond)+ }+ }++ if err != nil {+ g.Log().Error(context.Background(), "Failed to query MX records after retries:", err)+ return false+ }++ g.Log().Debug(context.Background(), "Query MX records success", mxRecords)++ aRecordHosts = append([]string{record.Value}, aRecordHosts...)++ if len(mxRecords) > 0 {+ return true+ }++ return false+}++func appendLogLine(filePath, line string) {+ defer func() {+ if r := recover(); r != nil {+ g.Log().Error(context.Background(), "panic in appendLogLine:", r)+ }+ }()+ f, err := os.OpenFile(filePath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644)+ if err != nil {+ g.Log().Error(context.Background(), "Failed to open log file:", err)+ return+ }+ defer f.Close()+ _, err = f.WriteString(line + "\n")+ if err != nil {+ g.Log().Error(context.Background(), "Failed to write log line:", err)+ }+}diff --git a/core/internal/controller/abnormal_recipient/abnormal_recipient_v1_clearabnormal_recipient.go b/core/internal/controller/abnormal_recipient/abnormal_recipient_v1_clearabnormal_recipient.gonew file mode 100644index 00000000..3e8bde3c--- /dev/null+++ b/core/internal/controller/abnormal_recipient/abnormal_recipient_v1_clearabnormal_recipient.go@@ -0,0 +1,27 @@+package abnormal_recipient++import (+ "billionmail-core/internal/consts"+ "billionmail-core/internal/service/public"+ "context"+ "github.com/gogf/gf/v2/errors/gerror"+ "github.com/gogf/gf/v2/frame/g"++ "billionmail-core/api/abnormal_recipient/v1"+)++func (c *ControllerV1) ClearabnormalRecipient(ctx context.Context, req *v1.ClearabnormalRecipientReq) (res *v1.ClearabnormalRecipientRes, err error) {++ _, err = g.DB().Model("abnormal_recipient").Where("1=1").Delete()+ if err != nil {+ res.SetError(gerror.New(public.LangCtx(ctx, "Clearing failed: {}", err.Error())))+ return+ }+ _ = public.WriteLog(ctx, public.LogParams{+ Type: consts.LOGTYPE.AbnormalRecipient,+ Log: "Clearing the abnormal email box was successful",+ })++ res.SetSuccess(public.LangCtx(ctx, "Success"))+ return+}diff --git a/core/internal/controller/abnormal_recipient/abnormal_recipient_v1_delete_abnormal_recipient.go b/core/internal/controller/abnormal_recipient/abnormal_recipient_v1_delete_abnormal_recipient.goindex bf42c2ab..ad0d5ac6 100644--- a/core/internal/controller/abnormal_recipient/abnormal_recipient_v1_delete_abnormal_recipient.go+++ b/core/internal/controller/abnormal_recipient/abnormal_recipient_v1_delete_abnormal_recipient.go@@ -22,6 +22,7 @@ func (c *ControllerV1) DeleteAbnormalRecipient(ctx context.Context, req *v1.DeleLog: "Delete abnormal recipient :" + abnormalRecipient.Recipient + " successfully",Data: abnormalRecipient,})+res.SetSuccess(public.LangCtx(ctx, "The exception recipient was removed successfully"))return}diff --git a/core/internal/controller/abnormal_recipient/abnormal_recipient_v1_get_scan_log.go b/core/internal/controller/abnormal_recipient/abnormal_recipient_v1_get_scan_log.gonew file mode 100644index 00000000..9e7d2c56--- /dev/null+++ b/core/internal/controller/abnormal_recipient/abnormal_recipient_v1_get_scan_log.go@@ -0,0 +1,26 @@+package abnormal_recipient++import (+ "billionmail-core/api/abnormal_recipient/v1"+ "billionmail-core/internal/service/public"+ "context"+ "github.com/gogf/gf/v2/errors/gerror"+ "os"+)++func (c *ControllerV1) GetScanLog(ctx context.Context, req *v1.GetScanLogReq) (res *v1.GetScanLogRes, err error) {+ var logDir = public.HostWorkDir + "/logs/core/check_email_valid.txt"++ res = &v1.GetScanLogRes{}++ content, err := os.ReadFile(logDir)+ if err != nil {++ res.SetError(gerror.New(public.LangCtx(ctx, "Failed to read log file: : {}", err.Error())))+ return+ }++ res.Data = string(content)+ res.SetSuccess(public.LangCtx(ctx, "Success"))+ return+}diff --git a/core/internal/controller/abnormal_recipient/abnormal_recipient_v1_set_abnormal_switch.go b/core/internal/controller/abnormal_recipient/abnormal_recipient_v1_set_abnormal_switch.gonew file mode 100644index 00000000..49b0056e--- /dev/null+++ b/core/internal/controller/abnormal_recipient/abnormal_recipient_v1_set_abnormal_switch.go@@ -0,0 +1,40 @@+package abnormal_recipient++import (+ "billionmail-core/internal/consts"+ "billionmail-core/internal/service/public"+ "context"+ "github.com/gogf/gf/v2/errors/gerror"+ "strconv"++ "billionmail-core/api/abnormal_recipient/v1"+)++func (c *ControllerV1) SetAbnormalSwitch(ctx context.Context, req *v1.SetAbnormalSwitchReq) (res *v1.SetAbnormalSwitchRes, err error) {+ res = &v1.SetAbnormalSwitchRes{}+ oper := strconv.Itoa(req.Oper)++ err = SetAbnormalSwitch(oper)+ if err != nil {+ res.SetError(gerror.New(public.LangCtx(ctx, "Failed to set the switch: {}", err.Error())))+ return+ }++ action := "Turn on"+ switch req.Oper {+ case 1:+ action = "Turn on"+ case 0:+ action = "Shut down"+ default:+ action = "Illegal operation"+ }++ _ = public.WriteLog(ctx, public.LogParams{+ Type: consts.LOGTYPE.AbnormalRecipient,+ Log: action + " Automatic scanning of abnormal email accounts",+ })++ res.SetSuccess(public.LangCtx(ctx, "Success"))+ return+}diff --git a/core/internal/service/abnormal_recipient/abnormal_recipient.go b/core/internal/service/abnormal_recipient/abnormal_recipient.goindex 225e1e39..dafba59e 100644--- a/core/internal/service/abnormal_recipient/abnormal_recipient.go+++ b/core/internal/service/abnormal_recipient/abnormal_recipient.go@@ -2,6 +2,7 @@ package abnormal_recipientimport ("billionmail-core/internal/model/entity"+ "billionmail-core/internal/service/maillog_stat""context""fmt""github.com/gogf/gf/v2/frame/g"@@ -84,3 +85,121 @@ func GetAbnormalRecipient(ctx context.Context, id int) (*entity.AbnormalRecipien}return &recipient, nil}++func BatchUpsertAbnormalRecipients(ctx context.Context, recipients []string, addType int, description string) error {+ now := time.Now().Unix()++ if len(recipients) == 0 {+ return nil+ }++ var existList []entity.AbnormalRecipient+ err := g.DB().Model("abnormal_recipient").WhereIn("recipient", recipients).Scan(&existList)+ if err != nil {+ return fmt.Errorf("Failed to query existing abnormal recipients: %w", err)+ }+ existMap := make(map[string]*entity.AbnormalRecipient)+ for _, r := range existList {+ existMap[r.Recipient] = &r+ }+ // 1. Update the existing one count+1+ for _, r := range existList {+ _, err := g.DB().Model("abnormal_recipient").Where("id", r.Id).Data(g.Map{+ "count": r.Count + 1,+ "description": description,+ "add_type": addType,+ }).Update()+ if err != nil {++ return fmt.Errorf("Failed to update abnormal recipient: %w", err)+ }++ }+ // 2. Inserting something that doesn't exist+ var insertList []g.Map+ for _, recipient := range recipients {+ if _, ok := existMap[recipient]; !ok {+ insertList = append(insertList, g.Map{+ "recipient": recipient,+ "count": 1,+ "add_type": addType,+ "description": description,+ "create_time": now,+ })+ }+ }+ if len(insertList) > 0 {++ _, err := g.DB().Model("abnormal_recipient").Data(insertList).InsertIgnore()+ if err != nil {++ return fmt.Errorf("Failed to insert abnormal recipients: %w", err)+ }+ }++ return nil+}++func AbnormalRecipientAutoStat(ctx context.Context) {++ var abnormalSwitch string+ val, err := g.DB().Model("bm_options").Where("name", "abnormal_mail_check_switch").Value("value")+ if err == nil && val != nil && val.String() != "" {+ abnormalSwitch = val.String()+ } else {+ abnormalSwitch = "1"+ }+ if abnormalSwitch != "1" {+ return+ }++ lastTime := getLastStatTime(ctx)+ now := time.Now().Unix()++ overview := maillog_stat.NewOverview()+ failedList := overview.FailedList(0, "", lastTime, now)++ recipientSet := make(map[string]struct{})+ for _, item := range failedList {+ if recipient, ok := item["recipient"].(string); ok && recipient != "" {+ recipientSet[recipient] = struct{}{}+ }+ }+ var recipients []string+ for r := range recipientSet {+ recipients = append(recipients, r)+ }++ if len(recipients) > 0 {+ _ = BatchUpsertAbnormalRecipients(ctx, recipients, 2, "Automatic statistics")+ }++ setLastStatTime(ctx, now)++}++func getLastStatTime(ctx context.Context) int64 {+ var lastTime int64+ val, err := g.DB().Model("bm_options").Where("name", "abnormal_recipient_last_time").Value("value")+ if err == nil && val != nil && val.String() != "" {+ lastTime = val.Int64()++ } else {+ lastTime = time.Now().Add(-30 * time.Minute).Unix()++ }+ return lastTime+}++func setLastStatTime(ctx context.Context, t int64) {+ _, err := g.DB().Model("bm_options").+ Data(g.Map{+ "name": "abnormal_recipient_last_time",+ "value": t,+ }).+ OnConflict("name").+ Save()+ if err != nil {+ g.Log().Warningf(ctx, "[abnormal Recipient] Error occurred in the last statistics time recording: %v", err)+ }+}diff --git a/core/internal/service/timers/timers.go b/core/internal/service/timers/timers.goindex 43c7066b..4f84f111 100644--- a/core/internal/service/timers/timers.go+++ b/core/internal/service/timers/timers.go@@ -1,6 +1,7 @@package timersimport (+ "billionmail-core/internal/service/abnormal_recipient""billionmail-core/internal/service/askai""billionmail-core/internal/service/batch_mail""billionmail-core/internal/service/collect"@@ -131,6 +132,10 @@ func Start(ctx context.Context) (err error) {// askai timersgtimer.Add(5*time.Second, askai.AutoGetProjectInfo)+ gtimer.Add(30*time.Minute, func() {+ abnormal_recipient.AbnormalRecipientAutoStat(context.Background())+ })+g.Log().Debug(ctx, "All timers started successfully")return nil}