BillionMail gives you open-source MailServer, NewsLetter, Email Marketing — fully self-hosted, dev-friendly, and free from monthly fees. Join the discord: https://discord.gg/asfXzBUhZr
Marketing task statistics - Query from the backend
lotkab3008feb58a
diff --git a/core/api/batch_mail/v1/batch_mail.go b/core/api/batch_mail/v1/batch_mail.goindex dca0690b..96dd8f33 100644--- a/core/api/batch_mail/v1/batch_mail.go+++ b/core/api/batch_mail/v1/batch_mail.go@@ -29,6 +29,11 @@ type EmailTask struct {Active int `json:"active" dc:"status"`AddType int `json:"add_type" dc:"add type"`EstimatedTimeWithWarmup int64 `json:"estimated_time_with_warmup" dc:"estimated time for warmup (if applicable)"`+ SendsCount int `json:"sendsCount" description:""`+ DeliveredCount int `json:"deliveredCount" description:""`+ BouncedCount int `json:"bouncedCount" description:""`+ DeferredCount int `json:"deferredCount" description:""`+ StatsUpdateTime int `json:"statsUpdateTime" description:""`}type GroupInfo struct {diff --git a/core/internal/controller/batch_mail/batch_mail_v1_list_tasks.go b/core/internal/controller/batch_mail/batch_mail_v1_list_tasks.goindex 4acd1630..e8ca4be3 100644--- a/core/internal/controller/batch_mail/batch_mail_v1_list_tasks.go+++ b/core/internal/controller/batch_mail/batch_mail_v1_list_tasks.go@@ -26,35 +26,18 @@ func (c *ControllerV1) ListTasks(ctx context.Context, req *v1.ListTasksReq) (resEmailTask: *task,}- stats := GetTaskStats(ctx, int64(task.Id))- if stats != nil {-- detail.SentCount = stats["sends"].(int)- detail.SuccessCount = stats["delivered"].(int)- detail.Deferred = stats["deferred"].(int)-- detail.ErrorCount = stats["bounced"].(int) + stats["deferred"].(int)-- //detail.DeliveryRate = stats["delivery_rate"].(float64)- //detail.BounceRate = stats["bounce_rate"].(float64)- //detail.OpenRate = stats["open_rate"].(float64)- //detail.ClickRate = stats["click_rate"].(float64)- //detail.Opened = stats["opened"].(int)- //detail.Clicked = stats["clicked"].(int)- }+ detail.SentCount = task.SendsCount+ detail.SuccessCount = task.DeliveredCount+ detail.Deferred = task.DeferredCount+ detail.ErrorCount = task.BouncedCount + task.DeferredCountsentCount := detail.SentCount- //sentCount, _ := batch_mail.GetSentCount(ctx, task.Id)- //detail.SentCount = sentCount-- // Fix: Check if recipient_count is validif task.RecipientCount <= 0 {actualCount, err := batch_mail.GetActualRecipientCount(ctx, task.Id)if err == nil && actualCount > 0 {- // Update the recipient count in the database_ = batch_mail.UpdateRecipientCount(ctx, task.Id, actualCount)detail.RecipientCount = actualCounttask.RecipientCount = actualCount@@ -71,13 +54,9 @@ func (c *ControllerV1) ListTasks(ctx context.Context, req *v1.ListTasksReq) (restask.Id, task.RecipientCount, sentCount)}- // Calculate progress with safeguards against division by zero- sentp, _ := batch_mail.GetSentCount(ctx, task.Id)-if task.RecipientCount > 0 {- detail.Progress = int(float64(sentp) / float64(task.RecipientCount) * 100)- // Ensure progress is between 0 and 100- if detail.Progress > 100 {+ detail.Progress = int(float64(sentCount) / float64(task.RecipientCount) * 100)+ if detail.Progress > 100 || detail.Progress == 99 {detail.Progress = 100}} else {@@ -85,17 +64,6 @@ func (c *ControllerV1) ListTasks(ctx context.Context, req *v1.ListTasksReq) (resdetail.Progress = 0}- // Get success and error counts- //successCount, errorCount, err := batch_mail.GetTaskSendingStats(ctx, task.Id)- //if err == nil {- // detail.SuccessCount = successCount- // detail.ErrorCount = errorCount- //} else {- // detail.SuccessCount = 0- // detail.ErrorCount = 0- // g.Log().Error(ctx, "Failed to get sending statistics: ", err)- //}-task_list[i] = detail}@@ -104,67 +72,3 @@ func (c *ControllerV1) ListTasks(ctx context.Context, req *v1.ListTasksReq) (resres.SetSuccess(public.LangCtx(ctx, "Task list retrieved successfully"))return}--func GetTaskStats(ctx context.Context, taskId int64) map[string]interface{} {- query := g.DB().Model("mailstat_send_mails sm")-- query = query.LeftJoin("mailstat_message_ids mi", "sm.postfix_message_id=mi.postfix_message_id")- query = query.LeftJoin("recipient_info ri", "mi.message_id=ri.message_id")-- query = query.Where("ri.task_id = ?", taskId)-- query.Fields(- "count(*) as sends",- "coalesce(sum(case when sm.status='sent' and sm.dsn like '2.%' then 1 else 0 end), 0) as delivered", // success count- "coalesce(sum(case when sm.status='bounced' then 1 else 0 end), 0) as bounced", // bounced count- "coalesce(sum(case when sm.status='deferred' then 1 else 0 end), 0) as deferred", // deferred- )-- result, err := query.One()- if err != nil {- g.Log().Error(ctx, err)- return nil- }-- sends := result["sends"].Int()- delivered := result["delivered"].Int()- bounced := result["bounced"].Int()- deferred := result["deferred"].Int()- //if deferred != 0 {- // g.Log().Warning(ctx, " task {} deferred: {}", taskId, deferred)- //}-- //// 通过campaign_id查打开和点击- //campaignId := int(taskId)- //openedCount, _ := g.DB().Model("mailstat_opened").- // Where("campaign_id", campaignId).- // Fields("count(distinct postfix_message_id) as opened").- // Value()- //clickedCount, _ := g.DB().Model("mailstat_clicked").- // Where("campaign_id", campaignId).- // Fields("count(distinct postfix_message_id) as clicked").- // Value()-- stats := map[string]interface{}{- "sends": sends,- "delivered": delivered,- "deferred": deferred,- //"opened": openedCount.Int(),- //"clicked": clickedCount.Int(),- "bounced": bounced,- }-- if sends > 0 {- stats["delivery_rate"] = public.Round(float64(delivered)/float64(sends)*100, 2)- stats["bounce_rate"] = public.Round(float64(bounced)/float64(sends)*100, 2)- //stats["open_rate"] = public.Round(float64(stats["opened"].(int))/float64(sends)*100, 2)- //stats["click_rate"] = public.Round(float64(stats["clicked"].(int))/float64(sends)*100, 2)- } else {- stats["delivery_rate"] = 0.0- stats["bounce_rate"] = 0.0- //stats["open_rate"] = 0.0- //stats["click_rate"] = 0.0- }-- return stats-}diff --git a/core/internal/model/entity/batch_mail.go b/core/internal/model/entity/batch_mail.goindex 83c14b96..289d18b3 100644--- a/core/internal/model/entity/batch_mail.go+++ b/core/internal/model/entity/batch_mail.go@@ -49,26 +49,32 @@ type EmailTemplate struct {// EmailTask Entitytype EmailTask struct {- Id int `json:"id" dc:"Task ID"`- TaskName string `json:"task_name" dc:"Task Name"`- Addresser string `json:"addresser" dc:"Sender"`- Subject string `json:"subject" dc:"Subject"`- FullName string `json:"full_name" dc:"Sender Name"`- RecipientCount int `json:"recipient_count" dc:"Recipient Count"`- TaskProcess int `json:"task_process" dc:"Task Status"`- Pause int `json:"pause" dc:"Pause Status"`- TemplateId int `json:"template_id" dc:"Template ID"`- IsRecord int `json:"is_record" dc:"Record to Outbox"`- Unsubscribe int `json:"unsubscribe" dc:"Allow Unsubscribe"`- Threads int `json:"threads" dc:"Thread Count"`- Etypes string `json:"etypes" dc:"Contact Group IDs"`- TrackOpen int `json:"track_open" dc:"Track Opens"`- TrackClick int `json:"track_click" dc:"Track Clicks"`- StartTime int `json:"start_time" dc:"Start Time"`- CreateTime int `json:"create_time" dc:"Create Time"`- UpdateTime int `json:"update_time" dc:"Update Time"`- Remark string `json:"remark" dc:"Remark"`- Active int `json:"active" dc:"Status"`+ Id int `json:"id" dc:"Task ID"`+ TaskName string `json:"task_name" dc:"Task Name"`+ Addresser string `json:"addresser" dc:"Sender"`+ Subject string `json:"subject" dc:"Subject"`+ FullName string `json:"full_name" dc:"Sender Name"`+ RecipientCount int `json:"recipient_count" dc:"Recipient Count"`+ TaskProcess int `json:"task_process" dc:"Task Status"`+ Pause int `json:"pause" dc:"Pause Status"`+ TemplateId int `json:"template_id" dc:"Template ID"`+ IsRecord int `json:"is_record" dc:"Record to Outbox"`+ Unsubscribe int `json:"unsubscribe" dc:"Allow Unsubscribe"`+ Threads int `json:"threads" dc:"Thread Count"`+ Etypes string `json:"etypes" dc:"Contact Group IDs"`+ TrackOpen int `json:"track_open" dc:"Track Opens"`+ TrackClick int `json:"track_click" dc:"Track Clicks"`+ StartTime int `json:"start_time" dc:"Start Time"`+ CreateTime int `json:"create_time" dc:"Create Time"`+ UpdateTime int `json:"update_time" dc:"Update Time"`+ Remark string `json:"remark" dc:"Remark"`+ Active int `json:"active" dc:"Status"`+ AddType int `json:"addType" description:""`+ SendsCount int `json:"sendsCount" description:""`+ DeliveredCount int `json:"deliveredCount" description:""`+ BouncedCount int `json:"bouncedCount" description:""`+ DeferredCount int `json:"deferredCount" description:""`+ StatsUpdateTime int `json:"statsUpdateTime" description:""`}// RecipientInfo Entitydiff --git a/core/internal/service/batch_mail/batch_mail.go b/core/internal/service/batch_mail/batch_mail.goindex 9d1669ec..493a702d 100644--- a/core/internal/service/batch_mail/batch_mail.go+++ b/core/internal/service/batch_mail/batch_mail.go@@ -3,6 +3,7 @@ package batch_mailimport ("billionmail-core/api/batch_mail/v1""billionmail-core/internal/model/entity"+ "billionmail-core/internal/service/maillog_stat""billionmail-core/internal/service/public""billionmail-core/internal/service/warmup""context"@@ -12,9 +13,14 @@ import ("github.com/gogf/gf/v2/frame/g""github.com/gogf/gf/v2/util/gconv""strings"+ "sync/atomic""time")+var (+ updateTaskStatsRunning int32 // 0: not running, 1: running+)+type CreateTaskArgs struct {Addresser string `json:"addresser"` // Sender email addressSubject string `json:"subject"` // Email subject@@ -492,3 +498,89 @@ func GetActualRecipientCount(ctx context.Context, taskId int) (int, error) {return count, nil}++// Regularly update the linked query data of marketing tasks+func UpdateTaskJoinMailstat(ctx context.Context) {++ if !atomic.CompareAndSwapInt32(&updateTaskStatsRunning, 0, 1) {+ g.Log().Info(ctx, "UpdateTaskJoinMailstat is already running, skipping this run.")+ return+ }+ defer atomic.StoreInt32(&updateTaskStatsRunning, 0)++ // Search for tasks that need to be updated+ // 1. Tasks that are currently in progress or have been paused (task_process = 1 or 3)+ // 2. Tasks that have been completed but have sent fewer messages than the number of recipients (task_process = 2 and sends_count < recipient_count)+ // 3. Tasks that have never been updated (stats_update_time = 0)+ var tasksToUpdate []*entity.EmailTask+ err := g.DB().Model("email_tasks").+ Where("task_process IN (?)", []int{1, 3}).+ WhereOr("task_process = 2 AND sends_count < recipient_count*0.99").+ WhereOr("stats_update_time", 0).+ Order("id DESC").+ Scan(&tasksToUpdate)+ if err != nil {+ g.Log().Errorf(ctx, "UpdateTaskJoinMailstat: failed to get tasks to update: %v", err)+ return+ }++ if len(tasksToUpdate) == 0 {+ g.Log().Debug(ctx, "UpdateTaskJoinMailstat: no tasks to update.")+ return+ }++ g.Log().Infof(ctx, "UpdateTaskJoinMailstat: found %d tasks to update.", len(tasksToUpdate))++ for _, task := range tasksToUpdate {+ //g.Log().Warningf(ctx, "start %d: %s", task.Id, task.TaskName)++ stats := getSingleTaskStats(ctx, int64(task.Id))+ if stats == nil {+ //g.Log().Warningf(ctx, "UpdateTaskJoinMailstat: could not get stats for task %d, skipping.", task.Id)+ continue+ }++ _, err := g.DB().Model("email_tasks").+ Where("id", task.Id).+ Data(g.Map{+ "sends_count": stats["sends"],+ "delivered_count": stats["delivered"],+ "bounced_count": stats["bounced"],+ "deferred_count": stats["deferred"],+ "stats_update_time": time.Now().Unix(),+ }).+ Update()++ if err != nil {+ g.Log().Errorf(ctx, "UpdateTaskJoinMailstat: failed to update stats for task %d: %v", task.Id, err)++ } else {+ g.Log().Debugf(ctx, "UpdateTaskJoinMailstat: successfully updated stats for task %d.", task.Id)+ }+ }++ return+}++// getSingleTaskStats Obtain the statistical data for a single task+func getSingleTaskStats(ctx context.Context, taskId int64) map[string]interface{} {++ overview := maillog_stat.NewOverview()+ stats := overview.OverviewDashboard(taskId, "", 0, 0)++ if stats == nil {+ g.Log().Errorf(ctx, "getSingleTaskStats: OverviewDashboard returned nil for task %d", taskId)+ return nil+ }++ requiredKeys := []string{"sends", "delivered", "bounced", "deferred"}+ for _, key := range requiredKeys {+ if _, ok := stats[key]; !ok {+ //g.Log().Warningf(ctx, "getSingleTaskStats: 任务 %d is missing key '%s'", taskId, key)++ stats[key] = 0+ }+ }++ return stats+}diff --git a/core/internal/service/database_initialization/batch_mail.go b/core/internal/service/database_initialization/batch_mail.goindex 317f10fd..70ec5370 100644--- a/core/internal/service/database_initialization/batch_mail.go+++ b/core/internal/service/database_initialization/batch_mail.go@@ -79,7 +79,12 @@ func init() {create_time INTEGER NOT NULL DEFAULT EXTRACT(EPOCH FROM NOW()),update_time INTEGER NOT NULL DEFAULT EXTRACT(EPOCH FROM NOW()),active SMALLINT NOT NULL DEFAULT 0,- add_type SMALLINT NOT NULL DEFAULT 0,+ add_type SMALLINT NOT NULL DEFAULT 0,+ sends_count INTEGER NOT NULL DEFAULT 0,+ delivered_count INTEGER NOT NULL DEFAULT 0,+ bounced_count INTEGER NOT NULL DEFAULT 0,+ deferred_count INTEGER NOT NULL DEFAULT 0,+ stats_update_time INTEGER NOT NULL DEFAULT 0,FOREIGN KEY (template_id) REFERENCES email_templates(id))`,@@ -168,6 +173,7 @@ func init() {`CREATE INDEX IF NOT EXISTS idx_recipient_info_is_sent ON recipient_info(is_sent)`,`CREATE INDEX IF NOT EXISTS idx_recipient_info_message_id ON recipient_info(message_id)`,`CREATE INDEX IF NOT EXISTS idx_recipient_info_task_sent ON recipient_info(task_id, is_sent)`,+ `CREATE INDEX IF NOT EXISTS idx_email_tasks_task_process ON email_tasks(task_process)`,}for _, sql := range batchMailSQLList {@@ -205,6 +211,13 @@ func init() {//email_templates_ = AddColumnIfNotExists("email_templates", "chat_id", "VARCHAR(255)", "''", false)+ // email_tasks+ _ = AddColumnIfNotExists("email_tasks", "sends_count", "INTEGER", "0", true)+ _ = AddColumnIfNotExists("email_tasks", "delivered_count", "INTEGER", "0", true)+ _ = AddColumnIfNotExists("email_tasks", "bounced_count", "INTEGER", "0", true)+ _ = AddColumnIfNotExists("email_tasks", "deferred_count", "INTEGER", "0", true)+ _ = AddColumnIfNotExists("email_tasks", "stats_update_time", "INTEGER", "0", true)+// unsubscribe_records_ = DropForeignKeyIfExists("unsubscribe_records", "group_id")_ = DropForeignKeyIfExists("unsubscribe_records", "template_id")diff --git a/core/internal/service/maillog_stat/overview.go b/core/internal/service/maillog_stat/overview.goindex 220753ac..d4d913db 100644--- a/core/internal/service/maillog_stat/overview.go+++ b/core/internal/service/maillog_stat/overview.go@@ -145,6 +145,12 @@ func (o *Overview) overviewDashboard(campaignID int64, domain string, startTime,return aggregate}+// overviewDashboard dashboard data+func (o *Overview) OverviewDashboard(campaignID int64, domain string, startTime, endTime int64) map[string]interface{} {+ aggregate := o.overviewDashboard(campaignID, domain, startTime, endTime)+ return aggregate+}+// Obtain the number of deferred queues with a 1-minute cachefunc (o *Overview) getPostfixDeferredQueueCount(ctx context.Context) (int, error) {cache := gcache.New()diff --git a/core/internal/service/timers/timers.go b/core/internal/service/timers/timers.goindex 4f84f111..855de4aa 100644--- a/core/internal/service/timers/timers.go+++ b/core/internal/service/timers/timers.go@@ -136,6 +136,11 @@ func Start(ctx context.Context) (err error) {abnormal_recipient.AbnormalRecipientAutoStat(context.Background())})+ // Regularly update the success and failure data of marketing tasks+ gtimer.Add(1*time.Minute, func() {+ batch_mail.UpdateTaskJoinMailstat(ctx)+ })+g.Log().Debug(ctx, "All timers started successfully")return nil}