Skip to content

BillionMail gives you open-source MailServer, NewsLetter, Email Marketing — fully self-hosted, dev-friendly, and free from monthly fees. Join the discord: https://discord.gg/asfXzBUhZr

Marketing task statistics - Query from the backend

lotkab3008feb58a
diff --git a/core/api/batch_mail/v1/batch_mail.go b/core/api/batch_mail/v1/batch_mail.go
index dca0690b..96dd8f33 100644
--- a/core/api/batch_mail/v1/batch_mail.go
+++ b/core/api/batch_mail/v1/batch_mail.go
@@ -29,6 +29,11 @@ type EmailTask struct {
Active int `json:"active" dc:"status"`
AddType int `json:"add_type" dc:"add type"`
EstimatedTimeWithWarmup int64 `json:"estimated_time_with_warmup" dc:"estimated time for warmup (if applicable)"`
+ SendsCount int `json:"sendsCount" description:""`
+ DeliveredCount int `json:"deliveredCount" description:""`
+ BouncedCount int `json:"bouncedCount" description:""`
+ DeferredCount int `json:"deferredCount" description:""`
+ StatsUpdateTime int `json:"statsUpdateTime" description:""`
}
type GroupInfo struct {
diff --git a/core/internal/controller/batch_mail/batch_mail_v1_list_tasks.go b/core/internal/controller/batch_mail/batch_mail_v1_list_tasks.go
index 4acd1630..e8ca4be3 100644
--- a/core/internal/controller/batch_mail/batch_mail_v1_list_tasks.go
+++ b/core/internal/controller/batch_mail/batch_mail_v1_list_tasks.go
@@ -26,35 +26,18 @@ func (c *ControllerV1) ListTasks(ctx context.Context, req *v1.ListTasksReq) (res
EmailTask: *task,
}
- stats := GetTaskStats(ctx, int64(task.Id))
- if stats != nil {
-
- detail.SentCount = stats["sends"].(int)
- detail.SuccessCount = stats["delivered"].(int)
- detail.Deferred = stats["deferred"].(int)
-
- detail.ErrorCount = stats["bounced"].(int) + stats["deferred"].(int)
-
- //detail.DeliveryRate = stats["delivery_rate"].(float64)
- //detail.BounceRate = stats["bounce_rate"].(float64)
- //detail.OpenRate = stats["open_rate"].(float64)
- //detail.ClickRate = stats["click_rate"].(float64)
- //detail.Opened = stats["opened"].(int)
- //detail.Clicked = stats["clicked"].(int)
- }
+ detail.SentCount = task.SendsCount
+ detail.SuccessCount = task.DeliveredCount
+ detail.Deferred = task.DeferredCount
+ detail.ErrorCount = task.BouncedCount + task.DeferredCount
sentCount := detail.SentCount
- //sentCount, _ := batch_mail.GetSentCount(ctx, task.Id)
- //detail.SentCount = sentCount
-
- // Fix: Check if recipient_count is valid
if task.RecipientCount <= 0 {
actualCount, err := batch_mail.GetActualRecipientCount(ctx, task.Id)
if err == nil && actualCount > 0 {
- // Update the recipient count in the database
_ = batch_mail.UpdateRecipientCount(ctx, task.Id, actualCount)
detail.RecipientCount = actualCount
task.RecipientCount = actualCount
@@ -71,13 +54,9 @@ func (c *ControllerV1) ListTasks(ctx context.Context, req *v1.ListTasksReq) (res
task.Id, task.RecipientCount, sentCount)
}
- // Calculate progress with safeguards against division by zero
- sentp, _ := batch_mail.GetSentCount(ctx, task.Id)
-
if task.RecipientCount > 0 {
- detail.Progress = int(float64(sentp) / float64(task.RecipientCount) * 100)
- // Ensure progress is between 0 and 100
- if detail.Progress > 100 {
+ detail.Progress = int(float64(sentCount) / float64(task.RecipientCount) * 100)
+ if detail.Progress > 100 || detail.Progress == 99 {
detail.Progress = 100
}
} else {
@@ -85,17 +64,6 @@ func (c *ControllerV1) ListTasks(ctx context.Context, req *v1.ListTasksReq) (res
detail.Progress = 0
}
- // Get success and error counts
- //successCount, errorCount, err := batch_mail.GetTaskSendingStats(ctx, task.Id)
- //if err == nil {
- // detail.SuccessCount = successCount
- // detail.ErrorCount = errorCount
- //} else {
- // detail.SuccessCount = 0
- // detail.ErrorCount = 0
- // g.Log().Error(ctx, "Failed to get sending statistics: ", err)
- //}
-
task_list[i] = detail
}
@@ -104,67 +72,3 @@ func (c *ControllerV1) ListTasks(ctx context.Context, req *v1.ListTasksReq) (res
res.SetSuccess(public.LangCtx(ctx, "Task list retrieved successfully"))
return
}
-
-func GetTaskStats(ctx context.Context, taskId int64) map[string]interface{} {
- query := g.DB().Model("mailstat_send_mails sm")
-
- query = query.LeftJoin("mailstat_message_ids mi", "sm.postfix_message_id=mi.postfix_message_id")
- query = query.LeftJoin("recipient_info ri", "mi.message_id=ri.message_id")
-
- query = query.Where("ri.task_id = ?", taskId)
-
- query.Fields(
- "count(*) as sends",
- "coalesce(sum(case when sm.status='sent' and sm.dsn like '2.%' then 1 else 0 end), 0) as delivered", // success count
- "coalesce(sum(case when sm.status='bounced' then 1 else 0 end), 0) as bounced", // bounced count
- "coalesce(sum(case when sm.status='deferred' then 1 else 0 end), 0) as deferred", // deferred
- )
-
- result, err := query.One()
- if err != nil {
- g.Log().Error(ctx, err)
- return nil
- }
-
- sends := result["sends"].Int()
- delivered := result["delivered"].Int()
- bounced := result["bounced"].Int()
- deferred := result["deferred"].Int()
- //if deferred != 0 {
- // g.Log().Warning(ctx, " task {} deferred: {}", taskId, deferred)
- //}
-
- //// 通过campaign_id查打开和点击
- //campaignId := int(taskId)
- //openedCount, _ := g.DB().Model("mailstat_opened").
- // Where("campaign_id", campaignId).
- // Fields("count(distinct postfix_message_id) as opened").
- // Value()
- //clickedCount, _ := g.DB().Model("mailstat_clicked").
- // Where("campaign_id", campaignId).
- // Fields("count(distinct postfix_message_id) as clicked").
- // Value()
-
- stats := map[string]interface{}{
- "sends": sends,
- "delivered": delivered,
- "deferred": deferred,
- //"opened": openedCount.Int(),
- //"clicked": clickedCount.Int(),
- "bounced": bounced,
- }
-
- if sends > 0 {
- stats["delivery_rate"] = public.Round(float64(delivered)/float64(sends)*100, 2)
- stats["bounce_rate"] = public.Round(float64(bounced)/float64(sends)*100, 2)
- //stats["open_rate"] = public.Round(float64(stats["opened"].(int))/float64(sends)*100, 2)
- //stats["click_rate"] = public.Round(float64(stats["clicked"].(int))/float64(sends)*100, 2)
- } else {
- stats["delivery_rate"] = 0.0
- stats["bounce_rate"] = 0.0
- //stats["open_rate"] = 0.0
- //stats["click_rate"] = 0.0
- }
-
- return stats
-}
diff --git a/core/internal/model/entity/batch_mail.go b/core/internal/model/entity/batch_mail.go
index 83c14b96..289d18b3 100644
--- a/core/internal/model/entity/batch_mail.go
+++ b/core/internal/model/entity/batch_mail.go
@@ -49,26 +49,32 @@ type EmailTemplate struct {
// EmailTask Entity
type EmailTask struct {
- Id int `json:"id" dc:"Task ID"`
- TaskName string `json:"task_name" dc:"Task Name"`
- Addresser string `json:"addresser" dc:"Sender"`
- Subject string `json:"subject" dc:"Subject"`
- FullName string `json:"full_name" dc:"Sender Name"`
- RecipientCount int `json:"recipient_count" dc:"Recipient Count"`
- TaskProcess int `json:"task_process" dc:"Task Status"`
- Pause int `json:"pause" dc:"Pause Status"`
- TemplateId int `json:"template_id" dc:"Template ID"`
- IsRecord int `json:"is_record" dc:"Record to Outbox"`
- Unsubscribe int `json:"unsubscribe" dc:"Allow Unsubscribe"`
- Threads int `json:"threads" dc:"Thread Count"`
- Etypes string `json:"etypes" dc:"Contact Group IDs"`
- TrackOpen int `json:"track_open" dc:"Track Opens"`
- TrackClick int `json:"track_click" dc:"Track Clicks"`
- StartTime int `json:"start_time" dc:"Start Time"`
- CreateTime int `json:"create_time" dc:"Create Time"`
- UpdateTime int `json:"update_time" dc:"Update Time"`
- Remark string `json:"remark" dc:"Remark"`
- Active int `json:"active" dc:"Status"`
+ Id int `json:"id" dc:"Task ID"`
+ TaskName string `json:"task_name" dc:"Task Name"`
+ Addresser string `json:"addresser" dc:"Sender"`
+ Subject string `json:"subject" dc:"Subject"`
+ FullName string `json:"full_name" dc:"Sender Name"`
+ RecipientCount int `json:"recipient_count" dc:"Recipient Count"`
+ TaskProcess int `json:"task_process" dc:"Task Status"`
+ Pause int `json:"pause" dc:"Pause Status"`
+ TemplateId int `json:"template_id" dc:"Template ID"`
+ IsRecord int `json:"is_record" dc:"Record to Outbox"`
+ Unsubscribe int `json:"unsubscribe" dc:"Allow Unsubscribe"`
+ Threads int `json:"threads" dc:"Thread Count"`
+ Etypes string `json:"etypes" dc:"Contact Group IDs"`
+ TrackOpen int `json:"track_open" dc:"Track Opens"`
+ TrackClick int `json:"track_click" dc:"Track Clicks"`
+ StartTime int `json:"start_time" dc:"Start Time"`
+ CreateTime int `json:"create_time" dc:"Create Time"`
+ UpdateTime int `json:"update_time" dc:"Update Time"`
+ Remark string `json:"remark" dc:"Remark"`
+ Active int `json:"active" dc:"Status"`
+ AddType int `json:"addType" description:""`
+ SendsCount int `json:"sendsCount" description:""`
+ DeliveredCount int `json:"deliveredCount" description:""`
+ BouncedCount int `json:"bouncedCount" description:""`
+ DeferredCount int `json:"deferredCount" description:""`
+ StatsUpdateTime int `json:"statsUpdateTime" description:""`
}
// RecipientInfo Entity
diff --git a/core/internal/service/batch_mail/batch_mail.go b/core/internal/service/batch_mail/batch_mail.go
index 9d1669ec..493a702d 100644
--- a/core/internal/service/batch_mail/batch_mail.go
+++ b/core/internal/service/batch_mail/batch_mail.go
@@ -3,6 +3,7 @@ package batch_mail
import (
"billionmail-core/api/batch_mail/v1"
"billionmail-core/internal/model/entity"
+ "billionmail-core/internal/service/maillog_stat"
"billionmail-core/internal/service/public"
"billionmail-core/internal/service/warmup"
"context"
@@ -12,9 +13,14 @@ import (
"github.com/gogf/gf/v2/frame/g"
"github.com/gogf/gf/v2/util/gconv"
"strings"
+ "sync/atomic"
"time"
)
+var (
+ updateTaskStatsRunning int32 // 0: not running, 1: running
+)
+
type CreateTaskArgs struct {
Addresser string `json:"addresser"` // Sender email address
Subject string `json:"subject"` // Email subject
@@ -492,3 +498,89 @@ func GetActualRecipientCount(ctx context.Context, taskId int) (int, error) {
return count, nil
}
+
+// Regularly update the linked query data of marketing tasks
+func UpdateTaskJoinMailstat(ctx context.Context) {
+
+ if !atomic.CompareAndSwapInt32(&updateTaskStatsRunning, 0, 1) {
+ g.Log().Info(ctx, "UpdateTaskJoinMailstat is already running, skipping this run.")
+ return
+ }
+ defer atomic.StoreInt32(&updateTaskStatsRunning, 0)
+
+ // Search for tasks that need to be updated
+ // 1. Tasks that are currently in progress or have been paused (task_process = 1 or 3)
+ // 2. Tasks that have been completed but have sent fewer messages than the number of recipients (task_process = 2 and sends_count < recipient_count)
+ // 3. Tasks that have never been updated (stats_update_time = 0)
+ var tasksToUpdate []*entity.EmailTask
+ err := g.DB().Model("email_tasks").
+ Where("task_process IN (?)", []int{1, 3}).
+ WhereOr("task_process = 2 AND sends_count < recipient_count*0.99").
+ WhereOr("stats_update_time", 0).
+ Order("id DESC").
+ Scan(&tasksToUpdate)
+ if err != nil {
+ g.Log().Errorf(ctx, "UpdateTaskJoinMailstat: failed to get tasks to update: %v", err)
+ return
+ }
+
+ if len(tasksToUpdate) == 0 {
+ g.Log().Debug(ctx, "UpdateTaskJoinMailstat: no tasks to update.")
+ return
+ }
+
+ g.Log().Infof(ctx, "UpdateTaskJoinMailstat: found %d tasks to update.", len(tasksToUpdate))
+
+ for _, task := range tasksToUpdate {
+ //g.Log().Warningf(ctx, "start %d: %s", task.Id, task.TaskName)
+
+ stats := getSingleTaskStats(ctx, int64(task.Id))
+ if stats == nil {
+ //g.Log().Warningf(ctx, "UpdateTaskJoinMailstat: could not get stats for task %d, skipping.", task.Id)
+ continue
+ }
+
+ _, err := g.DB().Model("email_tasks").
+ Where("id", task.Id).
+ Data(g.Map{
+ "sends_count": stats["sends"],
+ "delivered_count": stats["delivered"],
+ "bounced_count": stats["bounced"],
+ "deferred_count": stats["deferred"],
+ "stats_update_time": time.Now().Unix(),
+ }).
+ Update()
+
+ if err != nil {
+ g.Log().Errorf(ctx, "UpdateTaskJoinMailstat: failed to update stats for task %d: %v", task.Id, err)
+
+ } else {
+ g.Log().Debugf(ctx, "UpdateTaskJoinMailstat: successfully updated stats for task %d.", task.Id)
+ }
+ }
+
+ return
+}
+
+// getSingleTaskStats Obtain the statistical data for a single task
+func getSingleTaskStats(ctx context.Context, taskId int64) map[string]interface{} {
+
+ overview := maillog_stat.NewOverview()
+ stats := overview.OverviewDashboard(taskId, "", 0, 0)
+
+ if stats == nil {
+ g.Log().Errorf(ctx, "getSingleTaskStats: OverviewDashboard returned nil for task %d", taskId)
+ return nil
+ }
+
+ requiredKeys := []string{"sends", "delivered", "bounced", "deferred"}
+ for _, key := range requiredKeys {
+ if _, ok := stats[key]; !ok {
+ //g.Log().Warningf(ctx, "getSingleTaskStats: 任务 %d is missing key '%s'", taskId, key)
+
+ stats[key] = 0
+ }
+ }
+
+ return stats
+}
diff --git a/core/internal/service/database_initialization/batch_mail.go b/core/internal/service/database_initialization/batch_mail.go
index 317f10fd..70ec5370 100644
--- a/core/internal/service/database_initialization/batch_mail.go
+++ b/core/internal/service/database_initialization/batch_mail.go
@@ -79,7 +79,12 @@ func init() {
create_time INTEGER NOT NULL DEFAULT EXTRACT(EPOCH FROM NOW()),
update_time INTEGER NOT NULL DEFAULT EXTRACT(EPOCH FROM NOW()),
active SMALLINT NOT NULL DEFAULT 0,
- add_type SMALLINT NOT NULL DEFAULT 0,
+ add_type SMALLINT NOT NULL DEFAULT 0,
+ sends_count INTEGER NOT NULL DEFAULT 0,
+ delivered_count INTEGER NOT NULL DEFAULT 0,
+ bounced_count INTEGER NOT NULL DEFAULT 0,
+ deferred_count INTEGER NOT NULL DEFAULT 0,
+ stats_update_time INTEGER NOT NULL DEFAULT 0,
FOREIGN KEY (template_id) REFERENCES email_templates(id)
)`,
@@ -168,6 +173,7 @@ func init() {
`CREATE INDEX IF NOT EXISTS idx_recipient_info_is_sent ON recipient_info(is_sent)`,
`CREATE INDEX IF NOT EXISTS idx_recipient_info_message_id ON recipient_info(message_id)`,
`CREATE INDEX IF NOT EXISTS idx_recipient_info_task_sent ON recipient_info(task_id, is_sent)`,
+ `CREATE INDEX IF NOT EXISTS idx_email_tasks_task_process ON email_tasks(task_process)`,
}
for _, sql := range batchMailSQLList {
@@ -205,6 +211,13 @@ func init() {
//email_templates
_ = AddColumnIfNotExists("email_templates", "chat_id", "VARCHAR(255)", "''", false)
+ // email_tasks
+ _ = AddColumnIfNotExists("email_tasks", "sends_count", "INTEGER", "0", true)
+ _ = AddColumnIfNotExists("email_tasks", "delivered_count", "INTEGER", "0", true)
+ _ = AddColumnIfNotExists("email_tasks", "bounced_count", "INTEGER", "0", true)
+ _ = AddColumnIfNotExists("email_tasks", "deferred_count", "INTEGER", "0", true)
+ _ = AddColumnIfNotExists("email_tasks", "stats_update_time", "INTEGER", "0", true)
+
// unsubscribe_records
_ = DropForeignKeyIfExists("unsubscribe_records", "group_id")
_ = DropForeignKeyIfExists("unsubscribe_records", "template_id")
diff --git a/core/internal/service/maillog_stat/overview.go b/core/internal/service/maillog_stat/overview.go
index 220753ac..d4d913db 100644
--- a/core/internal/service/maillog_stat/overview.go
+++ b/core/internal/service/maillog_stat/overview.go
@@ -145,6 +145,12 @@ func (o *Overview) overviewDashboard(campaignID int64, domain string, startTime,
return aggregate
}
+// overviewDashboard dashboard data
+func (o *Overview) OverviewDashboard(campaignID int64, domain string, startTime, endTime int64) map[string]interface{} {
+ aggregate := o.overviewDashboard(campaignID, domain, startTime, endTime)
+ return aggregate
+}
+
// Obtain the number of deferred queues with a 1-minute cache
func (o *Overview) getPostfixDeferredQueueCount(ctx context.Context) (int, error) {
cache := gcache.New()
diff --git a/core/internal/service/timers/timers.go b/core/internal/service/timers/timers.go
index 4f84f111..855de4aa 100644
--- a/core/internal/service/timers/timers.go
+++ b/core/internal/service/timers/timers.go
@@ -136,6 +136,11 @@ func Start(ctx context.Context) (err error) {
abnormal_recipient.AbnormalRecipientAutoStat(context.Background())
})
+ // Regularly update the success and failure data of marketing tasks
+ gtimer.Add(1*time.Minute, func() {
+ batch_mail.UpdateTaskJoinMailstat(ctx)
+ })
+
g.Log().Debug(ctx, "All timers started successfully")
return nil
}