Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

feat: sort sync log list by status priority (#19777)

mkaaad36a6456f4df5
diff --git a/api/db/services/connector_service.py b/api/db/services/connector_service.py
index a44570fabb..a108ac386f 100644
--- a/api/db/services/connector_service.py
+++ b/api/db/services/connector_service.py
@@ -14,23 +14,21 @@
# limitations under the License.
#
import logging
-from datetime import datetime
import os
-from typing import Optional, Tuple, List
+from datetime import datetime
from anthropic import BaseModel
-from peewee import SQL, fn
+from peewee import SQL, Case, fn
from api.db import InputType
-from api.db.db_models import DB, Connector, SyncLogs, Connector2Kb, Knowledgebase
+from api.db.db_models import DB, Connector, Connector2Kb, Knowledgebase, SyncLogs
from api.db.services.common_service import CommonService
-from api.db.services.document_service import DocumentService
-from api.db.services.document_service import DocMetadataService
+from api.db.services.document_service import DocMetadataService, DocumentService
from api.db.services.knowledgebase_service import KnowledgebaseService
from api.utils.common import hash128
from common import settings
-from common.misc_utils import get_uuid
from common.constants import ConnectorTaskType, TaskStatus
+from common.misc_utils import get_uuid
from common.settings import TIMEZONE
from common.time_utils import current_timestamp, timestamp_to_date
@@ -302,7 +300,7 @@ class SyncLogsService(CommonService):
model = SyncLogs
@classmethod
- def list_sync_tasks(cls, connector_id=None, page_number=None, items_per_page=15) -> Tuple[List[dict], int]:
+ def list_sync_tasks(cls, connector_id=None, page_number=None, items_per_page=15) -> tuple[list[dict], int]:
fields = [
cls.model.id,
cls.model.connector_id,
@@ -320,6 +318,14 @@ class SyncLogsService(CommonService):
Knowledgebase.name.alias("kb_name"),
cls.model.status,
cls.model.update_time,
+ Case(
+ None,
+ [
+ (cls.model.status == TaskStatus.SCHEDULE, 0),
+ (cls.model.status == TaskStatus.RUNNING, 1),
+ ],
+ 2,
+ ).alias("status_rank"),
]
if not connector_id:
fields.append(Connector.config)
@@ -344,22 +350,25 @@ class SyncLogsService(CommonService):
expr = SQL("NOW() - INTERVAL `t2`.`refresh_freq` MINUTE")
query = query.where(Connector.input_type == InputType.POLL, Connector.status == TaskStatus.SCHEDULE, cls.model.status == TaskStatus.SCHEDULE, cls.model.update_date < expr)
- query = query.distinct().order_by(cls.model.update_time.desc())
+ query = query.distinct().order_by(SQL("status_rank"), cls.model.update_time.desc())
total = query.count()
if page_number:
query = query.paginate(page_number, items_per_page)
- return list(query.dicts()), total
+ rows = list(query.dicts())
+ for row in rows:
+ row.pop("status_rank", None)
+ return rows, total
@classmethod
- def list_due_sync_tasks(cls) -> List[dict]:
+ def list_due_sync_tasks(cls) -> list[dict]:
return cls._list_due_tasks_for_freq(
ConnectorTaskType.SYNC,
"refresh_freq",
)
@classmethod
- def list_due_prune_tasks(cls) -> List[dict]:
+ def list_due_prune_tasks(cls) -> list[dict]:
tasks = cls._list_due_tasks_for_freq(
ConnectorTaskType.PRUNE,
"prune_freq",
@@ -373,7 +382,7 @@ class SyncLogsService(CommonService):
]
@classmethod
- def _list_due_tasks_for_freq(cls, task_type: str, freq_field: str) -> List[dict]:
+ def _list_due_tasks_for_freq(cls, task_type: str, freq_field: str) -> list[dict]:
fields = [
cls.model.id,
cls.model.connector_id,
@@ -476,17 +485,15 @@ class SyncLogsService(CommonService):
task_id = get_uuid()
ConnectorService.update_by_id(connector_id, {"status": TaskStatus.SCHEDULE})
ret = cls.save(
- **{
- "id": task_id,
- "kb_id": kb_id,
- "status": TaskStatus.SCHEDULE,
- "connector_id": connector_id,
- "task_type": task_type,
- "poll_range_start": poll_range_start,
- "from_beginning": reindex,
- "total_docs_indexed": total_docs_indexed,
- "time_started": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
- }
+ id=task_id,
+ kb_id=kb_id,
+ status=TaskStatus.SCHEDULE,
+ connector_id=connector_id,
+ task_type=task_type,
+ poll_range_start=poll_range_start,
+ from_beginning=reindex,
+ total_docs_indexed=total_docs_indexed,
+ time_started=datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
)
if run_immediately:
DB.execute_sql(
@@ -554,7 +561,7 @@ class SyncLogsService(CommonService):
id: str
filename: str
blob: bytes
- fingerprint: Optional[str] = None
+ fingerprint: str | None = None
def read(self) -> bytes:
return self.blob
@@ -624,7 +631,7 @@ class Connector2KbService(CommonService):
if conn_id in old_conn_ids:
cls.filter_update([cls.model.connector_id == conn_id, cls.model.kb_id == kb_id], {"auto_parse": conn.get("auto_parse", "1")})
continue
- cls.save(**{"id": get_uuid(), "connector_id": conn_id, "kb_id": kb_id, "auto_parse": conn.get("auto_parse", "1")})
+ cls.save(id=get_uuid(), connector_id=conn_id, kb_id=kb_id, auto_parse=conn.get("auto_parse", "1"))
SyncLogsService.schedule(conn_id, kb_id, reindex=True, task_type=ConnectorTaskType.SYNC, run_immediately=True)
e, full_conn = ConnectorService.get_by_id(conn_id)
if e and (full_conn.config or {}).get("sync_deleted_files"):
diff --git a/internal/dao/connector.go b/internal/dao/connector.go
index 8ec0b626b1..1255eb73b1 100644
--- a/internal/dao/connector.go
+++ b/internal/dao/connector.go
@@ -516,6 +516,18 @@ func connectorConfigBool(config map[string]interface{}, key string) bool {
}
}
+// syncLogsStatusPriorityOrder orders sync log rows by status priority:
+// SCHEDULE first, RUNNING second, then the rest, newest update first within
+// each group. Returned as a single raw ORDER BY fragment because gorm.Expr is
+// silently dropped by gorm's Order and repeated Order calls merge in reverse.
+func syncLogsStatusPriorityOrder() string {
+ return fmt.Sprintf(
+ "CASE WHEN sync_logs.status = '%s' THEN 0 WHEN sync_logs.status = '%s' THEN 1 ELSE 2 END, sync_logs.update_time DESC",
+ string(entity.TaskStatusSchedule),
+ string(entity.TaskStatusRunning),
+ )
+}
+
// ListLogsByConnectorID lists sync logs for one connector with pagination.
func (dao *ConnectorDAO) ListLogsByConnectorID(ctx context.Context, db *gorm.DB, connectorID string, offset, limit int) ([]*entity.ConnectorSyncLog, int64, error) {
baseQuery := db.WithContext(ctx).Model(&entity.SyncLogs{}).
@@ -547,9 +559,10 @@ func (dao *ConnectorDAO) ListLogsByConnectorID(ctx context.Context, db *gorm.DB,
"connector.prune_freq AS prune_freq",
"knowledgebase.name AS kb_name",
"sync_logs.status",
+ "sync_logs.update_time",
).
Distinct().
- Order("sync_logs.update_date DESC").
+ Order(syncLogsStatusPriorityOrder()).
Offset(offset).
Limit(limit).
Scan(&logs).Error
@@ -596,9 +609,10 @@ func (dao *ConnectorDAO) ListLogs(ctx context.Context, db *gorm.DB, tenantIDs []
"connector.prune_freq AS prune_freq",
"knowledgebase.name AS kb_name",
"sync_logs.status",
+ "sync_logs.update_time",
).
Distinct().
- Order("sync_logs.update_date DESC")
+ Order(syncLogsStatusPriorityOrder())
if limit > 0 {
query = query.Offset(offset).Limit(limit)
}
diff --git a/internal/service/connector_logs_test.go b/internal/service/connector_logs_test.go
index 308aa32380..e5e8e9ed7c 100644
--- a/internal/service/connector_logs_test.go
+++ b/internal/service/connector_logs_test.go
@@ -20,6 +20,7 @@ import (
"context"
"fmt"
"testing"
+ "time"
"ragflow/internal/common"
"ragflow/internal/dao"
@@ -112,3 +113,104 @@ func TestListLogsPaginationModes(t *testing.T) {
t.Fatalf("ListLogs(normalize) = total %d len %d, want 5/5", total, len(normalized))
}
}
+
+func i64ptr(v int64) *int64 { return &v }
+func timeMillisPtr(v int64) *time.Time { ts := time.UnixMilli(v); return &ts }
+
+func TestListLogsStatusPriority(t *testing.T) {
+ db := setupServiceTestDB(t)
+ pushServiceDB(t, db)
+ if err := db.AutoMigrate(&entity.Connector{}, &entity.Connector2Kb{}, &entity.Knowledgebase{}, &entity.SyncLogs{}); err != nil {
+ t.Fatalf("migrate connector tables: %v", err)
+ }
+
+ if err := db.Create(&entity.Connector{
+ ID: "conn-1",
+ TenantID: "user-1",
+ Name: "conn-1",
+ Source: "rss",
+ InputType: "poll",
+ Config: entity.JSONMap{},
+ Status: string(entity.TaskStatusSchedule),
+ RefreshFreq: 5,
+ PruneFreq: 5,
+ TimeoutSecs: 60,
+ }).Error; err != nil {
+ t.Fatalf("insert connector: %v", err)
+ }
+ if err := db.Create(&entity.Knowledgebase{
+ ID: "kb-1",
+ TenantID: "user-1",
+ Name: "kb-1",
+ CreatedBy: "user-1",
+ EmbdID: "embd",
+ }).Error; err != nil {
+ t.Fatalf("insert kb: %v", err)
+ }
+ if err := db.Create(&entity.Connector2Kb{
+ ID: "conn-1-kb-1",
+ ConnectorID: "conn-1",
+ KbID: "kb-1",
+ AutoParse: "1",
+ }).Error; err != nil {
+ t.Fatalf("insert connector2kb: %v", err)
+ }
+
+ // Deliberately interleave statuses and update_time so a missing status
+ // priority or a missing secondary sort is observable in the result order.
+ seed := []entity.SyncLogs{
+ {ID: "task-done-1", ConnectorID: "conn-1", KbID: "kb-1", TaskType: dao.TaskTypeSync, Status: string(entity.TaskStatusDone), BaseModel: entity.BaseModel{UpdateTime: i64ptr(100), UpdateDate: timeMillisPtr(100)}},
+ {ID: "task-done-2", ConnectorID: "conn-1", KbID: "kb-1", TaskType: dao.TaskTypeSync, Status: string(entity.TaskStatusDone), BaseModel: entity.BaseModel{UpdateTime: i64ptr(200), UpdateDate: timeMillisPtr(200)}},
+ {ID: "task-running-1", ConnectorID: "conn-1", KbID: "kb-1", TaskType: dao.TaskTypeSync, Status: string(entity.TaskStatusRunning), BaseModel: entity.BaseModel{UpdateTime: i64ptr(300), UpdateDate: timeMillisPtr(300)}},
+ {ID: "task-schedule-1", ConnectorID: "conn-1", KbID: "kb-1", TaskType: dao.TaskTypeSync, Status: string(entity.TaskStatusSchedule), BaseModel: entity.BaseModel{UpdateTime: i64ptr(400), UpdateDate: timeMillisPtr(400)}},
+ {ID: "task-fail-1", ConnectorID: "conn-1", KbID: "kb-1", TaskType: dao.TaskTypeSync, Status: string(entity.TaskStatusFail), BaseModel: entity.BaseModel{UpdateTime: i64ptr(500), UpdateDate: timeMillisPtr(500)}},
+ {ID: "task-schedule-2", ConnectorID: "conn-1", KbID: "kb-1", TaskType: dao.TaskTypeSync, Status: string(entity.TaskStatusSchedule), BaseModel: entity.BaseModel{UpdateTime: i64ptr(600), UpdateDate: timeMillisPtr(600)}},
+ }
+ for _, log := range seed {
+ if err := db.Create(&log).Error; err != nil {
+ t.Fatalf("insert sync log %s: %v", log.ID, err)
+ }
+ }
+
+ want := []string{
+ "task-schedule-2",
+ "task-schedule-1",
+ "task-running-1",
+ "task-fail-1",
+ "task-done-2",
+ "task-done-1",
+ }
+
+ svc := NewConnectorService()
+ ctx := context.Background()
+
+ byConnector, total, code, err := svc.ListLog(ctx, "conn-1", "user-1", 1, 100)
+ if err != nil || code != common.CodeSuccess {
+ t.Fatalf("ListLog: code=%v err=%v", code, err)
+ }
+ if total != int64(len(want)) {
+ t.Fatalf("ListLog total = %d, want %d", total, len(want))
+ }
+ assertLogOrder(t, "ListLog", byConnector, want)
+
+ tenantWide, total, code, err := svc.ListLogs(ctx, "user-1", "", 1, 100)
+ if err != nil || code != common.CodeSuccess {
+ t.Fatalf("ListLogs: code=%v err=%v", code, err)
+ }
+ if total != int64(len(want)) {
+ t.Fatalf("ListLogs total = %d, want %d", total, len(want))
+ }
+ assertLogOrder(t, "ListLogs", tenantWide, want)
+}
+
+func assertLogOrder(t *testing.T, name string, logs []*entity.ConnectorSyncLog, want []string) {
+ t.Helper()
+ if len(logs) != len(want) {
+ t.Fatalf("%s returned %d logs, want %d", name, len(logs), len(want))
+ }
+ for i, log := range logs {
+ if log.ID != want[i] {
+ t.Fatalf("%s order[%d] = %s, want %s", name, i, log.ID, want[i])
+ }
+ }
+}
diff --git a/test/unit_test/api/db/services/test_connector_sync_logs_order.py b/test/unit_test/api/db/services/test_connector_sync_logs_order.py
new file mode 100644
index 0000000000..fe67c8a16a
--- /dev/null
+++ b/test/unit_test/api/db/services/test_connector_sync_logs_order.py
@@ -0,0 +1,78 @@
+#
+# Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""Sync task list orders SCHEDULE first, RUNNING second, then the rest."""
+
+from datetime import UTC, datetime
+
+from peewee import SqliteDatabase
+
+from api.db.db_models import DB, Connector, Connector2Kb, Knowledgebase, SyncLogs
+from api.db.services.connector_service import SyncLogsService
+from common.constants import TaskStatus
+
+_MODELS = [SyncLogs, Connector, Connector2Kb, Knowledgebase]
+
+
+def test_list_sync_tasks_status_priority():
+ sqlite_db = SqliteDatabase(":memory:")
+ sqlite_db.bind(_MODELS, bind_refs=False, bind_backrefs=False)
+ sqlite_db.connect()
+ sqlite_db.create_tables(_MODELS)
+ try:
+ Connector.create(
+ id="conn-1",
+ tenant_id="tenant-1",
+ name="conn-1",
+ source="rss",
+ input_type="poll",
+ status=TaskStatus.SCHEDULE,
+ refresh_freq=5,
+ prune_freq=5,
+ timeout_secs=60,
+ )
+ Knowledgebase.create(id="kb-1", tenant_id="tenant-1", name="kb-1", embd_id="", created_by="tenant-1")
+ Connector2Kb.create(id="c2k-1", connector_id="conn-1", kb_id="kb-1", auto_parse="1")
+
+ rows = [
+ ("task-done-1", TaskStatus.DONE, 100),
+ ("task-done-2", TaskStatus.DONE, 200),
+ ("task-running-1", TaskStatus.RUNNING, 300),
+ ("task-schedule-1", TaskStatus.SCHEDULE, 400),
+ ("task-fail-1", TaskStatus.FAIL, 500),
+ ("task-schedule-2", TaskStatus.SCHEDULE, 600),
+ ]
+ for task_id, status, ts in rows:
+ SyncLogs.create(id=task_id, connector_id="conn-1", kb_id="kb-1", status=status)
+ sqlite_db.execute_sql(
+ "UPDATE sync_logs SET update_time = ?, update_date = ? WHERE id = ?",
+ (ts, datetime.fromtimestamp(ts / 1000, tz=UTC), task_id),
+ )
+
+ logs, total = SyncLogsService.list_sync_tasks("conn-1", 1, 100)
+ assert total == len(rows)
+ assert [log["id"] for log in logs] == [
+ "task-schedule-2",
+ "task-schedule-1",
+ "task-running-1",
+ "task-fail-1",
+ "task-done-2",
+ "task-done-1",
+ ]
+ assert all("status_rank" not in log for log in logs)
+ finally:
+ sqlite_db.drop_tables(_MODELS)
+ sqlite_db.close()
+ DB.bind(_MODELS, bind_refs=False, bind_backrefs=False)