Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

feat: expose dataset connector sync logs (#20704)

buua4367b5dc82f8b2e
diff --git a/internal/dao/connector.go b/internal/dao/connector.go
index 2008bfdaaa..00ee9b528a 100644
--- a/internal/dao/connector.go
+++ b/internal/dao/connector.go
@@ -576,11 +576,26 @@ func (dao *ConnectorDAO) ListLogsByConnectorID(ctx context.Context, db *gorm.DB,
// ListLogs lists sync logs for the given tenant IDs with pagination.
// When datasetID is non-empty, only logs of that dataset are returned.
func (dao *ConnectorDAO) ListLogs(ctx context.Context, db *gorm.DB, tenantIDs []string, datasetID string, offset, limit int) ([]*entity.ConnectorSyncLog, int64, error) {
+ return dao.listLogs(ctx, db, tenantIDs, true, datasetID, offset, limit)
+}
+
+// ListLogsByDatasetID lists sync logs for connectors currently linked to a dataset.
+func (dao *ConnectorDAO) ListLogsByDatasetID(ctx context.Context, db *gorm.DB, datasetID string, offset, limit int) ([]*entity.ConnectorSyncLog, int64, error) {
+ if datasetID == "" {
+ return []*entity.ConnectorSyncLog{}, 0, nil
+ }
+ return dao.listLogs(ctx, db, nil, false, datasetID, offset, limit)
+}
+
+func (dao *ConnectorDAO) listLogs(ctx context.Context, db *gorm.DB, tenantIDs []string, filterTenant bool, datasetID string, offset, limit int) ([]*entity.ConnectorSyncLog, int64, error) {
baseQuery := db.WithContext(ctx).Model(&entity.SyncLogs{}).
Joins("JOIN connector ON sync_logs.connector_id = connector.id").
Joins("JOIN connector2kb ON sync_logs.connector_id = connector2kb.connector_id AND sync_logs.kb_id = connector2kb.kb_id").
- Joins("JOIN knowledgebase ON sync_logs.kb_id = knowledgebase.id").
- Where("connector.tenant_id IN ?", tenantIDs)
+ Joins("JOIN knowledgebase ON sync_logs.kb_id = knowledgebase.id")
+
+ if filterTenant {
+ baseQuery = baseQuery.Where("connector.tenant_id IN ?", tenantIDs)
+ }
if datasetID != "" {
baseQuery = baseQuery.Where("sync_logs.kb_id = ?", datasetID)
diff --git a/internal/handler/dataset.go b/internal/handler/dataset.go
index 1ca626c2f9..f33378c53b 100644
--- a/internal/handler/dataset.go
+++ b/internal/handler/dataset.go
@@ -531,6 +531,7 @@ func (h *DatasetsHandler) GetIngestionSummary(c *gin.Context) {
}
// ListIngestionLogs handles GET /api/v1/datasets/:dataset_id/ingestions.
+// The log_type=datasource query returns logs for connectors linked to the dataset.
func (h *DatasetsHandler) ListIngestionLogs(c *gin.Context) {
user, errorCode, errorMessage := GetUser(c)
if errorCode != common.CodeSuccess {
@@ -562,6 +563,17 @@ func (h *DatasetsHandler) ListIngestionLogs(c *gin.Context) {
pageSize = ps
}
+ ctx := c.Request.Context()
+ if strings.EqualFold(strings.TrimSpace(c.Query("log_type")), "datasource") {
+ result, code, err := h.datasetsService.ListSyncLogs(ctx, datasetID, user.ID, page, pageSize)
+ if err != nil {
+ common.ErrorWithCode(c, code, err.Error())
+ return
+ }
+ common.SuccessWithData(c, result, "success")
+ return
+ }
+
orderby := c.DefaultQuery("orderby", "create_time")
// desc defaults to true and is only disabled by the literal value "false".
desc := strings.ToLower(c.DefaultQuery("desc", "true")) != "false"
@@ -575,8 +587,6 @@ func (h *DatasetsHandler) ListIngestionLogs(c *gin.Context) {
// equivalent; the frontend only sends it on the Go backend.
documentID := c.Query("document_id")
- ctx := c.Request.Context()
-
result, code, err := h.datasetsService.ListIngestionLogs(ctx, datasetID, user.ID, page, pageSize, terms, operationStatus, createDateFrom, createDateTo, logType, keywords, documentID)
if err != nil {
common.ErrorWithCode(c, code, err.Error())
diff --git a/internal/handler/ingestion_messages_test.go b/internal/handler/ingestion_messages_test.go
index 950ac8f51d..3cc1cb2ef0 100644
--- a/internal/handler/ingestion_messages_test.go
+++ b/internal/handler/ingestion_messages_test.go
@@ -94,6 +94,56 @@ func TestDatasetsHandlerListIngestionMessagesValidatesAndPages(t *testing.T) {
}
}
+func TestDatasetsHandlerListIngestionLogsDatasourceLogType(t *testing.T) {
+ db := setupIngestionMessagesHandlerDB(t)
+ insertIngestionMessagesHandlerKB(t, db, "kb-1", "user-1")
+
+ connectors := []entity.Connector{
+ {ID: "connector-1", TenantID: "user-1", Name: "configured", Source: "rss", InputType: "poll", Config: entity.JSONMap{}, Status: string(entity.TaskStatusDone)},
+ {ID: "connector-2", TenantID: "user-1", Name: "unlinked", Source: "rss", InputType: "poll", Config: entity.JSONMap{}, Status: string(entity.TaskStatusDone)},
+ }
+ if err := db.Create(&connectors).Error; err != nil {
+ t.Fatalf("insert connectors: %v", err)
+ }
+ if err := db.Create(&entity.Connector2Kb{ID: "connector-1-kb-1", ConnectorID: "connector-1", KbID: "kb-1", AutoParse: "1"}).Error; err != nil {
+ t.Fatalf("link configured connector: %v", err)
+ }
+ logs := []entity.SyncLogs{
+ {ID: "sync-linked", ConnectorID: "connector-1", KbID: "kb-1", TaskType: dao.TaskTypeSync, Status: dao.SyncStatusDone, ErrorMsg: ""},
+ {ID: "sync-unlinked", ConnectorID: "connector-2", KbID: "kb-1", TaskType: dao.TaskTypeSync, Status: dao.SyncStatusDone, ErrorMsg: ""},
+ }
+ if err := db.Create(&logs).Error; err != nil {
+ t.Fatalf("insert sync logs: %v", err)
+ }
+
+ router := newIngestionLogsHandlerRouter()
+ response := httptest.NewRecorder()
+ router.ServeHTTP(response, httptest.NewRequest(http.MethodGet, "/api/v1/datasets/kb-1/ingestions?log_type=datasource&page=1&page_size=10", nil))
+ if response.Code != http.StatusOK {
+ t.Fatalf("status = %d, body = %s", response.Code, response.Body.String())
+ }
+ var body struct {
+ Code common.ErrorCode `json:"code"`
+ Data struct {
+ Total int64 `json:"total"`
+ Logs []struct {
+ ID string `json:"id"`
+ ConnectorID string `json:"connector_id"`
+ KbID string `json:"kb_id"`
+ } `json:"logs"`
+ } `json:"data"`
+ }
+ if err := json.Unmarshal(response.Body.Bytes(), &body); err != nil {
+ t.Fatalf("unmarshal response: %v", err)
+ }
+ if body.Code != common.CodeSuccess || body.Data.Total != 1 || len(body.Data.Logs) != 1 {
+ t.Fatalf("response = %+v, raw=%s, want one configured connector log", body, response.Body.String())
+ }
+ if body.Data.Logs[0].ID != "sync-linked" || body.Data.Logs[0].ConnectorID != "connector-1" || body.Data.Logs[0].KbID != "kb-1" {
+ t.Fatalf("sync log = %+v, want configured connector's log", body.Data.Logs[0])
+ }
+}
+
func newIngestionMessagesHandlerRouter() *gin.Engine {
gin.SetMode(gin.TestMode)
h := NewDatasetsHandler(dataset.NewDatasetService(), nil)
@@ -105,13 +155,24 @@ func newIngestionMessagesHandlerRouter() *gin.Engine {
return r
}
+func newIngestionLogsHandlerRouter() *gin.Engine {
+ gin.SetMode(gin.TestMode)
+ h := NewDatasetsHandler(dataset.NewDatasetService(), nil)
+ r := gin.New()
+ r.GET("/api/v1/datasets/:dataset_id/ingestions", func(c *gin.Context) {
+ c.Set("user", &entity.User{ID: "user-1"})
+ h.ListIngestionLogs(c)
+ })
+ return r
+}
+
func setupIngestionMessagesHandlerDB(t *testing.T) *gorm.DB {
t.Helper()
db, err := gorm.Open(sqlite.Open("file:"+url.QueryEscape(t.Name())+"?mode=memory&cache=shared"), &gorm.Config{TranslateError: true})
if err != nil {
t.Fatalf("open sqlite: %v", err)
}
- if err := db.AutoMigrate(&entity.Knowledgebase{}, &entity.PipelineOperationLog{}, &entity.IngestionTaskLog{}, &entity.User{}, &entity.UserTenant{}); err != nil {
+ if err := db.AutoMigrate(&entity.Knowledgebase{}, &entity.PipelineOperationLog{}, &entity.IngestionTaskLog{}, &entity.User{}, &entity.UserTenant{}, &entity.Connector{}, &entity.Connector2Kb{}, &entity.SyncLogs{}); err != nil {
t.Fatalf("migrate test schema: %v", err)
}
original := dao.DB
@@ -145,4 +206,14 @@ func insertIngestionMessagesHandlerKB(t *testing.T, db *gorm.DB, id, userID stri
}).Error; err != nil {
t.Fatalf("insert knowledgebase: %v", err)
}
+ activeStatus := string(entity.StatusValid)
+ if err := db.Create(&entity.UserTenant{
+ ID: id + "-owner-membership",
+ UserID: userID,
+ TenantID: userID,
+ Role: "owner",
+ Status: &activeStatus,
+ }).Error; err != nil {
+ t.Fatalf("insert tenant owner: %v", err)
+ }
}
diff --git a/internal/service/dataset/ingestion.go b/internal/service/dataset/ingestion.go
index bdf4e27df5..14e022bf4a 100644
--- a/internal/service/dataset/ingestion.go
+++ b/internal/service/dataset/ingestion.go
@@ -228,6 +228,37 @@ func isReadableIngestionLog(log *entity.PipelineOperationLog) bool {
return log.RunCount == nil || *log.RunCount > 0
}
+// ListSyncLogs returns paginated sync logs for connectors currently linked to an accessible dataset.
+func (d *DatasetService) ListSyncLogs(ctx context.Context, datasetID, userID string, page, pageSize int) (map[string]interface{}, common.ErrorCode, error) {
+ if datasetID == "" {
+ return nil, common.CodeDataError, errors.New(`lack of "Dataset ID"`)
+ }
+ if err := d.CheckAccess(ctx, permission.Subject{UserID: userID}, datasetID, permission.OperationRead); err != nil {
+ code, permissionErr := permissionresponse.Normalize(err)
+ return nil, code, permissionErr
+ }
+
+ if page <= 0 {
+ page = 1
+ }
+ if pageSize <= 0 {
+ pageSize = 30
+ }
+
+ logs, total, err := d.connectorDAO.ListLogsByDatasetID(ctx, dao.DB, datasetID, (page-1)*pageSize, pageSize)
+ if err != nil {
+ return nil, common.CodeServerError, fmt.Errorf("list dataset connector sync logs: %w", err)
+ }
+ if logs == nil {
+ logs = []*entity.ConnectorSyncLog{}
+ }
+
+ return map[string]interface{}{
+ "total": total,
+ "logs": logs,
+ }, common.CodeSuccess, nil
+}
+
func (d *DatasetService) ListIngestionLogs(ctx context.Context, datasetID, userID string, page, pageSize int, terms []dao.OrderTerm, operationStatus []string, createDateFrom, createDateTo, logType, keywords, documentID string) (map[string]interface{}, common.ErrorCode, error) {
if datasetID == "" {
return nil, common.CodeDataError, errors.New(`lack of "Dataset ID"`)
diff --git a/internal/service/dataset/ingestion_messages_test.go b/internal/service/dataset/ingestion_messages_test.go
index 33b2c5961b..f9c959dd6b 100644
--- a/internal/service/dataset/ingestion_messages_test.go
+++ b/internal/service/dataset/ingestion_messages_test.go
@@ -113,6 +113,49 @@ func TestDatasetIngestionLogUsesEventStream(t *testing.T) {
assertLatestEventMap(t, log, 1, "Knowledge compilation completed")
}
+func TestListSyncLogsReturnsOnlyCurrentlyLinkedDatasetConnectors(t *testing.T) {
+ db := setupServiceTestDB(t)
+ pushServiceDB(t, db)
+ insertCompilationOwnerKB(t, "kb-1", "user-1")
+ insertCompilationOwnerKB(t, "kb-2", "user-1")
+
+ connectors := []entity.Connector{
+ {ID: "connector-1", TenantID: "user-1", Name: "linked", Source: "rss", InputType: "poll", Config: entity.JSONMap{}, Status: string(entity.TaskStatusDone)},
+ {ID: "connector-2", TenantID: "user-1", Name: "other", Source: "rss", InputType: "poll", Config: entity.JSONMap{}, Status: string(entity.TaskStatusDone)},
+ }
+ if err := db.Create(&connectors).Error; err != nil {
+ t.Fatalf("insert connectors: %v", err)
+ }
+ if err := db.Create(&entity.Connector2Kb{ID: "connector-1-kb-1", ConnectorID: "connector-1", KbID: "kb-1", AutoParse: "1"}).Error; err != nil {
+ t.Fatalf("link connector to kb-1: %v", err)
+ }
+ if err := db.Create(&entity.Connector2Kb{ID: "connector-2-kb-2", ConnectorID: "connector-2", KbID: "kb-2", AutoParse: "1"}).Error; err != nil {
+ t.Fatalf("link connector to kb-2: %v", err)
+ }
+
+ logs := []entity.SyncLogs{
+ {ID: "sync-kb-1", ConnectorID: "connector-1", KbID: "kb-1", TaskType: dao.TaskTypeSync, Status: dao.SyncStatusDone, ErrorMsg: ""},
+ {ID: "sync-kb-2", ConnectorID: "connector-2", KbID: "kb-2", TaskType: dao.TaskTypeSync, Status: dao.SyncStatusDone, ErrorMsg: ""},
+ // A stale/unlinked row must not appear in the dataset's configured connector logs.
+ {ID: "sync-unlinked", ConnectorID: "connector-2", KbID: "kb-1", TaskType: dao.TaskTypeSync, Status: dao.SyncStatusDone, ErrorMsg: ""},
+ }
+ if err := db.Create(&logs).Error; err != nil {
+ t.Fatalf("insert sync logs: %v", err)
+ }
+
+ result, code, err := NewDatasetService().ListSyncLogs(t.Context(), "kb-1", "user-1", 1, 30)
+ if err != nil || code != common.CodeSuccess {
+ t.Fatalf("ListSyncLogs = (%+v, %v, %v), want success", result, code, err)
+ }
+ if result["total"] != int64(1) {
+ t.Fatalf("total = %#v, want 1", result["total"])
+ }
+ items, ok := result["logs"].([]*entity.ConnectorSyncLog)
+ if !ok || len(items) != 1 || items[0].ID != "sync-kb-1" {
+ t.Fatalf("logs = %#v, want only sync-kb-1", result["logs"])
+ }
+}
+
func TestListIngestionLogsIncludesPythonFileLogsAndStatusFilters(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)