RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
feat: expose dataset connector sync logs (#20704)
buua4367b5dc82f8b2e
diff --git a/internal/dao/connector.go b/internal/dao/connector.goindex 2008bfdaaa..00ee9b528a 100644--- a/internal/dao/connector.go+++ b/internal/dao/connector.go@@ -576,11 +576,26 @@ func (dao *ConnectorDAO) ListLogsByConnectorID(ctx context.Context, db *gorm.DB,// ListLogs lists sync logs for the given tenant IDs with pagination.// When datasetID is non-empty, only logs of that dataset are returned.func (dao *ConnectorDAO) ListLogs(ctx context.Context, db *gorm.DB, tenantIDs []string, datasetID string, offset, limit int) ([]*entity.ConnectorSyncLog, int64, error) {+ return dao.listLogs(ctx, db, tenantIDs, true, datasetID, offset, limit)+}++// ListLogsByDatasetID lists sync logs for connectors currently linked to a dataset.+func (dao *ConnectorDAO) ListLogsByDatasetID(ctx context.Context, db *gorm.DB, datasetID string, offset, limit int) ([]*entity.ConnectorSyncLog, int64, error) {+ if datasetID == "" {+ return []*entity.ConnectorSyncLog{}, 0, nil+ }+ return dao.listLogs(ctx, db, nil, false, datasetID, offset, limit)+}++func (dao *ConnectorDAO) listLogs(ctx context.Context, db *gorm.DB, tenantIDs []string, filterTenant bool, datasetID string, offset, limit int) ([]*entity.ConnectorSyncLog, int64, error) {baseQuery := db.WithContext(ctx).Model(&entity.SyncLogs{}).Joins("JOIN connector ON sync_logs.connector_id = connector.id").Joins("JOIN connector2kb ON sync_logs.connector_id = connector2kb.connector_id AND sync_logs.kb_id = connector2kb.kb_id").- Joins("JOIN knowledgebase ON sync_logs.kb_id = knowledgebase.id").- Where("connector.tenant_id IN ?", tenantIDs)+ Joins("JOIN knowledgebase ON sync_logs.kb_id = knowledgebase.id")++ if filterTenant {+ baseQuery = baseQuery.Where("connector.tenant_id IN ?", tenantIDs)+ }if datasetID != "" {baseQuery = baseQuery.Where("sync_logs.kb_id = ?", datasetID)diff --git a/internal/handler/dataset.go b/internal/handler/dataset.goindex 1ca626c2f9..f33378c53b 100644--- a/internal/handler/dataset.go+++ b/internal/handler/dataset.go@@ -531,6 +531,7 @@ func (h *DatasetsHandler) GetIngestionSummary(c *gin.Context) {}// ListIngestionLogs handles GET /api/v1/datasets/:dataset_id/ingestions.+// The log_type=datasource query returns logs for connectors linked to the dataset.func (h *DatasetsHandler) ListIngestionLogs(c *gin.Context) {user, errorCode, errorMessage := GetUser(c)if errorCode != common.CodeSuccess {@@ -562,6 +563,17 @@ func (h *DatasetsHandler) ListIngestionLogs(c *gin.Context) {pageSize = ps}+ ctx := c.Request.Context()+ if strings.EqualFold(strings.TrimSpace(c.Query("log_type")), "datasource") {+ result, code, err := h.datasetsService.ListSyncLogs(ctx, datasetID, user.ID, page, pageSize)+ if err != nil {+ common.ErrorWithCode(c, code, err.Error())+ return+ }+ common.SuccessWithData(c, result, "success")+ return+ }+orderby := c.DefaultQuery("orderby", "create_time")// desc defaults to true and is only disabled by the literal value "false".desc := strings.ToLower(c.DefaultQuery("desc", "true")) != "false"@@ -575,8 +587,6 @@ func (h *DatasetsHandler) ListIngestionLogs(c *gin.Context) {// equivalent; the frontend only sends it on the Go backend.documentID := c.Query("document_id")- ctx := c.Request.Context()-result, code, err := h.datasetsService.ListIngestionLogs(ctx, datasetID, user.ID, page, pageSize, terms, operationStatus, createDateFrom, createDateTo, logType, keywords, documentID)if err != nil {common.ErrorWithCode(c, code, err.Error())diff --git a/internal/handler/ingestion_messages_test.go b/internal/handler/ingestion_messages_test.goindex 950ac8f51d..3cc1cb2ef0 100644--- a/internal/handler/ingestion_messages_test.go+++ b/internal/handler/ingestion_messages_test.go@@ -94,6 +94,56 @@ func TestDatasetsHandlerListIngestionMessagesValidatesAndPages(t *testing.T) {}}+func TestDatasetsHandlerListIngestionLogsDatasourceLogType(t *testing.T) {+ db := setupIngestionMessagesHandlerDB(t)+ insertIngestionMessagesHandlerKB(t, db, "kb-1", "user-1")++ connectors := []entity.Connector{+ {ID: "connector-1", TenantID: "user-1", Name: "configured", Source: "rss", InputType: "poll", Config: entity.JSONMap{}, Status: string(entity.TaskStatusDone)},+ {ID: "connector-2", TenantID: "user-1", Name: "unlinked", Source: "rss", InputType: "poll", Config: entity.JSONMap{}, Status: string(entity.TaskStatusDone)},+ }+ if err := db.Create(&connectors).Error; err != nil {+ t.Fatalf("insert connectors: %v", err)+ }+ if err := db.Create(&entity.Connector2Kb{ID: "connector-1-kb-1", ConnectorID: "connector-1", KbID: "kb-1", AutoParse: "1"}).Error; err != nil {+ t.Fatalf("link configured connector: %v", err)+ }+ logs := []entity.SyncLogs{+ {ID: "sync-linked", ConnectorID: "connector-1", KbID: "kb-1", TaskType: dao.TaskTypeSync, Status: dao.SyncStatusDone, ErrorMsg: ""},+ {ID: "sync-unlinked", ConnectorID: "connector-2", KbID: "kb-1", TaskType: dao.TaskTypeSync, Status: dao.SyncStatusDone, ErrorMsg: ""},+ }+ if err := db.Create(&logs).Error; err != nil {+ t.Fatalf("insert sync logs: %v", err)+ }++ router := newIngestionLogsHandlerRouter()+ response := httptest.NewRecorder()+ router.ServeHTTP(response, httptest.NewRequest(http.MethodGet, "/api/v1/datasets/kb-1/ingestions?log_type=datasource&page=1&page_size=10", nil))+ if response.Code != http.StatusOK {+ t.Fatalf("status = %d, body = %s", response.Code, response.Body.String())+ }+ var body struct {+ Code common.ErrorCode `json:"code"`+ Data struct {+ Total int64 `json:"total"`+ Logs []struct {+ ID string `json:"id"`+ ConnectorID string `json:"connector_id"`+ KbID string `json:"kb_id"`+ } `json:"logs"`+ } `json:"data"`+ }+ if err := json.Unmarshal(response.Body.Bytes(), &body); err != nil {+ t.Fatalf("unmarshal response: %v", err)+ }+ if body.Code != common.CodeSuccess || body.Data.Total != 1 || len(body.Data.Logs) != 1 {+ t.Fatalf("response = %+v, raw=%s, want one configured connector log", body, response.Body.String())+ }+ if body.Data.Logs[0].ID != "sync-linked" || body.Data.Logs[0].ConnectorID != "connector-1" || body.Data.Logs[0].KbID != "kb-1" {+ t.Fatalf("sync log = %+v, want configured connector's log", body.Data.Logs[0])+ }+}+func newIngestionMessagesHandlerRouter() *gin.Engine {gin.SetMode(gin.TestMode)h := NewDatasetsHandler(dataset.NewDatasetService(), nil)@@ -105,13 +155,24 @@ func newIngestionMessagesHandlerRouter() *gin.Engine {return r}+func newIngestionLogsHandlerRouter() *gin.Engine {+ gin.SetMode(gin.TestMode)+ h := NewDatasetsHandler(dataset.NewDatasetService(), nil)+ r := gin.New()+ r.GET("/api/v1/datasets/:dataset_id/ingestions", func(c *gin.Context) {+ c.Set("user", &entity.User{ID: "user-1"})+ h.ListIngestionLogs(c)+ })+ return r+}+func setupIngestionMessagesHandlerDB(t *testing.T) *gorm.DB {t.Helper()db, err := gorm.Open(sqlite.Open("file:"+url.QueryEscape(t.Name())+"?mode=memory&cache=shared"), &gorm.Config{TranslateError: true})if err != nil {t.Fatalf("open sqlite: %v", err)}- if err := db.AutoMigrate(&entity.Knowledgebase{}, &entity.PipelineOperationLog{}, &entity.IngestionTaskLog{}, &entity.User{}, &entity.UserTenant{}); err != nil {+ if err := db.AutoMigrate(&entity.Knowledgebase{}, &entity.PipelineOperationLog{}, &entity.IngestionTaskLog{}, &entity.User{}, &entity.UserTenant{}, &entity.Connector{}, &entity.Connector2Kb{}, &entity.SyncLogs{}); err != nil {t.Fatalf("migrate test schema: %v", err)}original := dao.DB@@ -145,4 +206,14 @@ func insertIngestionMessagesHandlerKB(t *testing.T, db *gorm.DB, id, userID stri}).Error; err != nil {t.Fatalf("insert knowledgebase: %v", err)}+ activeStatus := string(entity.StatusValid)+ if err := db.Create(&entity.UserTenant{+ ID: id + "-owner-membership",+ UserID: userID,+ TenantID: userID,+ Role: "owner",+ Status: &activeStatus,+ }).Error; err != nil {+ t.Fatalf("insert tenant owner: %v", err)+ }}diff --git a/internal/service/dataset/ingestion.go b/internal/service/dataset/ingestion.goindex bdf4e27df5..14e022bf4a 100644--- a/internal/service/dataset/ingestion.go+++ b/internal/service/dataset/ingestion.go@@ -228,6 +228,37 @@ func isReadableIngestionLog(log *entity.PipelineOperationLog) bool {return log.RunCount == nil || *log.RunCount > 0}+// ListSyncLogs returns paginated sync logs for connectors currently linked to an accessible dataset.+func (d *DatasetService) ListSyncLogs(ctx context.Context, datasetID, userID string, page, pageSize int) (map[string]interface{}, common.ErrorCode, error) {+ if datasetID == "" {+ return nil, common.CodeDataError, errors.New(`lack of "Dataset ID"`)+ }+ if err := d.CheckAccess(ctx, permission.Subject{UserID: userID}, datasetID, permission.OperationRead); err != nil {+ code, permissionErr := permissionresponse.Normalize(err)+ return nil, code, permissionErr+ }++ if page <= 0 {+ page = 1+ }+ if pageSize <= 0 {+ pageSize = 30+ }++ logs, total, err := d.connectorDAO.ListLogsByDatasetID(ctx, dao.DB, datasetID, (page-1)*pageSize, pageSize)+ if err != nil {+ return nil, common.CodeServerError, fmt.Errorf("list dataset connector sync logs: %w", err)+ }+ if logs == nil {+ logs = []*entity.ConnectorSyncLog{}+ }++ return map[string]interface{}{+ "total": total,+ "logs": logs,+ }, common.CodeSuccess, nil+}+func (d *DatasetService) ListIngestionLogs(ctx context.Context, datasetID, userID string, page, pageSize int, terms []dao.OrderTerm, operationStatus []string, createDateFrom, createDateTo, logType, keywords, documentID string) (map[string]interface{}, common.ErrorCode, error) {if datasetID == "" {return nil, common.CodeDataError, errors.New(`lack of "Dataset ID"`)diff --git a/internal/service/dataset/ingestion_messages_test.go b/internal/service/dataset/ingestion_messages_test.goindex 33b2c5961b..f9c959dd6b 100644--- a/internal/service/dataset/ingestion_messages_test.go+++ b/internal/service/dataset/ingestion_messages_test.go@@ -113,6 +113,49 @@ func TestDatasetIngestionLogUsesEventStream(t *testing.T) {assertLatestEventMap(t, log, 1, "Knowledge compilation completed")}+func TestListSyncLogsReturnsOnlyCurrentlyLinkedDatasetConnectors(t *testing.T) {+ db := setupServiceTestDB(t)+ pushServiceDB(t, db)+ insertCompilationOwnerKB(t, "kb-1", "user-1")+ insertCompilationOwnerKB(t, "kb-2", "user-1")++ connectors := []entity.Connector{+ {ID: "connector-1", TenantID: "user-1", Name: "linked", Source: "rss", InputType: "poll", Config: entity.JSONMap{}, Status: string(entity.TaskStatusDone)},+ {ID: "connector-2", TenantID: "user-1", Name: "other", Source: "rss", InputType: "poll", Config: entity.JSONMap{}, Status: string(entity.TaskStatusDone)},+ }+ if err := db.Create(&connectors).Error; err != nil {+ t.Fatalf("insert connectors: %v", err)+ }+ if err := db.Create(&entity.Connector2Kb{ID: "connector-1-kb-1", ConnectorID: "connector-1", KbID: "kb-1", AutoParse: "1"}).Error; err != nil {+ t.Fatalf("link connector to kb-1: %v", err)+ }+ if err := db.Create(&entity.Connector2Kb{ID: "connector-2-kb-2", ConnectorID: "connector-2", KbID: "kb-2", AutoParse: "1"}).Error; err != nil {+ t.Fatalf("link connector to kb-2: %v", err)+ }++ logs := []entity.SyncLogs{+ {ID: "sync-kb-1", ConnectorID: "connector-1", KbID: "kb-1", TaskType: dao.TaskTypeSync, Status: dao.SyncStatusDone, ErrorMsg: ""},+ {ID: "sync-kb-2", ConnectorID: "connector-2", KbID: "kb-2", TaskType: dao.TaskTypeSync, Status: dao.SyncStatusDone, ErrorMsg: ""},+ // A stale/unlinked row must not appear in the dataset's configured connector logs.+ {ID: "sync-unlinked", ConnectorID: "connector-2", KbID: "kb-1", TaskType: dao.TaskTypeSync, Status: dao.SyncStatusDone, ErrorMsg: ""},+ }+ if err := db.Create(&logs).Error; err != nil {+ t.Fatalf("insert sync logs: %v", err)+ }++ result, code, err := NewDatasetService().ListSyncLogs(t.Context(), "kb-1", "user-1", 1, 30)+ if err != nil || code != common.CodeSuccess {+ t.Fatalf("ListSyncLogs = (%+v, %v, %v), want success", result, code, err)+ }+ if result["total"] != int64(1) {+ t.Fatalf("total = %#v, want 1", result["total"])+ }+ items, ok := result["logs"].([]*entity.ConnectorSyncLog)+ if !ok || len(items) != 1 || items[0].ID != "sync-kb-1" {+ t.Fatalf("logs = %#v, want only sync-kb-1", result["logs"])+ }+}+func TestListIngestionLogsIncludesPythonFileLogsAndStatusFilters(t *testing.T) {db := setupServiceTestDB(t)pushServiceDB(t, db)