RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix: filter compilation status by kind (#19666)
buua436839a4ec815ac
diff --git a/internal/handler/compilation_status_test.go b/internal/handler/compilation_status_test.goindex b76b490685..1e13087ee2 100644--- a/internal/handler/compilation_status_test.go+++ b/internal/handler/compilation_status_test.go@@ -101,6 +101,19 @@ func getCompilationStatus(t *testing.T, r *gin.Engine, datasetID string) (int, creturn resp.Code, body}+func getCompilationStatusWithKind(t *testing.T, r *gin.Engine, datasetID, kind string) (int, compilationStatusResponse) {+ t.Helper()+ resp := httptest.NewRecorder()+ req := httptest.NewRequest(http.MethodGet,+ "/api/v1/datasets/"+datasetID+"/compilation/status?kind="+url.QueryEscape(kind), nil)+ r.ServeHTTP(resp, req)+ var body compilationStatusResponse+ if err := json.Unmarshal(resp.Body.Bytes(), &body); err != nil {+ t.Fatalf("unmarshal response: %v body=%s", err, resp.Body.String())+ }+ return resp.Code, body+}+// TestCompilationStatusHandler_NoRowIdle verifies a dataset with no scheduling// row returns idle with zero counts.func TestCompilationStatusHandler_NoRowIdle(t *testing.T) {@@ -135,8 +148,8 @@ func TestCompilationStatusHandler_FullOutput(t *testing.T) {row := entity.KnowledgeCompileDataset{DatasetID: "kb-status-full",TenantID: "user-1",- BacklogDocIDs: `[{"doc_id":"d2","event_type":"completed","seq":2}]`,- InflightDocIDs: `[{"doc_id":"d1","event_type":"completed","seq":1}]`,+ BacklogDocIDs: `[{"doc_id":"d2","event_type":"completed","task_types":["Graph"]}]`,+ InflightDocIDs: `[{"doc_id":"d1","event_type":"completed","task_types":["Graph"]}]`,State: entity.DatasetStatePending,ErrorMsg: "merge failed: boom",}@@ -144,7 +157,7 @@ func TestCompilationStatusHandler_FullOutput(t *testing.T) {t.Fatalf("insert scheduling row: %v", err)}- status, body := getCompilationStatus(t, newCompilationStatusHandlerRouter(), "kb-status-full")+ status, body := getCompilationStatusWithKind(t, newCompilationStatusHandlerRouter(), "kb-status-full", "graph")if status != http.StatusOK {t.Fatalf("status=%d want 200", status)}@@ -154,6 +167,9 @@ func TestCompilationStatusHandler_FullOutput(t *testing.T) {if body.Data["state"] != entity.DatasetStatePending {t.Fatalf("state=%v want pending", body.Data["state"])}+ if body.Data["kind"] != "graph" {+ t.Fatalf("kind=%v want graph", body.Data["kind"])+ }if n, _ := body.Data["inflight"].(float64); n != 1 {t.Fatalf("inflight=%v want 1", body.Data["inflight"])}diff --git a/internal/handler/dataset.go b/internal/handler/dataset.goindex d0f341cdab..9e766f563e 100644--- a/internal/handler/dataset.go+++ b/internal/handler/dataset.go@@ -1021,7 +1021,8 @@ func (h *DatasetsHandler) AggregateTags(c *gin.Context) {// GetCompilationStatus returns the dataset-level knowledge-compile lifecycle// state (scheduler contract for API_PROXY_SCHEME=go/hybrid). It replaces the-// Python-era TraceIndex task-progress endpoint for the Go backend.+// Python-era TraceIndex task-progress endpoint for the Go backend. The optional+// `kind` query parameter scopes the status to one compile type.func (h *DatasetsHandler) GetCompilationStatus(c *gin.Context) {user, errorCode, errorMessage := GetUser(c)if errorCode != common.CodeSuccess {@@ -1035,7 +1036,8 @@ func (h *DatasetsHandler) GetCompilationStatus(c *gin.Context) {}userID := strings.TrimSpace(user.ID)ctx := c.Request.Context()- st, code, err := h.datasetsService.GetDatasetCompilationStatus(ctx, userID, datasetID)+ kind := strings.TrimSpace(c.Query("kind"))+ st, code, err := h.datasetsService.GetDatasetCompilationStatus(ctx, userID, datasetID, kind)if err != nil {common.ErrorWithCode(c, code, err.Error())returndiff --git a/internal/service/dataset/compilation_status.go b/internal/service/dataset/compilation_status.goindex 712ab4cc1d..6ce2a8f712 100644--- a/internal/service/dataset/compilation_status.go+++ b/internal/service/dataset/compilation_status.go@@ -4,6 +4,7 @@ import ("context""encoding/json""errors"+ "strings""time""gorm.io/gorm"@@ -23,6 +24,7 @@ import (// so the frontend should test `error != ""` on its own (and hide the counts)// rather than treating it as a peer of state.type CompilationStatus struct {+ Kind string `json:"kind"` // normalized requested compile kindState string `json:"state"` // idle | pending | running | completedError string `json:"error,omitempty"` // most recent batch diagnostic (empty when none)Progress float64 `json:"progress"` // current claimed batch progress@@ -37,14 +39,19 @@ type CompilationStatus struct {// GetDatasetCompilationStatus returns the scheduling-row lifecycle state for a// dataset after verifying the calling user owns it. When no row exists the// dataset has never had any compile work, so the state is idle.-func (d *DatasetService) GetDatasetCompilationStatus(ctx context.Context, userID, datasetID string) (CompilationStatus, common.ErrorCode, error) {+func (d *DatasetService) GetDatasetCompilationStatus(ctx context.Context, userID, datasetID, kind string) (CompilationStatus, common.ErrorCode, error) {if datasetID == "" {return CompilationStatus{}, common.CodeDataError, errors.New("dataset_id is required")}if !d.kbDAO.Accessible(ctx, dao.DB, datasetID, userID) {return CompilationStatus{}, common.CodeDataError, errors.New("no authorization")}- st := CompilationStatus{State: entity.DatasetStateIdle}+ requestedKind := strings.ToLower(strings.TrimSpace(kind))+ normalizedKind := normalizeCompilationKind(requestedKind)+ if requestedKind != "" && normalizedKind == "" {+ return CompilationStatus{}, common.CodeDataError, errors.New("unsupported compilation kind")+ }+ st := CompilationStatus{Kind: requestedKind, State: entity.DatasetStateIdle}db := dao.GetDB()if db == nil {return st, common.CodeSuccess, nil@@ -67,13 +74,101 @@ func (d *DatasetService) GetDatasetCompilationStatus(ctx context.Context, userIDst.Progress = row.Progressst.CurrentPhase = row.CurrentPhasest.ProgressMsg = row.ProgressMsg- st.Inflight = jsonArrayLen(row.InflightDocIDs)- st.Backlog = jsonArrayLen(row.BacklogDocIDs)+ if requestedKind != "" {+ inflight, backlog := compilationStatusCounts(row.InflightDocIDs, row.BacklogDocIDs, normalizedKind)+ if inflight == 0 && backlog == 0 && (row.InflightDocIDs != "" || row.BacklogDocIDs != "") {+ st.State = entity.DatasetStateIdle+ st.Error = ""+ st.Progress = 0+ st.CurrentPhase = ""+ st.ProgressMsg = ""+ }+ st.Inflight = inflight+ st.Backlog = backlog+ } else {+ st.Inflight = jsonArrayLen(row.InflightDocIDs)+ st.Backlog = jsonArrayLen(row.BacklogDocIDs)+ }st.LastCompletedAt = row.LastCompletedAtst.UpdatedAt = row.UpdatedAtreturn st, common.CodeSuccess, nil}+type compilationStatusEntry struct {+ Variants []string `json:"variants"`+ TaskTypes []string `json:"task_types"`+}++func compilationStatusCounts(inflightJSON, backlogJSON, kind string) (int, int) {+ return countCompilationStatusEntries(inflightJSON, kind), countCompilationStatusEntries(backlogJSON, kind)+}++func countCompilationStatusEntries(raw, kind string) int {+ entries := parseCompilationStatusEntries(raw)+ count := 0+ for _, entry := range entries {+ if compilationStatusEntryMatches(entry, kind) {+ count+++ }+ }+ return count+}++func parseCompilationStatusEntries(raw string) []compilationStatusEntry {+ if raw == "" {+ return nil+ }+ var entries []compilationStatusEntry+ if err := json.Unmarshal([]byte(raw), &entries); err != nil {+ return nil+ }+ return entries+}++func compilationStatusEntryMatches(entry compilationStatusEntry, kind string) bool {+ for _, taskType := range entry.TaskTypes {+ if normalizeCompilationKind(taskType) == kind {+ return true+ }+ }+ if len(entry.TaskTypes) > 0 {+ return false+ }+ if len(entry.Variants) == 0 {+ // Legacy entries without routing metadata can affect every product kind.+ return true+ }+ for _, variant := range entry.Variants {+ if normalizeCompilationKind(variant) == kind {+ return true+ }+ if strings.EqualFold(strings.TrimSpace(variant), "structure") &&+ (kind == "Graph" || kind == "Timeline" || kind == "PageIndex") {+ return true+ }+ }+ return false+}++func normalizeCompilationKind(kind string) string {+ switch strings.ToLower(strings.TrimSpace(kind)) {+ case "wiki", "artifact":+ return "Wiki"+ case "tree", "raptor":+ return "Tree"+ case "graph", "knowledge_graph", "knowledgegraph":+ return "Graph"+ case "mindmap", "mind_map":+ return "Mindmap"+ case "timeline":+ return "Timeline"+ case "pageindex", "page_index", "skill":+ return "PageIndex"+ default:+ return ""+ }+}+// jsonArrayLen counts the top-level elements of a JSON array stored as TEXT.// The *_doc_ids columns hold a `[]BacklogEntry` array ({doc_id,event_type,seq}),// so each element is one scheduling entry (NOT a deduplicated doc). Empty ordiff --git a/internal/service/dataset/compilation_status_test.go b/internal/service/dataset/compilation_status_test.goindex 535391e81e..7120d5e3ca 100644--- a/internal/service/dataset/compilation_status_test.go+++ b/internal/service/dataset/compilation_status_test.go@@ -93,7 +93,7 @@ func TestGetDatasetCompilationStatus_NoRowIsIdle(t *testing.T) {insertCompilationOwnerKB(t, "kb-no-row", "user-1")st, code, err := testCompilationStatusService().GetDatasetCompilationStatus(- t.Context(), "user-1", "kb-no-row")+ t.Context(), "user-1", "kb-no-row", "graph")if err != nil {t.Fatalf("unexpected error: %v", err)}@@ -103,6 +103,9 @@ func TestGetDatasetCompilationStatus_NoRowIsIdle(t *testing.T) {if st.State != entity.DatasetStateIdle {t.Fatalf("state=%q want idle", st.State)}+ if st.Kind != "graph" {+ t.Fatalf("kind=%q want graph", st.Kind)+ }if st.Inflight != 0 || st.Backlog != 0 {t.Fatalf("expected zero counts for idle, got inflight=%d backlog=%d", st.Inflight, st.Backlog)}@@ -124,8 +127,8 @@ func TestGetDatasetCompilationStatus_FullOutput(t *testing.T) {row := entity.KnowledgeCompileDataset{DatasetID: "kb-full",TenantID: "user-1",- BacklogDocIDs: `[{"doc_id":"d3","event_type":"completed","seq":3}]`,- InflightDocIDs: `[{"doc_id":"d1","event_type":"completed","seq":1},{"doc_id":"d2","event_type":"completed","seq":2}]`,+ BacklogDocIDs: `[{"doc_id":"d3","event_type":"completed","task_types":["Timeline"]}]`,+ InflightDocIDs: `[{"doc_id":"d1","event_type":"completed","task_types":["Graph"]},{"doc_id":"d2","event_type":"completed","task_types":["Graph"]}]`,State: entity.DatasetStateRunning,ErrorMsg: "merge failed: boom",Progress: 0.72,@@ -138,7 +141,7 @@ func TestGetDatasetCompilationStatus_FullOutput(t *testing.T) {}st, code, err := testCompilationStatusService().GetDatasetCompilationStatus(- t.Context(), "user-1", "kb-full")+ t.Context(), "user-1", "kb-full", "graph")if err != nil {t.Fatalf("unexpected error: %v", err)}@@ -148,8 +151,11 @@ func TestGetDatasetCompilationStatus_FullOutput(t *testing.T) {if st.State != entity.DatasetStateRunning {t.Fatalf("state=%q want running", st.State)}- if st.Inflight != 2 || st.Backlog != 1 {- t.Fatalf("want inflight=2 backlog=1, got inflight=%d backlog=%d", st.Inflight, st.Backlog)+ if st.Kind != "graph" {+ t.Fatalf("kind=%q want graph", st.Kind)+ }+ if st.Inflight != 2 || st.Backlog != 0 {+ t.Fatalf("want inflight=2 backlog=0, got inflight=%d backlog=%d", st.Inflight, st.Backlog)}if st.Error != "merge failed: boom" {t.Fatalf("error=%q want %q", st.Error, "merge failed: boom")@@ -179,7 +185,7 @@ func TestGetDatasetCompilationStatus_Unauthorized(t *testing.T) {}st, code, err := testCompilationStatusService().GetDatasetCompilationStatus(- t.Context(), "user-2", "kb-other")+ t.Context(), "user-2", "kb-other", "graph")if err == nil {t.Fatalf("expected authorization error, got nil (status=%+v)", st)}@@ -192,7 +198,7 @@ func TestGetDatasetCompilationStatus_Unauthorized(t *testing.T) {func TestGetDatasetCompilationStatus_EmptyID(t *testing.T) {setupCompilationStatusTestDB(t)_, code, err := testCompilationStatusService().GetDatasetCompilationStatus(- t.Context(), "user-1", "")+ t.Context(), "user-1", "", "graph")if err == nil {t.Fatal("expected error for empty dataset_id")}