Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix: filter compilation status by kind (#19666)

buua436839a4ec815ac
diff --git a/internal/handler/compilation_status_test.go b/internal/handler/compilation_status_test.go
index b76b490685..1e13087ee2 100644
--- a/internal/handler/compilation_status_test.go
+++ b/internal/handler/compilation_status_test.go
@@ -101,6 +101,19 @@ func getCompilationStatus(t *testing.T, r *gin.Engine, datasetID string) (int, c
return resp.Code, body
}
+func getCompilationStatusWithKind(t *testing.T, r *gin.Engine, datasetID, kind string) (int, compilationStatusResponse) {
+ t.Helper()
+ resp := httptest.NewRecorder()
+ req := httptest.NewRequest(http.MethodGet,
+ "/api/v1/datasets/"+datasetID+"/compilation/status?kind="+url.QueryEscape(kind), nil)
+ r.ServeHTTP(resp, req)
+ var body compilationStatusResponse
+ if err := json.Unmarshal(resp.Body.Bytes(), &body); err != nil {
+ t.Fatalf("unmarshal response: %v body=%s", err, resp.Body.String())
+ }
+ return resp.Code, body
+}
+
// TestCompilationStatusHandler_NoRowIdle verifies a dataset with no scheduling
// row returns idle with zero counts.
func TestCompilationStatusHandler_NoRowIdle(t *testing.T) {
@@ -135,8 +148,8 @@ func TestCompilationStatusHandler_FullOutput(t *testing.T) {
row := entity.KnowledgeCompileDataset{
DatasetID: "kb-status-full",
TenantID: "user-1",
- BacklogDocIDs: `[{"doc_id":"d2","event_type":"completed","seq":2}]`,
- InflightDocIDs: `[{"doc_id":"d1","event_type":"completed","seq":1}]`,
+ BacklogDocIDs: `[{"doc_id":"d2","event_type":"completed","task_types":["Graph"]}]`,
+ InflightDocIDs: `[{"doc_id":"d1","event_type":"completed","task_types":["Graph"]}]`,
State: entity.DatasetStatePending,
ErrorMsg: "merge failed: boom",
}
@@ -144,7 +157,7 @@ func TestCompilationStatusHandler_FullOutput(t *testing.T) {
t.Fatalf("insert scheduling row: %v", err)
}
- status, body := getCompilationStatus(t, newCompilationStatusHandlerRouter(), "kb-status-full")
+ status, body := getCompilationStatusWithKind(t, newCompilationStatusHandlerRouter(), "kb-status-full", "graph")
if status != http.StatusOK {
t.Fatalf("status=%d want 200", status)
}
@@ -154,6 +167,9 @@ func TestCompilationStatusHandler_FullOutput(t *testing.T) {
if body.Data["state"] != entity.DatasetStatePending {
t.Fatalf("state=%v want pending", body.Data["state"])
}
+ if body.Data["kind"] != "graph" {
+ t.Fatalf("kind=%v want graph", body.Data["kind"])
+ }
if n, _ := body.Data["inflight"].(float64); n != 1 {
t.Fatalf("inflight=%v want 1", body.Data["inflight"])
}
diff --git a/internal/handler/dataset.go b/internal/handler/dataset.go
index d0f341cdab..9e766f563e 100644
--- a/internal/handler/dataset.go
+++ b/internal/handler/dataset.go
@@ -1021,7 +1021,8 @@ func (h *DatasetsHandler) AggregateTags(c *gin.Context) {
// GetCompilationStatus returns the dataset-level knowledge-compile lifecycle
// state (scheduler contract for API_PROXY_SCHEME=go/hybrid). It replaces the
-// Python-era TraceIndex task-progress endpoint for the Go backend.
+// Python-era TraceIndex task-progress endpoint for the Go backend. The optional
+// `kind` query parameter scopes the status to one compile type.
func (h *DatasetsHandler) GetCompilationStatus(c *gin.Context) {
user, errorCode, errorMessage := GetUser(c)
if errorCode != common.CodeSuccess {
@@ -1035,7 +1036,8 @@ func (h *DatasetsHandler) GetCompilationStatus(c *gin.Context) {
}
userID := strings.TrimSpace(user.ID)
ctx := c.Request.Context()
- st, code, err := h.datasetsService.GetDatasetCompilationStatus(ctx, userID, datasetID)
+ kind := strings.TrimSpace(c.Query("kind"))
+ st, code, err := h.datasetsService.GetDatasetCompilationStatus(ctx, userID, datasetID, kind)
if err != nil {
common.ErrorWithCode(c, code, err.Error())
return
diff --git a/internal/service/dataset/compilation_status.go b/internal/service/dataset/compilation_status.go
index 712ab4cc1d..6ce2a8f712 100644
--- a/internal/service/dataset/compilation_status.go
+++ b/internal/service/dataset/compilation_status.go
@@ -4,6 +4,7 @@ import (
"context"
"encoding/json"
"errors"
+ "strings"
"time"
"gorm.io/gorm"
@@ -23,6 +24,7 @@ import (
// so the frontend should test `error != ""` on its own (and hide the counts)
// rather than treating it as a peer of state.
type CompilationStatus struct {
+ Kind string `json:"kind"` // normalized requested compile kind
State string `json:"state"` // idle | pending | running | completed
Error string `json:"error,omitempty"` // most recent batch diagnostic (empty when none)
Progress float64 `json:"progress"` // current claimed batch progress
@@ -37,14 +39,19 @@ type CompilationStatus struct {
// GetDatasetCompilationStatus returns the scheduling-row lifecycle state for a
// dataset after verifying the calling user owns it. When no row exists the
// dataset has never had any compile work, so the state is idle.
-func (d *DatasetService) GetDatasetCompilationStatus(ctx context.Context, userID, datasetID string) (CompilationStatus, common.ErrorCode, error) {
+func (d *DatasetService) GetDatasetCompilationStatus(ctx context.Context, userID, datasetID, kind string) (CompilationStatus, common.ErrorCode, error) {
if datasetID == "" {
return CompilationStatus{}, common.CodeDataError, errors.New("dataset_id is required")
}
if !d.kbDAO.Accessible(ctx, dao.DB, datasetID, userID) {
return CompilationStatus{}, common.CodeDataError, errors.New("no authorization")
}
- st := CompilationStatus{State: entity.DatasetStateIdle}
+ requestedKind := strings.ToLower(strings.TrimSpace(kind))
+ normalizedKind := normalizeCompilationKind(requestedKind)
+ if requestedKind != "" && normalizedKind == "" {
+ return CompilationStatus{}, common.CodeDataError, errors.New("unsupported compilation kind")
+ }
+ st := CompilationStatus{Kind: requestedKind, State: entity.DatasetStateIdle}
db := dao.GetDB()
if db == nil {
return st, common.CodeSuccess, nil
@@ -67,13 +74,101 @@ func (d *DatasetService) GetDatasetCompilationStatus(ctx context.Context, userID
st.Progress = row.Progress
st.CurrentPhase = row.CurrentPhase
st.ProgressMsg = row.ProgressMsg
- st.Inflight = jsonArrayLen(row.InflightDocIDs)
- st.Backlog = jsonArrayLen(row.BacklogDocIDs)
+ if requestedKind != "" {
+ inflight, backlog := compilationStatusCounts(row.InflightDocIDs, row.BacklogDocIDs, normalizedKind)
+ if inflight == 0 && backlog == 0 && (row.InflightDocIDs != "" || row.BacklogDocIDs != "") {
+ st.State = entity.DatasetStateIdle
+ st.Error = ""
+ st.Progress = 0
+ st.CurrentPhase = ""
+ st.ProgressMsg = ""
+ }
+ st.Inflight = inflight
+ st.Backlog = backlog
+ } else {
+ st.Inflight = jsonArrayLen(row.InflightDocIDs)
+ st.Backlog = jsonArrayLen(row.BacklogDocIDs)
+ }
st.LastCompletedAt = row.LastCompletedAt
st.UpdatedAt = row.UpdatedAt
return st, common.CodeSuccess, nil
}
+type compilationStatusEntry struct {
+ Variants []string `json:"variants"`
+ TaskTypes []string `json:"task_types"`
+}
+
+func compilationStatusCounts(inflightJSON, backlogJSON, kind string) (int, int) {
+ return countCompilationStatusEntries(inflightJSON, kind), countCompilationStatusEntries(backlogJSON, kind)
+}
+
+func countCompilationStatusEntries(raw, kind string) int {
+ entries := parseCompilationStatusEntries(raw)
+ count := 0
+ for _, entry := range entries {
+ if compilationStatusEntryMatches(entry, kind) {
+ count++
+ }
+ }
+ return count
+}
+
+func parseCompilationStatusEntries(raw string) []compilationStatusEntry {
+ if raw == "" {
+ return nil
+ }
+ var entries []compilationStatusEntry
+ if err := json.Unmarshal([]byte(raw), &entries); err != nil {
+ return nil
+ }
+ return entries
+}
+
+func compilationStatusEntryMatches(entry compilationStatusEntry, kind string) bool {
+ for _, taskType := range entry.TaskTypes {
+ if normalizeCompilationKind(taskType) == kind {
+ return true
+ }
+ }
+ if len(entry.TaskTypes) > 0 {
+ return false
+ }
+ if len(entry.Variants) == 0 {
+ // Legacy entries without routing metadata can affect every product kind.
+ return true
+ }
+ for _, variant := range entry.Variants {
+ if normalizeCompilationKind(variant) == kind {
+ return true
+ }
+ if strings.EqualFold(strings.TrimSpace(variant), "structure") &&
+ (kind == "Graph" || kind == "Timeline" || kind == "PageIndex") {
+ return true
+ }
+ }
+ return false
+}
+
+func normalizeCompilationKind(kind string) string {
+ switch strings.ToLower(strings.TrimSpace(kind)) {
+ case "wiki", "artifact":
+ return "Wiki"
+ case "tree", "raptor":
+ return "Tree"
+ case "graph", "knowledge_graph", "knowledgegraph":
+ return "Graph"
+ case "mindmap", "mind_map":
+ return "Mindmap"
+ case "timeline":
+ return "Timeline"
+ case "pageindex", "page_index", "skill":
+ return "PageIndex"
+ default:
+ return ""
+ }
+}
+
// jsonArrayLen counts the top-level elements of a JSON array stored as TEXT.
// The *_doc_ids columns hold a `[]BacklogEntry` array ({doc_id,event_type,seq}),
// so each element is one scheduling entry (NOT a deduplicated doc). Empty or
diff --git a/internal/service/dataset/compilation_status_test.go b/internal/service/dataset/compilation_status_test.go
index 535391e81e..7120d5e3ca 100644
--- a/internal/service/dataset/compilation_status_test.go
+++ b/internal/service/dataset/compilation_status_test.go
@@ -93,7 +93,7 @@ func TestGetDatasetCompilationStatus_NoRowIsIdle(t *testing.T) {
insertCompilationOwnerKB(t, "kb-no-row", "user-1")
st, code, err := testCompilationStatusService().GetDatasetCompilationStatus(
- t.Context(), "user-1", "kb-no-row")
+ t.Context(), "user-1", "kb-no-row", "graph")
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
@@ -103,6 +103,9 @@ func TestGetDatasetCompilationStatus_NoRowIsIdle(t *testing.T) {
if st.State != entity.DatasetStateIdle {
t.Fatalf("state=%q want idle", st.State)
}
+ if st.Kind != "graph" {
+ t.Fatalf("kind=%q want graph", st.Kind)
+ }
if st.Inflight != 0 || st.Backlog != 0 {
t.Fatalf("expected zero counts for idle, got inflight=%d backlog=%d", st.Inflight, st.Backlog)
}
@@ -124,8 +127,8 @@ func TestGetDatasetCompilationStatus_FullOutput(t *testing.T) {
row := entity.KnowledgeCompileDataset{
DatasetID: "kb-full",
TenantID: "user-1",
- BacklogDocIDs: `[{"doc_id":"d3","event_type":"completed","seq":3}]`,
- InflightDocIDs: `[{"doc_id":"d1","event_type":"completed","seq":1},{"doc_id":"d2","event_type":"completed","seq":2}]`,
+ BacklogDocIDs: `[{"doc_id":"d3","event_type":"completed","task_types":["Timeline"]}]`,
+ InflightDocIDs: `[{"doc_id":"d1","event_type":"completed","task_types":["Graph"]},{"doc_id":"d2","event_type":"completed","task_types":["Graph"]}]`,
State: entity.DatasetStateRunning,
ErrorMsg: "merge failed: boom",
Progress: 0.72,
@@ -138,7 +141,7 @@ func TestGetDatasetCompilationStatus_FullOutput(t *testing.T) {
}
st, code, err := testCompilationStatusService().GetDatasetCompilationStatus(
- t.Context(), "user-1", "kb-full")
+ t.Context(), "user-1", "kb-full", "graph")
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
@@ -148,8 +151,11 @@ func TestGetDatasetCompilationStatus_FullOutput(t *testing.T) {
if st.State != entity.DatasetStateRunning {
t.Fatalf("state=%q want running", st.State)
}
- if st.Inflight != 2 || st.Backlog != 1 {
- t.Fatalf("want inflight=2 backlog=1, got inflight=%d backlog=%d", st.Inflight, st.Backlog)
+ if st.Kind != "graph" {
+ t.Fatalf("kind=%q want graph", st.Kind)
+ }
+ if st.Inflight != 2 || st.Backlog != 0 {
+ t.Fatalf("want inflight=2 backlog=0, got inflight=%d backlog=%d", st.Inflight, st.Backlog)
}
if st.Error != "merge failed: boom" {
t.Fatalf("error=%q want %q", st.Error, "merge failed: boom")
@@ -179,7 +185,7 @@ func TestGetDatasetCompilationStatus_Unauthorized(t *testing.T) {
}
st, code, err := testCompilationStatusService().GetDatasetCompilationStatus(
- t.Context(), "user-2", "kb-other")
+ t.Context(), "user-2", "kb-other", "graph")
if err == nil {
t.Fatalf("expected authorization error, got nil (status=%+v)", st)
}
@@ -192,7 +198,7 @@ func TestGetDatasetCompilationStatus_Unauthorized(t *testing.T) {
func TestGetDatasetCompilationStatus_EmptyID(t *testing.T) {
setupCompilationStatusTestDB(t)
_, code, err := testCompilationStatusService().GetDatasetCompilationStatus(
- t.Context(), "user-1", "")
+ t.Context(), "user-1", "", "graph")
if err == nil {
t.Fatal("expected error for empty dataset_id")
}