Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(nlp): re-enable the datasetnav integration test in the integration CI job (#20740)

Caobaining18eca005563e3
diff --git a/.github/workflows/sep-tests.yml b/.github/workflows/sep-tests.yml
index 6f99f148cf..29961eb3c8 100644
--- a/.github/workflows/sep-tests.yml
+++ b/.github/workflows/sep-tests.yml
@@ -725,6 +725,14 @@ jobs:
run: |
set -euo pipefail
COMPOSE_FILE=docker/docker-compose.yml
+ # This teardown runs with `if: always()`. The prepare step exports
+ # COMPOSE_PROJECT_NAME, but if an earlier step failed and skipped the
+ # prepare step there is no stack to remove — and `set -u` would turn
+ # that clean no-op into a hard "unbound variable" error.
+ if [[ -z "${COMPOSE_PROJECT_NAME:-}" ]]; then
+ echo "COMPOSE_PROJECT_NAME is unset; nothing to tear down"
+ exit 0
+ fi
# Sometimes `docker compose down` fail due to hang container, heavy load etc. Need to remove such containers to release resources(for example, listen ports).
sudo docker compose -f "${COMPOSE_FILE}" -p ${COMPOSE_PROJECT_NAME} down -v || true
sudo docker ps -a --filter "label=com.docker.compose.project=${COMPOSE_PROJECT_NAME}" -q | xargs -r sudo docker rm -f
@@ -1254,6 +1262,14 @@ jobs:
run: |
set -euo pipefail
COMPOSE_FILE=docker/docker-compose.yml
+ # This teardown runs with `if: always()`. The prepare step exports
+ # COMPOSE_PROJECT_NAME, but if an earlier step failed and skipped the
+ # prepare step there is no stack to remove — and `set -u` would turn
+ # that clean no-op into a hard "unbound variable" error.
+ if [[ -z "${COMPOSE_PROJECT_NAME:-}" ]]; then
+ echo "COMPOSE_PROJECT_NAME is unset; nothing to tear down"
+ exit 0
+ fi
# Sometimes `docker compose down` fail due to hang container, heavy load etc. Need to remove such containers to release resources(for example, listen ports).
sudo docker compose -f "${COMPOSE_FILE}" -p ${COMPOSE_PROJECT_NAME} down -v || true
sudo docker ps -a --filter "label=com.docker.compose.project=${COMPOSE_PROJECT_NAME}" -q | xargs -r sudo docker rm -f
@@ -1825,16 +1841,10 @@ jobs:
# a fully-ready MinIO that this CI stack does not reliably provide
# (the compose /live probe passes while the S3 API still 502s), so they
# are exercised elsewhere, not here.
- # internal/service/nlp is also excluded: its DatasetNav integration test
- # compiles only against source fixes (Infinity 3013/3052, nav
- # available_int, doc_ids_kwd) that are intentionally out of scope for this
- # CI-enabling PR, so the package is left out of this job's package list
- # rather than patched. Those fixes land in their own PR.
PKGS=$(go list ./... \
| grep -v '/test/api' \
| grep -v '^ragflow/internal/deepdoc' \
- | grep -v '/internal/storage$' \
- | grep -v '^ragflow/internal/service/nlp$')
+ | grep -v '/internal/storage$')
./build.sh --test-integration-go $PKGS
- name: Tear down
@@ -1842,6 +1852,14 @@ jobs:
run: |
set -euo pipefail
COMPOSE_FILE=docker/docker-compose.yml
+ # This teardown runs with `if: always()`. The prepare step exports
+ # COMPOSE_PROJECT_NAME, but if an earlier step failed and skipped the
+ # prepare step there is no stack to remove — and `set -u` would turn
+ # that clean no-op into a hard "unbound variable" error.
+ if [[ -z "${COMPOSE_PROJECT_NAME:-}" ]]; then
+ echo "COMPOSE_PROJECT_NAME is unset; nothing to tear down"
+ exit 0
+ fi
sudo docker compose -f "${COMPOSE_FILE}" -p ${COMPOSE_PROJECT_NAME} down -v || true
sudo docker ps -a --filter "label=com.docker.compose.project=${COMPOSE_PROJECT_NAME}" -q | xargs -r sudo docker rm -f
# Release the per-port docker-network guards reserved by the prepare step.
diff --git a/internal/service/nlp/datasetnav_integration_test.go b/internal/service/nlp/datasetnav_integration_test.go
index a814334ca5..1f8567ac49 100644
--- a/internal/service/nlp/datasetnav_integration_test.go
+++ b/internal/service/nlp/datasetnav_integration_test.go
@@ -7,7 +7,6 @@ import (
"context"
"os"
"path/filepath"
- "strings"
"testing"
"ragflow/internal/common"
@@ -46,9 +45,17 @@ func findNavRow(t *testing.T, tenantID, kbID, docID string) map[string]interface
t.Skip("no live document engine")
}
idx := "ragflow_" + tenantID
+ // The nav row's role marker is type_kwd (nav_doc/nav_cluster). Since the #20587
+ // field-name refactor compile_kwd carries the real compilation kind
+ // ("tree"/"page_index"), so it is no longer the selector that scopes nav rows.
req := &types.SearchRequest{
- IndexNames: []string{idx},
- Filter: map[string]interface{}{"doc_id": []string{docID}, "compile_kwd": []string{"dataset_nav"}},
+ IndexNames: []string{idx},
+ // KbIDs is what lets the engine resolve the per-KB table on Infinity
+ // (ragflow_<tenant>_<kb>); without it the read matches no table and the
+ // row is never found. ES keys on the tenant index alone, so it is inert
+ // there.
+ KbIDs: []string{kbID},
+ Filter: map[string]interface{}{"doc_id": []string{docID}, "type_kwd": []string{"nav_doc"}},
SelectFields: []string{"available_int", "compile_kwd", "type_kwd"},
Limit: 10,
}
@@ -71,14 +78,10 @@ func findNavRow(t *testing.T, tenantID, kbID, docID string) map[string]interface
//
// Run with: bash build.sh --test-integration ./internal/service/nlp/...
//
-// Excluded from the integration CI job: this test exercises the DatasetNav flow
-// end-to-end against a live Infinity backend and surfaces several upstream source
-// bugs (Infinity 3013 on pure vector matches, 3052 on empty keyword filters,
-// doc_ids_kwd JSON encoding, NavService.Search available_int pinning,
-// findBestCluster empty-title descent). Those are genuine source fixes that belong
-// in their own PR — this PR only enables the integration-test CI job, so the
-// internal/service/nlp package is excluded from the job's package list until the
-// source fixes land. Re-enable once those are merged.
+// Runs in the integration CI job (infinity). The nav row's role marker is
+// type_kwd (nav_doc/nav_cluster); compile_kwd carries the real compilation kind
+// ("tree"/"page_index") since the #20587 field-name refactor, so the row is
+// located and asserted by type_kwd.
func TestDatasetNav_AvailableIntZero_Isolation(t *testing.T) {
if err := common.InitLogger("info", common.FileOutput{}, ""); err != nil {
t.Fatalf("init logger: %v", err)
@@ -115,12 +118,12 @@ func TestDatasetNav_AvailableIntZero_Isolation(t *testing.T) {
t.Fatal("NavService.Search returned no hits; nav row is not reachable")
}
- // The written nav row must carry compile_kwd=dataset_nav and available_int=0,
- // so the default retriever (available_int=1 filter) will not surface it.
+ // The written nav row must be a nav_doc with available_int=0, so the default
+ // retriever (available_int=1 filter) will not surface it.
row := findNavRow(t, tenantID, kbID, docID)
- // compile_kwd may come back list-wrapped by the engine, so use firstStrOrSlice.
- if ck := firstStrOrSlice(row["compile_kwd"]); !strings.Contains(ck, "dataset_nav") {
- t.Errorf("nav row compile_kwd = %q, want dataset_nav", ck)
+ // type_kwd may come back list-wrapped by the engine, so use firstStrOrSlice.
+ if tk := firstStrOrSlice(row["type_kwd"]); tk != "nav_doc" {
+ t.Errorf("nav row type_kwd = %q, want nav_doc", tk)
}
avail := intValue(row["available_int"])
if avail != 0 {