Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

chore(deepdoc): source native testdata from infiniflow/ragflow-testdata (#19136)

xugangqiang00c3904d0b5c
diff --git a/.github/workflows/sep-tests.yml b/.github/workflows/sep-tests.yml
index b4f4db16bb..609d34e0ac 100644
--- a/.github/workflows/sep-tests.yml
+++ b/.github/workflows/sep-tests.yml
@@ -1580,7 +1580,7 @@ jobs:
- name: Cache external DeepDoc testdata
# The native backend tests fetch pinned test assets from
- # xugangqiang/ragflow-testdata (see scripts/fetch_deepdoc_testdata.sh).
+ # infiniflow/ragflow-testdata (see scripts/fetch_deepdoc_testdata.sh).
# Cache them across runs, keyed by the pinned ref recorded in
# internal/deepdoc/native/testdata.ref so a ref bump invalidates the cache.
uses: actions/cache@v4
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
deleted file mode 100644
index c7ca472306..0000000000
--- a/.github/workflows/tests.yml
+++ /dev/null
@@ -1,1181 +0,0 @@
-name: tests
-permissions:
- contents: read
-
-on:
- push:
- branches:
- - 'main'
- - '*.*.*'
- paths-ignore:
- - 'docs/**'
- - '*.md'
- - '*.mdx'
- # The only difference between pull_request and pull_request_target is the context in which the workflow runs:
- # — pull_request_target workflows use the workflow files from the default branch, and secrets are available.
- # — pull_request workflows use the workflow files from the pull request branch, and secrets are unavailable.
- pull_request:
- types: [opened, synchronize, reopened, ready_for_review, labeled]
- paths-ignore:
- - 'docs/**'
- - '*.md'
- - '*.mdx'
- schedule:
- - cron: '0 16 * * *' # This schedule runs every 16:00:00Z(00:00:00+08:00)
-
-# https://docs.github.com/en/actions/using-jobs/using-concurrency
-concurrency:
- group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
- cancel-in-progress: true
-
-jobs:
- ragflow_preflight:
- name: ragflow_preflight
- # https://docs.github.com/en/actions/using-jobs/using-conditions-to-control-job-execution
- # https://github.com/orgs/community/discussions/26261
- if: ${{ github.event_name != 'pull_request' || (github.event.pull_request.draft == false && contains(github.event.pull_request.labels.*.name, 'ci') && (github.event.action != 'labeled' || github.event.label.name == 'ci')) }}
- runs-on: [ "self-hosted", "ragflow-test" ]
- outputs:
- http_api_test_level: ${{ steps.test_level.outputs.http_api_test_level }}
- steps:
- - name: Ensure workspace ownership
- run: |
- echo "Workflow triggered by ${{ github.event_name }}"
- echo "chown -R ${USER} ${GITHUB_WORKSPACE}" && sudo chown -R ${USER} ${GITHUB_WORKSPACE}
-
- # https://github.com/actions/checkout/issues/1781
- - name: Check out code
- uses: actions/checkout@v6
- with:
- ref: ${{ (github.event_name == 'pull_request' || github.event_name == 'pull_request_target') && format('refs/pull/{0}/merge', github.event.pull_request.number) || github.sha }}
- fetch-depth: 0
- fetch-tags: true
-
- - name: Check workflow duplication
- if: ${{ !cancelled() && !failure() }}
- run: |
- if [[ ${GITHUB_EVENT_NAME} != "pull_request" && ${GITHUB_EVENT_NAME} != "schedule" ]]; then
- HEAD=$(git rev-parse HEAD)
- # Find a PR that introduced a given commit
- gh auth login --with-token <<< "${{ secrets.GITHUB_TOKEN }}"
- PR_NUMBER=$(gh pr list --search ${HEAD} --state merged --json number --jq .[0].number)
- echo "HEAD=${HEAD}"
- echo "PR_NUMBER=${PR_NUMBER}"
- if [[ -n "${PR_NUMBER}" ]]; then
- PR_SHA_FP=${RUNNER_WORKSPACE_PREFIX}/artifacts/${GITHUB_REPOSITORY}/PR_${PR_NUMBER}
- if [[ -f "${PR_SHA_FP}" ]]; then
- read -r PR_SHA PR_RUN_ID < "${PR_SHA_FP}"
- # Calculate the hash of the current workspace content
- HEAD_SHA=$(git rev-parse HEAD^{tree})
- if [[ "${HEAD_SHA}" == "${PR_SHA}" ]]; then
- echo "Cancel myself since the workspace content hash is the same with PR #${PR_NUMBER} merged. See ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${PR_RUN_ID} for details."
- gh run cancel ${GITHUB_RUN_ID}
- while true; do
- status=$(gh run view ${GITHUB_RUN_ID} --json status -q .status)
- [ "${status}" = "completed" ] && break
- sleep 5
- done
- exit 1
- fi
- fi
- fi
- elif [[ ${GITHUB_EVENT_NAME} == "pull_request" ]]; then
- PR_NUMBER=${{ github.event.pull_request.number }}
- PR_SHA_FP=${RUNNER_WORKSPACE_PREFIX}/artifacts/${GITHUB_REPOSITORY}/PR_${PR_NUMBER}
- # Calculate the hash of the current workspace content
- PR_SHA=$(git rev-parse HEAD^{tree})
- echo "PR #${PR_NUMBER} workspace content hash: ${PR_SHA}"
- mkdir -p ${RUNNER_WORKSPACE_PREFIX}/artifacts/${GITHUB_REPOSITORY}
- echo "${PR_SHA} ${GITHUB_RUN_ID}" > ${PR_SHA_FP}
- fi
- ARTIFACTS_DIR=${RUNNER_WORKSPACE_PREFIX}/artifacts/${GITHUB_REPOSITORY}/${GITHUB_RUN_ID}
- echo "ARTIFACTS_DIR=${ARTIFACTS_DIR}" >> ${GITHUB_ENV}
- rm -rf ${ARTIFACTS_DIR} && mkdir -p ${ARTIFACTS_DIR}
-
- - name: Run Lefthook on changed files
- run: |
- set -euo pipefail
- if [[ "${GITHUB_EVENT_NAME}" == "pull_request" || "${GITHUB_EVENT_NAME}" == "pull_request_target" ]]; then
- changed_files=$(mktemp)
- trap 'rm -f "$changed_files"' EXIT
- git diff --name-only -z ${{ github.event.pull_request.base.sha }}...${{ github.event.pull_request.head.sha }} \
- | while IFS= read -r -d '' file; do
- if [[ -f "$file" ]]; then
- printf '%s\0' "$file"
- fi
- done > "$changed_files"
- echo "Changed files to run lefthook on:"
- if [[ -s "$changed_files" ]]; then
- while IFS= read -r -d '' file; do
- printf ' %q\n' "$file"
- done < "$changed_files"
- else
- echo " (none — lefthook will be a no-op)"
- fi
- # LEFTHOOK_CHECK_ONLY=1 makes the pre-commit jobs verify without
- # applying --fix or `git add`, so CI only checks and reports
- # failures instead of rewriting the working tree.
- LEFTHOOK_CHECK_ONLY=1 lefthook run pre-commit --files-from-stdin --no-auto-install < "$changed_files"
- fi
-
- - name: Set test level
- id: test_level
- run: |
- set -euo pipefail
- if [[ ${GITHUB_EVENT_NAME} == "schedule" ]]; then
- export HTTP_API_TEST_LEVEL=p3
- else
- export HTTP_API_TEST_LEVEL=p2
- fi
- echo "HTTP_API_TEST_LEVEL=${HTTP_API_TEST_LEVEL}" >> ${GITHUB_ENV}
- echo "http_api_test_level=${HTTP_API_TEST_LEVEL}" >> ${GITHUB_OUTPUT}
-
- - name: Prepare Python test environment
- run: |
- git config --global url."https://oauth2:${{ secrets.GITEE_TOKEN }}@gitee.com/".insteadOf "https://gitee.com/"
- uv sync --python 3.13 --group test --frozen
- uv pip install -e sdk/python
-
- - name: Run unit test
- run: |
- source .venv/bin/activate
- which pytest || echo "pytest not in PATH"
- echo "Start to run unit test"
- python3 run_tests.py -i
-
-
-
- ragflow_tests_infinity:
- name: ragflow_tests_infinity
- needs: ragflow_preflight
- if: ${{ github.event_name != 'pull_request' || (github.event.pull_request.draft == false && contains(github.event.pull_request.labels.*.name, 'ci') && (github.event.action != 'labeled' || github.event.label.name == 'ci')) }}
- runs-on: [ "self-hosted", "ragflow-test" ]
- env:
- DOC_ENGINE: infinity
- RAGFLOW_IMAGE: infiniflow/ragflow:${{ github.run_id }}-infinity
- HTTP_API_TEST_LEVEL: ${{ needs.ragflow_preflight.outputs.http_api_test_level }}
- steps:
- - name: Ensure workspace ownership
- run: |
- echo "Workflow triggered by ${{ github.event_name }}"
- echo "chown -R ${USER} ${GITHUB_WORKSPACE}" && sudo chown -R ${USER} ${GITHUB_WORKSPACE}
-
- - name: Check out code
- uses: actions/checkout@v6
- with:
- ref: ${{ (github.event_name == 'pull_request' || github.event_name == 'pull_request_target') && format('refs/pull/{0}/merge', github.event.pull_request.number) || github.sha }}
- fetch-depth: 0
- fetch-tags: true
-
- - name: Build ragflow go server
- run: |
- set -euo pipefail
- BUILDER_CONTAINER=ragflow_build_${GITHUB_RUN_ID}_${DOC_ENGINE}_$(od -An -N4 -tx4 /dev/urandom | tr -d ' ')
- cleanup_builder() {
- if [[ -n "${BUILDER_CONTAINER:-}" ]]; then
- sudo docker rm -f -v "${BUILDER_CONTAINER}" >/dev/null 2>&1 || true
- fi
- }
- trap cleanup_builder EXIT
-
- TZ=${TZ:-$(readlink -f /etc/localtime | awk -F '/zoneinfo/' '{print $2}')}
- sudo docker run --privileged -d --name "${BUILDER_CONTAINER}" \
- -e TZ="${TZ}" \
- -e UV_INDEX=https://mirrors.aliyun.com/pypi/simple \
- -v "${PWD}:/ragflow" \
- -v "${PWD}/internal/binding/cpp/resource:/usr/share/infinity/resource" \
- infiniflow/infinity_builder:ubuntu22_clang20
- sudo docker exec "${BUILDER_CONTAINER}" bash -c 'git config --global safe.directory "*" && cd /ragflow && ./build.sh --cpp'
- ./build.sh --go
-
- - name: Run Go unit tests
- # Runs after `./build.sh --go`, which guarantees the C++ static
- # library (librag_tokenizer_c_api.a) is present on disk. The Go
- # test binaries link against it transitively through
- # `internal/binding`, so running `go test` before the C++ build
- # fails the link step.
- #
- # Excludes packages whose tests fail for environmental reasons
- # unrelated to the diff:
- # - internal/tokenizer is split: the pure-Go BPE loader tests
- # (bpe_loader_test.go) run in the default tier; the C++ binding /
- # dict-dependent tests (tokenizer_test.go,
- # tokenizer_concurrent_test.go) and the on-disk anchor test
- # (bpe_loader_anchors_test.go) are tagged `manual` and need the
- # docker builder's /usr/share/infinity/resource, so they stay out
- # of the default run.
- run: |
- set -euo pipefail
-
- # Provide the cl100k BPE table for the offline tokenizer loader.
- # The loader reads <repo>/ragflow_deps/cl100k_base.tiktoken (or
- # TIKTOKEN_CACHE_DIR). Fetch it directly: this CI runner can reach
- # the upstream blob (pre-PR tests used tiktoken-go's stock network
- # loader and passed), so a direct download is small and fast. As a
- # fallback, copy from the infiniflow/ragflow_deps image only if it is
- # already cached locally (we do NOT pull the whole image just for one
- # 1.6MB file). If neither works, the loader still fails loudly (no
- # silent 0-token degradation). The loader itself stays offline.
- if [ ! -f ragflow_deps/cl100k_base.tiktoken ]; then
- mkdir -p ragflow_deps
- if command -v curl >/dev/null 2>&1; then
- if curl -fsSL -o ragflow_deps/cl100k_base.tiktoken \
- https://openaipublic.blob.core.windows.net/encodings/cl100k_base.tiktoken; then
- echo "tiktoken: cl100k table fetched via curl from openai blob"
- else
- echo "tiktoken: curl fetch failed (network/GFW); will try cached image"
- fi
- fi
- if [ ! -f ragflow_deps/cl100k_base.tiktoken ] && docker image inspect infiniflow/ragflow_deps:latest >/dev/null 2>&1; then
- CID=$(docker create infiniflow/ragflow_deps:latest true) || true
- if [ -n "${CID:-}" ]; then
- if docker cp "$CID":/cl100k_base.tiktoken ragflow_deps/cl100k_base.tiktoken 2>/dev/null; then
- echo "tiktoken: cl100k table copied from cached infiniflow/ragflow_deps image"
- fi
- docker rm -f "$CID" >/dev/null 2>&1 || true
- fi
- fi
- fi
- if [ -f ragflow_deps/cl100k_base.tiktoken ]; then
- echo "tiktoken: cl100k table provisioned ($(wc -c < ragflow_deps/cl100k_base.tiktoken) bytes)"
- else
- echo "tiktoken: cl100k table NOT provisioned — offline loader will fail loudly"
- fi
-
- PKGS=$(go list ./... 2>/dev/null \
- | grep -v '/internal/storage$' \
- | grep -v '/internal/agent$' \
- | grep -v '/internal/handler$' || true)
- if [ -z "$PKGS" ]; then
- ./build.sh --test
- else
- ./build.sh --test -- $PKGS
- fi
-
- - name: Build ragflow:nightly
- run: |
- set -euo pipefail
- GITEE_TOKEN_FILE="$(mktemp)"
- printf '%s' '${{ secrets.GITEE_TOKEN }}' > "${GITEE_TOKEN_FILE}"
- sudo docker pull ubuntu:24.04
- sudo DOCKER_BUILDKIT=1 docker build --build-arg NEED_MIRROR=1 --build-arg HTTPS_PROXY=${HTTPS_PROXY} --build-arg HTTP_PROXY=${HTTP_PROXY} --secret id=gitee_token,src=${GITEE_TOKEN_FILE} -f Dockerfile -t ${RAGFLOW_IMAGE} .
- rm -f "${GITEE_TOKEN_FILE}"
-
- - name: Prepare Python test environment
- run: |
- git config --global url."https://oauth2:${{ secrets.GITEE_TOKEN }}@gitee.com/".insteadOf "https://gitee.com/"
- uv sync --python 3.13 --group test --frozen
- uv pip install -e sdk/python
-
- - name: Prepare function test environment
- working-directory: docker
- run: |
- set -euo pipefail
- # install ss
- sudo apt update && sudo apt install -y iproute2
- RUNNER_WORKSPACE_PREFIX=${RUNNER_WORKSPACE_PREFIX:-${HOME}}
- COMPOSE_PROJECT_NAME="${GITHUB_RUN_ID}-${DOC_ENGINE}"
- echo "COMPOSE_PROJECT_NAME=${COMPOSE_PROJECT_NAME}" >> ${GITHUB_ENV}
- echo "RAGFLOW_CONTAINER=${COMPOSE_PROJECT_NAME}-ragflow-cpu-1" >> ${GITHUB_ENV}
- ARTIFACTS_DIR=${RUNNER_WORKSPACE_PREFIX}/artifacts/${GITHUB_REPOSITORY}/${GITHUB_RUN_ID}/${DOC_ENGINE}
- echo "ARTIFACTS_DIR=${ARTIFACTS_DIR}" >> ${GITHUB_ENV}
- rm -rf "${ARTIFACTS_DIR}" && mkdir -p "${ARTIFACTS_DIR}"
-
- # Determine runner number (default to 1 if not found)
- RUNNER_NUM=$(sudo docker inspect $(hostname) --format '{{index .Config.Labels "com.docker.compose.container-number"}}' 2>/dev/null || true)
- RUNNER_NUM=${RUNNER_NUM:-1}
-
- # Engine-specific offset partitions keep concurrent engine jobs from
- # choosing the same host ports when they land on the same self-hosted runner.
- # A lock plus reservation file closes the check/start race between parallel jobs.
- PORT_BASES=(1200 1201 23817 23820 5432 5455 9000 9001 6379 6380 6601 9380 9381 9382 9384 9383 9385 80 443 4222)
- PARTITION_SIZE=6000
- case "${DOC_ENGINE}" in
- elasticsearch) PARTITION_BASE=1000 ;;
- infinity) PARTITION_BASE=31000 ;;
- *) echo "Unsupported DOC_ENGINE=${DOC_ENGINE}" >&2; exit 1 ;;
- esac
- PORT_LOCK_DIR=${RUNNER_WORKSPACE_PREFIX}/artifacts/${GITHUB_REPOSITORY}/port-locks
- mkdir -p "${PORT_LOCK_DIR}"
-
- port_offset_available() {
- local offset=$1
- local base port
- for base in "${PORT_BASES[@]}"; do
- port=$((base + offset))
- if ss -ltnH "sport = :${port}" | grep -q .; then
- return 1
- fi
- done
- return 0
- }
-
- cleanup_stale_port_locks() {
- local now stale_after lock lock_ts
- now=$(date -u +%s)
- stale_after=$((6 * 60 * 60))
- for lock in "${PORT_LOCK_DIR}"/*.lock; do
- [[ -e "${lock}" ]] || continue
- lock_ts=$(awk '{print $3}' "${lock}" 2>/dev/null || true)
- if [[ "${lock_ts}" =~ ^[0-9]+$ ]] && (( now - lock_ts > stale_after )); then
- rm -f "${lock}"
- fi
- done
- }
-
- reserve_port_offset() {
- local attempt candidate reservation
- cleanup_stale_port_locks
- for attempt in $(seq 0 59); do
- candidate=$(( PARTITION_BASE + ((GITHUB_RUN_ID + RUNNER_NUM * 1000 + attempt * 97) % PARTITION_SIZE) ))
- reservation="${PORT_LOCK_DIR}/${candidate}.lock"
- if ( set -o noclobber; echo "${GITHUB_RUN_ID} ${DOC_ENGINE} $(date -u +%s)" > "${reservation}" ) 2>/dev/null; then
- if port_offset_available "${candidate}"; then
- PORT_OFFSET=${candidate}
- PORT_RESERVATION=${reservation}
- return 0
- fi
- rm -f "${reservation}"
- fi
- done
- return 1
- }
-
- if ! reserve_port_offset; then
- echo "Failed to reserve a free host port range for ${DOC_ENGINE} docker compose" >&2
- exit 1
- fi
- echo "PORT_RESERVATION=${PORT_RESERVATION}" >> ${GITHUB_ENV}
- echo "Using ${DOC_ENGINE} host port offset ${PORT_OFFSET}"
- ES_PORT=$((1200 + PORT_OFFSET))
- OS_PORT=$((1201 + PORT_OFFSET))
- INFINITY_THRIFT_PORT=$((23817 + PORT_OFFSET))
- INFINITY_HTTP_PORT=$((23820 + PORT_OFFSET))
- INFINITY_PSQL_PORT=$((5432 + PORT_OFFSET))
- EXPOSE_MYSQL_PORT=$((5455 + PORT_OFFSET))
- MINIO_PORT=$((9000 + PORT_OFFSET))
- MINIO_CONSOLE_PORT=$((9001 + PORT_OFFSET))
- REDIS_PORT=$((6379 + PORT_OFFSET))
- NATS_PORT=$((4222 + PORT_OFFSET))
- TEI_PORT=$((6380 + PORT_OFFSET))
- KIBANA_PORT=$((6601 + PORT_OFFSET))
- SVR_HTTP_PORT=$((9380 + PORT_OFFSET))
- ADMIN_SVR_HTTP_PORT=$((9381 + PORT_OFFSET))
- SVR_MCP_PORT=$((9382 + PORT_OFFSET))
- GO_HTTP_PORT=$((9384 + PORT_OFFSET))
- GO_ADMIN_PORT=$((9383 + PORT_OFFSET))
- SANDBOX_EXECUTOR_MANAGER_PORT=$((9385 + PORT_OFFSET))
- SVR_WEB_HTTP_PORT=$((80 + PORT_OFFSET))
- SVR_WEB_HTTPS_PORT=$((443 + PORT_OFFSET))
-
- # Persist computed ports into .env so docker-compose uses the correct host bindings.
- # Remove previous CI overrides first; docker compose uses the last duplicate key.
- sed -i '/^ES_PORT=/d;/^OS_PORT=/d;/^INFINITY_THRIFT_PORT=/d;/^INFINITY_HTTP_PORT=/d;/^INFINITY_PSQL_PORT=/d;/^EXPOSE_MYSQL_PORT=/d;/^MINIO_PORT=/d;/^MINIO_CONSOLE_PORT=/d;/^REDIS_PORT=/d;/^TEI_PORT=/d;/^KIBANA_PORT=/d;/^SVR_HTTP_PORT=/d;/^ADMIN_SVR_HTTP_PORT=/d;/^SVR_MCP_PORT=/d;/^GO_HTTP_PORT=/d;/^GO_ADMIN_PORT=/d;/^SANDBOX_EXECUTOR_MANAGER_PORT=/d;/^SVR_WEB_HTTP_PORT=/d;/^SVR_WEB_HTTPS_PORT=/d;/^NATS_PORT=/d;/^COMPOSE_PROFILES=/d;/^TEI_MODEL=/d;/^RAGFLOW_IMAGE=/d;/^DOC_ENGINE=/d' .env
- {
- echo ""
- echo "ES_PORT=${ES_PORT}"
- echo "OS_PORT=${OS_PORT}"
- echo "INFINITY_THRIFT_PORT=${INFINITY_THRIFT_PORT}"
- echo "INFINITY_HTTP_PORT=${INFINITY_HTTP_PORT}"
- echo "INFINITY_PSQL_PORT=${INFINITY_PSQL_PORT}"
- echo "EXPOSE_MYSQL_PORT=${EXPOSE_MYSQL_PORT}"
- echo "MINIO_PORT=${MINIO_PORT}"
- echo "MINIO_CONSOLE_PORT=${MINIO_CONSOLE_PORT}"
- echo "REDIS_PORT=${REDIS_PORT}"
- echo "NATS_PORT=${NATS_PORT}"
- echo "TEI_PORT=${TEI_PORT}"
- echo "KIBANA_PORT=${KIBANA_PORT}"
- echo "SVR_HTTP_PORT=${SVR_HTTP_PORT}"
- echo "ADMIN_SVR_HTTP_PORT=${ADMIN_SVR_HTTP_PORT}"
- echo "SVR_MCP_PORT=${SVR_MCP_PORT}"
- echo "GO_HTTP_PORT=${GO_HTTP_PORT}"
- echo "GO_ADMIN_PORT=${GO_ADMIN_PORT}"
- echo "SANDBOX_EXECUTOR_MANAGER_PORT=${SANDBOX_EXECUTOR_MANAGER_PORT}"
- echo "SVR_WEB_HTTP_PORT=${SVR_WEB_HTTP_PORT}"
- echo "SVR_WEB_HTTPS_PORT=${SVR_WEB_HTTPS_PORT}"
- echo "COMPOSE_PROFILES=${DOC_ENGINE},cpu,tei-cpu,deepdoc,metadata-mysql"
- echo "TEI_MODEL=BAAI/bge-small-en-v1.5"
- echo "RAGFLOW_IMAGE=${RAGFLOW_IMAGE}"
- echo "DOC_ENGINE=${DOC_ENGINE}"
- } >> .env
- echo "HOST_ADDRESS=http://host.docker.internal:${SVR_HTTP_PORT}" >> ${GITHUB_ENV}
-
- # Patch entrypoint.sh for coverage
- sed -i '/"\$PY" api\/ragflow_server.py \${INIT_SUPERUSER_ARGS} &/c\ echo "Ensuring coverage is installed..."\n "$PY" -m pip install coverage -i https://mirrors.aliyun.com/pypi/simple\n export COVERAGE_FILE=/ragflow/logs/.coverage\n echo "Starting ragflow_server with coverage..."\n "$PY" -m coverage run --source=./api/apps --omit="*/tests/*,*/migrations/*" -a api/ragflow_server.py ${INIT_SUPERUSER_ARGS} &' ./entrypoint.sh
-
-
- - name: Start ragflow:nightly for Infinity
- run: |
- sudo docker compose -f docker/docker-compose.yml -p ${COMPOSE_PROJECT_NAME} down -v || true
- sudo docker ps -a --filter "label=com.docker.compose.project=${COMPOSE_PROJECT_NAME}" -q | xargs -r sudo docker rm -f
- sudo docker compose -f docker/docker-compose.yml -p ${COMPOSE_PROJECT_NAME} up -d
-
- - name: Run sdk tests against Infinity
- run: |
- export http_proxy=""; export https_proxy=""; export no_proxy=""; export HTTP_PROXY=""; export HTTPS_PROXY=""; export NO_PROXY=""
- svc_ready=0
- for i in $(seq 1 60); do
- if sudo docker exec ${RAGFLOW_CONTAINER} curl -sf --connect-timeout 5 "${HOST_ADDRESS}/api/v1/system/ping" > /dev/null 2>&1; then
- svc_ready=1
- break
- fi
- echo "Waiting for service to be available... ($i/60)"
- sleep 5
- done
- if [ "$svc_ready" -ne 1 ]; then
- echo "Service did not become ready after 5 minutes. Docker logs:"
- sudo docker logs ${RAGFLOW_CONTAINER}
- exit 1
- fi
- echo "Start to run test sdk on Infinity"
- source .venv/bin/activate && set -o pipefail; DOC_ENGINE=infinity pytest -s --tb=short --level=${HTTP_API_TEST_LEVEL} --junitxml=pytest-infinity-sdk.xml --cov=sdk/python/ragflow_sdk --cov-branch --cov-report=xml:coverage-infinity-sdk.xml test/testcases/test_sdk_api 2>&1 | tee infinity_sdk_test.log
-
- - name: Run New RESTFUL api tests against Infinity
- run: |
- export http_proxy=""; export https_proxy=""; export no_proxy=""; export HTTP_PROXY=""; export HTTPS_PROXY=""; export NO_PROXY=""
- svc_ready=0
- for i in $(seq 1 60); do
- if sudo docker exec ${RAGFLOW_CONTAINER} curl -sf --connect-timeout 5 "${HOST_ADDRESS}/api/v1/system/ping" > /dev/null 2>&1; then
- svc_ready=1
- break
- fi
- echo "Waiting for service to be available... ($i/60)"
- sleep 5
- done
- if [ "$svc_ready" -ne 1 ]; then
- echo "Service did not become ready after 5 minutes. Docker logs:"
- sudo docker logs ${RAGFLOW_CONTAINER}
- exit 1
- fi
- source .venv/bin/activate && set -o pipefail; DOC_ENGINE=infinity pytest -s --tb=short --level=${HTTP_API_TEST_LEVEL} test/testcases/restful_api 2>&1 | tee infinity_restful_api_test.log
-
- - name: RAGFlow CLI retrieval test Infinity
- env:
- PYTHONPATH: ${{ github.workspace }}
- run: |
- set -euo pipefail
- source .venv/bin/activate
-
- export http_proxy=""; export https_proxy=""; export no_proxy=""; export HTTP_PROXY=""; export HTTPS_PROXY=""; export NO_PROXY=""
-
- EMAIL="ci-${GITHUB_RUN_ID}@example.com"
- PASS="ci-pass-${GITHUB_RUN_ID}"
- DATASET="ci_dataset_${GITHUB_RUN_ID}"
-
- CLI="python admin/client/ragflow_cli.py"
-
- LOG_FILE="infinity_cli_test.log"
- : > "${LOG_FILE}"
-
- ERROR_RE='Traceback|ModuleNotFoundError|ImportError|Parse error|Bad response|Fail to|code:\\s*[1-9]'
- run_cli() {
- local logfile="$1"
- shift
- local allow_re=""
- if [[ "${1:-}" == "--allow" ]]; then
- allow_re="$2"
- shift 2
- fi
- local cmd_display="$*"
- echo "===== $(date -u +\"%Y-%m-%dT%H:%M:%SZ\") CMD: ${cmd_display} =====" | tee -a "${logfile}"
- local tmp_log
- tmp_log="$(mktemp)"
- set +e
- timeout 500s "$@" 2>&1 | tee "${tmp_log}"
- local status=${PIPESTATUS[0]}
- set -e
- cat "${tmp_log}" >> "${logfile}"
- if grep -qiE "${ERROR_RE}" "${tmp_log}"; then
- if [[ -n "${allow_re}" ]] && grep -qiE "${allow_re}" "${tmp_log}"; then
- echo "Allowed CLI error markers in ${logfile}"
- rm -f "${tmp_log}"
- return 0
- fi
- echo "Detected CLI error markers in ${logfile}"
- rm -f "${tmp_log}"
- exit 1
- fi
- rm -f "${tmp_log}"
- return ${status}
- }
-
- set -a
- source docker/.env
- set +a
-
- HOST_ADDRESS="http://host.docker.internal:${SVR_HTTP_PORT}"
- USER_HOST="$(echo "${HOST_ADDRESS}" | sed -E 's#^https?://([^:/]+).*#\1#')"
- USER_PORT="${SVR_HTTP_PORT}"
- ADMIN_HOST="${USER_HOST}"
- ADMIN_PORT="${ADMIN_SVR_HTTP_PORT}"
-
- svc_ready=0
- for i in $(seq 1 60); do
- if sudo docker exec ${RAGFLOW_CONTAINER} curl -sf --connect-timeout 5 "${HOST_ADDRESS}/api/v1/system/ping" > /dev/null 2>&1; then
- svc_ready=1
- break
- fi
- echo "Waiting for service to be available... ($i/60)"
- sleep 5
- done
- if [ "$svc_ready" -ne 1 ]; then
- echo "Service did not become ready after 5 minutes. Docker logs:"
- sudo docker logs ${RAGFLOW_CONTAINER}
- exit 1
- fi
-
- admin_ready=0
- for i in $(seq 1 30); do
- if run_cli "${LOG_FILE}" $CLI --type admin --host "$ADMIN_HOST" --port "$ADMIN_PORT" --username "[email protected]" --password "admin" command "ping"; then
- admin_ready=1
- break
- fi
- sleep 1
- done
- if [[ "${admin_ready}" -ne 1 ]]; then
- echo "Admin service did not become ready"
- exit 1
- fi
-
- run_cli "${LOG_FILE}" $CLI --type admin --host "$ADMIN_HOST" --port "$ADMIN_PORT" --username "[email protected]" --password "admin" command "show version"
- ALLOW_USER_EXISTS_RE='already exists|already exist|duplicate|already.*registered|exist(s)?'
- run_cli "${LOG_FILE}" --allow "${ALLOW_USER_EXISTS_RE}" $CLI --type admin --host "$ADMIN_HOST" --port "$ADMIN_PORT" --username "[email protected]" --password "admin" command "create user '$EMAIL' '$PASS'"
-
- user_ready=0
- for i in $(seq 1 30); do
- if run_cli "${LOG_FILE}" $CLI --type user --host "$USER_HOST" --port "$USER_PORT" --username "$EMAIL" --password "$PASS" command "ping"; then
- user_ready=1
- break
- fi
- sleep 1
- done
- if [[ "${user_ready}" -ne 1 ]]; then
- echo "User service did not become ready"
- exit 1
- fi
-
- run_cli "${LOG_FILE}" $CLI --type user --host "$USER_HOST" --port "$USER_PORT" --username "$EMAIL" --password "$PASS" command "show version"
- run_cli "${LOG_FILE}" $CLI --type user --host "$USER_HOST" --port "$USER_PORT" --username "$EMAIL" --password "$PASS" command "create dataset '$DATASET' with embedding 'BAAI/bge-small-en-v1.5@Builtin' parser 'auto'"
- run_cli "${LOG_FILE}" $CLI --type user --host "$USER_HOST" --port "$USER_PORT" --username "$EMAIL" --password "$PASS" command "import 'test/benchmark/test_docs/Doc1.pdf,test/benchmark/test_docs/Doc2.pdf' into dataset '$DATASET'"
- run_cli "${LOG_FILE}" $CLI --type user --host "$USER_HOST" --port "$USER_PORT" --username "$EMAIL" --password "$PASS" command "parse dataset '$DATASET' sync"
- run_cli "${LOG_FILE}" $CLI --type user --host "$USER_HOST" --port "$USER_PORT" --username "$EMAIL" --password "$PASS" command "Benchmark 16 100 search 'what are these documents about' on datasets '$DATASET'"
-
- - name: Stop ragflow to save coverage Infinity
- if: ${{ !cancelled() }}
- run: |
- # Send SIGINT to ragflow_server.py to trigger coverage save
- PID=$(sudo docker exec ${RAGFLOW_CONTAINER} ps aux | grep "ragflow_server.py" | grep -v grep | awk '{print $2}' | head -n 1)
- if [ -n "$PID" ]; then
- echo "Sending SIGINT to ragflow_server.py (PID: $PID)..."
- sudo docker exec ${RAGFLOW_CONTAINER} kill -INT $PID
- # Wait for process to exit and coverage file to be written
- sleep 10
- else
- echo "ragflow_server.py not found!"
- fi
- sudo docker compose -f docker/docker-compose.yml -p ${COMPOSE_PROJECT_NAME} stop
-
- - name: Generate server coverage report Infinity
- if: ${{ !cancelled() }}
- run: |
- # .coverage file should be in docker/ragflow-logs/.coverage
- if [ -f docker/ragflow-logs/.coverage ]; then
- echo "Found .coverage file"
- cp docker/ragflow-logs/.coverage .coverage
- source .venv/bin/activate
- # Create .coveragerc to map container paths to host paths
- echo "[paths]" > .coveragerc
- echo "source =" >> .coveragerc
- echo " ." >> .coveragerc
- echo " /ragflow" >> .coveragerc
- coverage xml -o coverage-infinity-server.xml
- rm .coveragerc
- else
- echo ".coverage file not found!"
- fi
-
- - name: Upload coverage reports to Codecov
- uses: codecov/codecov-action@v5
- if: ${{ !cancelled() }}
- with:
- token: ${{ secrets.CODECOV_TOKEN }}
- fail_ci_if_error: false
-
- - name: Collect ragflow log Infinity
- if: ${{ !cancelled() }}
- run: |
- if [ -d docker/ragflow-logs ]; then
- cp -r docker/ragflow-logs ${ARTIFACTS_DIR}/ragflow-logs-infinity
- echo "ragflow log" && tail -n 200 docker/ragflow-logs/ragflow_server.log || true
- else
- echo "No docker/ragflow-logs directory found; skipping log collection"
- fi
- sudo rm -rf docker/ragflow-logs || true
-
- - name: Stop ragflow:nightly for Infinity
- if: always() # always run this step even if previous steps failed
- run: |
- # Sometimes `docker compose down` fail due to hang container, heavy load etc. Need to remove such containers to release resources(for example, listen ports).
- sudo docker compose -f docker/docker-compose.yml -p ${COMPOSE_PROJECT_NAME} down -v || true
- sudo docker ps -a --filter "label=com.docker.compose.project=${COMPOSE_PROJECT_NAME}" -q | xargs -r sudo docker rm -f
- if [[ -n ${RAGFLOW_IMAGE} ]]; then
- sudo docker rmi -f ${RAGFLOW_IMAGE}
- fi
- if [[ -n ${PORT_RESERVATION:-} ]]; then
- rm -f "${PORT_RESERVATION}"
- fi
-
-
-
- ragflow_tests_elasticsearch:
- name: ragflow_tests_elasticsearch
- needs: ragflow_preflight
- if: ${{ github.event_name != 'pull_request' || (github.event.pull_request.draft == false && contains(github.event.pull_request.labels.*.name, 'ci') && (github.event.action != 'labeled' || github.event.label.name == 'ci')) }}
- runs-on: [ "self-hosted", "ragflow-test" ]
- env:
- DOC_ENGINE: elasticsearch
- RAGFLOW_IMAGE: infiniflow/ragflow:${{ github.run_id }}-elasticsearch
- HTTP_API_TEST_LEVEL: ${{ needs.ragflow_preflight.outputs.http_api_test_level }}
- steps:
- - name: Ensure workspace ownership
- run: |
- echo "Workflow triggered by ${{ github.event_name }}"
- echo "chown -R ${USER} ${GITHUB_WORKSPACE}" && sudo chown -R ${USER} ${GITHUB_WORKSPACE}
-
- - name: Check out code
- uses: actions/checkout@v6
- with:
- ref: ${{ (github.event_name == 'pull_request' || github.event_name == 'pull_request_target') && format('refs/pull/{0}/merge', github.event.pull_request.number) || github.sha }}
- fetch-depth: 0
- fetch-tags: true
-
- - name: Build ragflow go server
- run: |
- set -euo pipefail
- BUILDER_CONTAINER=ragflow_build_${GITHUB_RUN_ID}_${DOC_ENGINE}_$(od -An -N4 -tx4 /dev/urandom | tr -d ' ')
- cleanup_builder() {
- if [[ -n "${BUILDER_CONTAINER:-}" ]]; then
- sudo docker rm -f -v "${BUILDER_CONTAINER}" >/dev/null 2>&1 || true
- fi
- }
- trap cleanup_builder EXIT
-
- TZ=${TZ:-$(readlink -f /etc/localtime | awk -F '/zoneinfo/' '{print $2}')}
- sudo docker run --privileged -d --name "${BUILDER_CONTAINER}" \
- -e TZ="${TZ}" \
- -e UV_INDEX=https://mirrors.aliyun.com/pypi/simple \
- -v "${PWD}:/ragflow" \
- -v "${PWD}/internal/binding/cpp/resource:/usr/share/infinity/resource" \
- infiniflow/infinity_builder:ubuntu22_clang20
- sudo docker exec "${BUILDER_CONTAINER}" bash -c 'git config --global safe.directory "*" && cd /ragflow && ./build.sh --cpp'
- ./build.sh --go
-
- - name: Run Go unit tests
- # Runs after `./build.sh --go`, which guarantees the C++ static
- # library (librag_tokenizer_c_api.a) is present on disk. The Go
- # test binaries link against it transitively through
- # `internal/binding`, so running `go test` before the C++ build
- # fails the link step.
- #
- # Excludes packages whose tests fail for environmental reasons
- # unrelated to the diff:
- # - internal/tokenizer is split: the pure-Go BPE loader tests
- # (bpe_loader_test.go) run in the default tier; the C++ binding /
- # dict-dependent tests (tokenizer_test.go,
- # tokenizer_concurrent_test.go) and the on-disk anchor test
- # (bpe_loader_anchors_test.go) are tagged `manual` and need the
- # docker builder's /usr/share/infinity/resource, so they stay out
- # of the default run.
- run: |
- set -euo pipefail
-
- # Provide the cl100k BPE table for the offline tokenizer loader.
- # The loader reads <repo>/ragflow_deps/cl100k_base.tiktoken (or
- # TIKTOKEN_CACHE_DIR). Fetch it directly: this CI runner can reach
- # the upstream blob (pre-PR tests used tiktoken-go's stock network
- # loader and passed), so a direct download is small and fast. As a
- # fallback, copy from the infiniflow/ragflow_deps image only if it is
- # already cached locally (we do NOT pull the whole image just for one
- # 1.6MB file). If neither works, the loader still fails loudly (no
- # silent 0-token degradation). The loader itself stays offline.
- if [ ! -f ragflow_deps/cl100k_base.tiktoken ]; then
- mkdir -p ragflow_deps
- if command -v curl >/dev/null 2>&1; then
- if curl -fsSL -o ragflow_deps/cl100k_base.tiktoken \
- https://openaipublic.blob.core.windows.net/encodings/cl100k_base.tiktoken; then
- echo "tiktoken: cl100k table fetched via curl from openai blob"
- else
- echo "tiktoken: curl fetch failed (network/GFW); will try cached image"
- fi
- fi
- if [ ! -f ragflow_deps/cl100k_base.tiktoken ] && docker image inspect infiniflow/ragflow_deps:latest >/dev/null 2>&1; then
- CID=$(docker create infiniflow/ragflow_deps:latest true) || true
- if [ -n "${CID:-}" ]; then
- if docker cp "$CID":/cl100k_base.tiktoken ragflow_deps/cl100k_base.tiktoken 2>/dev/null; then
- echo "tiktoken: cl100k table copied from cached infiniflow/ragflow_deps image"
- fi
- docker rm -f "$CID" >/dev/null 2>&1 || true
- fi
- fi
- fi
- if [ -f ragflow_deps/cl100k_base.tiktoken ]; then
- echo "tiktoken: cl100k table provisioned ($(wc -c < ragflow_deps/cl100k_base.tiktoken) bytes)"
- else
- echo "tiktoken: cl100k table NOT provisioned — offline loader will fail loudly"
- fi
-
- PKGS=$(go list ./... 2>/dev/null || true)
- if [ -z "$PKGS" ]; then
- ./build.sh --test
- else
- ./build.sh --test -- $PKGS
- fi
-
- - name: Build ragflow:nightly
- run: |
- set -euo pipefail
- GITEE_TOKEN_FILE="$(mktemp)"
- printf '%s' '${{ secrets.GITEE_TOKEN }}' > "${GITEE_TOKEN_FILE}"
- sudo docker pull ubuntu:24.04
- sudo DOCKER_BUILDKIT=1 docker build --build-arg NEED_MIRROR=1 --build-arg HTTPS_PROXY=${HTTPS_PROXY} --build-arg HTTP_PROXY=${HTTP_PROXY} --secret id=gitee_token,src=${GITEE_TOKEN_FILE} -f Dockerfile -t ${RAGFLOW_IMAGE} .
- rm -f "${GITEE_TOKEN_FILE}"
-
- - name: Prepare Python test environment
- run: |
- git config --global url."https://oauth2:${{ secrets.GITEE_TOKEN }}@gitee.com/".insteadOf "https://gitee.com/"
- uv sync --python 3.13 --group test --frozen
- uv pip install -e sdk/python
-
- - name: Prepare function test environment
- working-directory: docker
- run: |
- set -euo pipefail
- # install ss
- sudo apt update && sudo apt install -y iproute2
- RUNNER_WORKSPACE_PREFIX=${RUNNER_WORKSPACE_PREFIX:-${HOME}}
- COMPOSE_PROJECT_NAME="${GITHUB_RUN_ID}-${DOC_ENGINE}"
- echo "COMPOSE_PROJECT_NAME=${COMPOSE_PROJECT_NAME}" >> ${GITHUB_ENV}
- echo "RAGFLOW_CONTAINER=${COMPOSE_PROJECT_NAME}-ragflow-cpu-1" >> ${GITHUB_ENV}
- ARTIFACTS_DIR=${RUNNER_WORKSPACE_PREFIX}/artifacts/${GITHUB_REPOSITORY}/${GITHUB_RUN_ID}/${DOC_ENGINE}
- echo "ARTIFACTS_DIR=${ARTIFACTS_DIR}" >> ${GITHUB_ENV}
- rm -rf "${ARTIFACTS_DIR}" && mkdir -p "${ARTIFACTS_DIR}"
-
- # Determine runner number (default to 1 if not found)
- RUNNER_NUM=$(sudo docker inspect $(hostname) --format '{{index .Config.Labels "com.docker.compose.container-number"}}' 2>/dev/null || true)
- RUNNER_NUM=${RUNNER_NUM:-1}
-
- # Engine-specific offset partitions keep concurrent engine jobs from
- # choosing the same host ports when they land on the same self-hosted runner.
- # A lock plus reservation file closes the check/start race between parallel jobs.
- PORT_BASES=(1200 1201 23817 23820 5432 5455 9000 9001 6379 6380 6601 9380 9381 9382 9384 9383 9385 80 443 4222)
- PARTITION_SIZE=6000
- case "${DOC_ENGINE}" in
- elasticsearch) PARTITION_BASE=1000 ;;
- infinity) PARTITION_BASE=31000 ;;
- *) echo "Unsupported DOC_ENGINE=${DOC_ENGINE}" >&2; exit 1 ;;
- esac
- PORT_LOCK_DIR=${RUNNER_WORKSPACE_PREFIX}/artifacts/${GITHUB_REPOSITORY}/port-locks
- mkdir -p "${PORT_LOCK_DIR}"
-
- port_offset_available() {
- local offset=$1
- local base port
- for base in "${PORT_BASES[@]}"; do
- port=$((base + offset))
- if ss -ltnH "sport = :${port}" | grep -q .; then
- return 1
- fi
- done
- return 0
- }
-
- cleanup_stale_port_locks() {
- local now stale_after lock lock_ts
- now=$(date -u +%s)
- stale_after=$((6 * 60 * 60))
- for lock in "${PORT_LOCK_DIR}"/*.lock; do
- [[ -e "${lock}" ]] || continue
- lock_ts=$(awk '{print $3}' "${lock}" 2>/dev/null || true)
- if [[ "${lock_ts}" =~ ^[0-9]+$ ]] && (( now - lock_ts > stale_after )); then
- rm -f "${lock}"
- fi
- done
- }
-
- reserve_port_offset() {
- local attempt candidate reservation
- cleanup_stale_port_locks
- for attempt in $(seq 0 59); do
- candidate=$(( PARTITION_BASE + ((GITHUB_RUN_ID + RUNNER_NUM * 1000 + attempt * 97) % PARTITION_SIZE) ))
- reservation="${PORT_LOCK_DIR}/${candidate}.lock"
- if ( set -o noclobber; echo "${GITHUB_RUN_ID} ${DOC_ENGINE} $(date -u +%s)" > "${reservation}" ) 2>/dev/null; then
- if port_offset_available "${candidate}"; then
- PORT_OFFSET=${candidate}
- PORT_RESERVATION=${reservation}
- return 0
- fi
- rm -f "${reservation}"
- fi
- done
- return 1
- }
-
- if ! reserve_port_offset; then
- echo "Failed to reserve a free host port range for ${DOC_ENGINE} docker compose" >&2
- exit 1
- fi
- echo "PORT_RESERVATION=${PORT_RESERVATION}" >> ${GITHUB_ENV}
- echo "Using ${DOC_ENGINE} host port offset ${PORT_OFFSET}"
- ES_PORT=$((1200 + PORT_OFFSET))
- OS_PORT=$((1201 + PORT_OFFSET))
- INFINITY_THRIFT_PORT=$((23817 + PORT_OFFSET))
- INFINITY_HTTP_PORT=$((23820 + PORT_OFFSET))
- INFINITY_PSQL_PORT=$((5432 + PORT_OFFSET))
- EXPOSE_MYSQL_PORT=$((5455 + PORT_OFFSET))
- MINIO_PORT=$((9000 + PORT_OFFSET))
- MINIO_CONSOLE_PORT=$((9001 + PORT_OFFSET))
- REDIS_PORT=$((6379 + PORT_OFFSET))
- NATS_PORT=$((4222 + PORT_OFFSET))
- TEI_PORT=$((6380 + PORT_OFFSET))
- KIBANA_PORT=$((6601 + PORT_OFFSET))
- SVR_HTTP_PORT=$((9380 + PORT_OFFSET))
- ADMIN_SVR_HTTP_PORT=$((9381 + PORT_OFFSET))
- SVR_MCP_PORT=$((9382 + PORT_OFFSET))
- GO_HTTP_PORT=$((9384 + PORT_OFFSET))
- GO_ADMIN_PORT=$((9383 + PORT_OFFSET))
- SANDBOX_EXECUTOR_MANAGER_PORT=$((9385 + PORT_OFFSET))
- SVR_WEB_HTTP_PORT=$((80 + PORT_OFFSET))
- SVR_WEB_HTTPS_PORT=$((443 + PORT_OFFSET))
-
- # Persist computed ports into .env so docker-compose uses the correct host bindings.
- # Remove previous CI overrides first; docker compose uses the last duplicate key.
- sed -i '/^ES_PORT=/d;/^OS_PORT=/d;/^INFINITY_THRIFT_PORT=/d;/^INFINITY_HTTP_PORT=/d;/^INFINITY_PSQL_PORT=/d;/^EXPOSE_MYSQL_PORT=/d;/^MINIO_PORT=/d;/^MINIO_CONSOLE_PORT=/d;/^REDIS_PORT=/d;/^TEI_PORT=/d;/^KIBANA_PORT=/d;/^SVR_HTTP_PORT=/d;/^ADMIN_SVR_HTTP_PORT=/d;/^SVR_MCP_PORT=/d;/^GO_HTTP_PORT=/d;/^GO_ADMIN_PORT=/d;/^SANDBOX_EXECUTOR_MANAGER_PORT=/d;/^SVR_WEB_HTTP_PORT=/d;/^SVR_WEB_HTTPS_PORT=/d;/^NATS_PORT=/d;/^COMPOSE_PROFILES=/d;/^TEI_MODEL=/d;/^RAGFLOW_IMAGE=/d;/^DOC_ENGINE=/d' .env
- {
- echo ""
- echo "ES_PORT=${ES_PORT}"
- echo "OS_PORT=${OS_PORT}"
- echo "INFINITY_THRIFT_PORT=${INFINITY_THRIFT_PORT}"
- echo "INFINITY_HTTP_PORT=${INFINITY_HTTP_PORT}"
- echo "INFINITY_PSQL_PORT=${INFINITY_PSQL_PORT}"
- echo "EXPOSE_MYSQL_PORT=${EXPOSE_MYSQL_PORT}"
- echo "MINIO_PORT=${MINIO_PORT}"
- echo "MINIO_CONSOLE_PORT=${MINIO_CONSOLE_PORT}"
- echo "REDIS_PORT=${REDIS_PORT}"
- echo "NATS_PORT=${NATS_PORT}"
- echo "TEI_PORT=${TEI_PORT}"
- echo "KIBANA_PORT=${KIBANA_PORT}"
- echo "SVR_HTTP_PORT=${SVR_HTTP_PORT}"
- echo "ADMIN_SVR_HTTP_PORT=${ADMIN_SVR_HTTP_PORT}"
- echo "SVR_MCP_PORT=${SVR_MCP_PORT}"
- echo "GO_HTTP_PORT=${GO_HTTP_PORT}"
- echo "GO_ADMIN_PORT=${GO_ADMIN_PORT}"
- echo "SANDBOX_EXECUTOR_MANAGER_PORT=${SANDBOX_EXECUTOR_MANAGER_PORT}"
- echo "SVR_WEB_HTTP_PORT=${SVR_WEB_HTTP_PORT}"
- echo "SVR_WEB_HTTPS_PORT=${SVR_WEB_HTTPS_PORT}"
- echo "COMPOSE_PROFILES=${DOC_ENGINE},cpu,tei-cpu,deepdoc,metadata-mysql"
- echo "TEI_MODEL=BAAI/bge-small-en-v1.5"
- echo "RAGFLOW_IMAGE=${RAGFLOW_IMAGE}"
- echo "DOC_ENGINE=${DOC_ENGINE}"
- } >> .env
- echo "HOST_ADDRESS=http://host.docker.internal:${SVR_HTTP_PORT}" >> ${GITHUB_ENV}
-
- # Patch entrypoint.sh for coverage
- sed -i '/"\$PY" api\/ragflow_server.py \${INIT_SUPERUSER_ARGS} &/c\ echo "Ensuring coverage is installed..."\n "$PY" -m pip install coverage -i https://mirrors.aliyun.com/pypi/simple\n export COVERAGE_FILE=/ragflow/logs/.coverage\n echo "Starting ragflow_server with coverage..."\n "$PY" -m coverage run --source=./api/apps --omit="*/tests/*,*/migrations/*" -a api/ragflow_server.py ${INIT_SUPERUSER_ARGS} &' ./entrypoint.sh
-
-
- - name: Start ragflow:nightly for Elasticsearch
- run: |
- sudo docker compose -f docker/docker-compose.yml -p ${COMPOSE_PROJECT_NAME} down -v || true
- sudo docker ps -a --filter "label=com.docker.compose.project=${COMPOSE_PROJECT_NAME}" -q | xargs -r sudo docker rm -f
- sudo docker compose -f docker/docker-compose.yml -p ${COMPOSE_PROJECT_NAME} up -d
-
- - name: Run sdk tests against Elasticsearch
- run: |
- export http_proxy=""; export https_proxy=""; export no_proxy=""; export HTTP_PROXY=""; export HTTPS_PROXY=""; export NO_PROXY=""
- svc_ready=0
- for i in $(seq 1 60); do
- if sudo docker exec ${RAGFLOW_CONTAINER} curl -sf --connect-timeout 5 "${HOST_ADDRESS}/api/v1/system/ping" > /dev/null 2>&1; then
- svc_ready=1
- break
- fi
- echo "Waiting for service to be available... ($i/60)"
- sleep 5
- done
- if [ "$svc_ready" -ne 1 ]; then
- echo "Service did not become ready after 5 minutes. Docker logs:"
- sudo docker logs ${RAGFLOW_CONTAINER}
- exit 1
- fi
- echo "Start to run test sdk on Elasticsearch"
- source .venv/bin/activate && set -o pipefail; pytest -s --tb=short --level=${HTTP_API_TEST_LEVEL} --junitxml=pytest-es-sdk.xml --cov=sdk/python/ragflow_sdk --cov-branch --cov-report=xml:coverage-es-sdk.xml test/testcases/test_sdk_api 2>&1 | tee es_sdk_test.log
-
- - name: Run New RESTFUL api tests against Elasticsearch
- run: |
- export http_proxy=""; export https_proxy=""; export no_proxy=""; export HTTP_PROXY=""; export HTTPS_PROXY=""; export NO_PROXY=""
- svc_ready=0
- for i in $(seq 1 60); do
- if sudo docker exec ${RAGFLOW_CONTAINER} curl -sf --connect-timeout 5 "${HOST_ADDRESS}/api/v1/system/ping" > /dev/null 2>&1; then
- svc_ready=1
- break
- fi
- echo "Waiting for service to be available... ($i/60)"
- sleep 5
- done
- if [ "$svc_ready" -ne 1 ]; then
- echo "Service did not become ready after 5 minutes. Docker logs:"
- sudo docker logs ${RAGFLOW_CONTAINER}
- exit 1
- fi
- source .venv/bin/activate && set -o pipefail; pytest -s --tb=short --level=${HTTP_API_TEST_LEVEL} test/testcases/restful_api 2>&1 | tee es_restful_api_test.log
-
- - name: RAGFlow CLI retrieval test Elasticsearch
- env:
- PYTHONPATH: ${{ github.workspace }}
- run: |
- set -euo pipefail
- source .venv/bin/activate
-
- export http_proxy=""; export https_proxy=""; export no_proxy=""; export HTTP_PROXY=""; export HTTPS_PROXY=""; export NO_PROXY=""
-
- EMAIL="ci-${GITHUB_RUN_ID}@example.com"
- PASS="ci-pass-${GITHUB_RUN_ID}"
- DATASET="ci_dataset_${GITHUB_RUN_ID}"
-
- CLI="python admin/client/ragflow_cli.py"
-
- LOG_FILE="es_cli_test.log"
- : > "${LOG_FILE}"
-
- ERROR_RE='Traceback|ModuleNotFoundError|ImportError|Parse error|Bad response|Fail to|code:\\s*[1-9]'
- run_cli() {
- local logfile="$1"
- shift
- local allow_re=""
- if [[ "${1:-}" == "--allow" ]]; then
- allow_re="$2"
- shift 2
- fi
- local cmd_display="$*"
- echo "===== $(date -u +\"%Y-%m-%dT%H:%M:%SZ\") CMD: ${cmd_display} =====" | tee -a "${logfile}"
- local tmp_log
- tmp_log="$(mktemp)"
- set +e
- timeout 500s "$@" 2>&1 | tee "${tmp_log}"
- local status=${PIPESTATUS[0]}
- set -e
- cat "${tmp_log}" >> "${logfile}"
- if grep -qiE "${ERROR_RE}" "${tmp_log}"; then
- if [[ -n "${allow_re}" ]] && grep -qiE "${allow_re}" "${tmp_log}"; then
- echo "Allowed CLI error markers in ${logfile}"
- rm -f "${tmp_log}"
- return 0
- fi
- echo "Detected CLI error markers in ${logfile}"
- rm -f "${tmp_log}"
- exit 1
- fi
- rm -f "${tmp_log}"
- return ${status}
- }
-
- set -a
- source docker/.env
- set +a
-
- HOST_ADDRESS="http://host.docker.internal:${SVR_HTTP_PORT}"
- USER_HOST="$(echo "${HOST_ADDRESS}" | sed -E 's#^https?://([^:/]+).*#\1#')"
- USER_PORT="${SVR_HTTP_PORT}"
- ADMIN_HOST="${USER_HOST}"
- ADMIN_PORT="${ADMIN_SVR_HTTP_PORT}"
-
- svc_ready=0
- for i in $(seq 1 60); do
- if sudo docker exec ${RAGFLOW_CONTAINER} curl -sf --connect-timeout 5 "${HOST_ADDRESS}/api/v1/system/ping" > /dev/null 2>&1; then
- svc_ready=1
- break
- fi
- echo "Waiting for service to be available... ($i/60)"
- sleep 5
- done
- if [ "$svc_ready" -ne 1 ]; then
- echo "Service did not become ready after 5 minutes. Docker logs:"
- sudo docker logs ${RAGFLOW_CONTAINER}
- exit 1
- fi
-
- admin_ready=0
- for i in $(seq 1 30); do
- if run_cli "${LOG_FILE}" $CLI --type admin --host "$ADMIN_HOST" --port "$ADMIN_PORT" --username "[email protected]" --password "admin" command "ping"; then
- admin_ready=1
- break
- fi
- sleep 1
- done
- if [[ "${admin_ready}" -ne 1 ]]; then
- echo "Admin service did not become ready"
- exit 1
- fi
-
- run_cli "${LOG_FILE}" $CLI --type admin --host "$ADMIN_HOST" --port "$ADMIN_PORT" --username "[email protected]" --password "admin" command "show version"
- ALLOW_USER_EXISTS_RE='already exists|already exist|duplicate|already.*registered|exist(s)?'
- run_cli "${LOG_FILE}" --allow "${ALLOW_USER_EXISTS_RE}" $CLI --type admin --host "$ADMIN_HOST" --port "$ADMIN_PORT" --username "[email protected]" --password "admin" command "create user '$EMAIL' '$PASS'"
-
- user_ready=0
- for i in $(seq 1 30); do
- if run_cli "${LOG_FILE}" $CLI --type user --host "$USER_HOST" --port "$USER_PORT" --username "$EMAIL" --password "$PASS" command "ping"; then
- user_ready=1
- break
- fi
- sleep 1
- done
- if [[ "${user_ready}" -ne 1 ]]; then
- echo "User service did not become ready"
- exit 1
- fi
-
- run_cli "${LOG_FILE}" $CLI --type user --host "$USER_HOST" --port "$USER_PORT" --username "$EMAIL" --password "$PASS" command "show version"
- run_cli "${LOG_FILE}" $CLI --type user --host "$USER_HOST" --port "$USER_PORT" --username "$EMAIL" --password "$PASS" command "create dataset '$DATASET' with embedding 'BAAI/bge-small-en-v1.5@Builtin' parser 'auto'"
- run_cli "${LOG_FILE}" $CLI --type user --host "$USER_HOST" --port "$USER_PORT" --username "$EMAIL" --password "$PASS" command "import 'test/benchmark/test_docs/Doc1.pdf,test/benchmark/test_docs/Doc2.pdf' into dataset '$DATASET'"
- run_cli "${LOG_FILE}" $CLI --type user --host "$USER_HOST" --port "$USER_PORT" --username "$EMAIL" --password "$PASS" command "parse dataset '$DATASET' sync"
- run_cli "${LOG_FILE}" $CLI --type user --host "$USER_HOST" --port "$USER_PORT" --username "$EMAIL" --password "$PASS" command "Benchmark 16 100 search 'what are these documents about' on datasets '$DATASET'"
-
- - name: Stop ragflow to save coverage Elasticsearch
- if: ${{ !cancelled() }}
- run: |
- # Send SIGINT to ragflow_server.py to trigger coverage save
- PID=$(sudo docker exec ${RAGFLOW_CONTAINER} ps aux | grep "ragflow_server.py" | grep -v grep | awk '{print $2}' | head -n 1)
- if [ -n "$PID" ]; then
- echo "Sending SIGINT to ragflow_server.py (PID: $PID)..."
- sudo docker exec ${RAGFLOW_CONTAINER} kill -INT $PID
- # Wait for process to exit and coverage file to be written
- sleep 10
- else
- echo "ragflow_server.py not found!"
- fi
- sudo docker compose -f docker/docker-compose.yml -p ${COMPOSE_PROJECT_NAME} stop
-
- - name: Generate server coverage report Elasticsearch
- if: ${{ !cancelled() }}
- run: |
- # .coverage file should be in docker/ragflow-logs/.coverage
- if [ -f docker/ragflow-logs/.coverage ]; then
- echo "Found .coverage file"
- cp docker/ragflow-logs/.coverage .coverage
- source .venv/bin/activate
- # Create .coveragerc to map container paths to host paths
- echo "[paths]" > .coveragerc
- echo "source =" >> .coveragerc
- echo " ." >> .coveragerc
- echo " /ragflow" >> .coveragerc
- coverage xml -o coverage-es-server.xml
- rm .coveragerc
- # Clean up for next run
- sudo rm docker/ragflow-logs/.coverage
- else
- echo ".coverage file not found!"
- fi
-
- - name: Collect ragflow log Elasticsearch
- if: ${{ !cancelled() }}
- run: |
- if [ -d docker/ragflow-logs ]; then
- cp -r docker/ragflow-logs ${ARTIFACTS_DIR}/ragflow-logs-es
- echo "ragflow log" && tail -n 200 docker/ragflow-logs/ragflow_server.log || true
- else
- echo "No docker/ragflow-logs directory found; skipping log collection"
- fi
- sudo rm -rf docker/ragflow-logs || true
-
- - name: Stop ragflow:nightly for Elasticsearch
- if: always() # always run this step even if previous steps failed
- run: |
- # Sometimes `docker compose down` fail due to hang container, heavy load etc. Need to remove such containers to release resources(for example, listen ports).
- sudo docker compose -f docker/docker-compose.yml -p ${COMPOSE_PROJECT_NAME} down -v || true
- sudo docker ps -a --filter "label=com.docker.compose.project=${COMPOSE_PROJECT_NAME}" -q | xargs -r sudo docker rm -f
- if [[ -n ${RAGFLOW_IMAGE} ]]; then
- sudo docker rmi -f ${RAGFLOW_IMAGE}
- fi
- if [[ -n ${PORT_RESERVATION:-} ]]; then
- rm -f "${PORT_RESERVATION}"
- fi
-
- ragflow_native_backend:
- name: ragflow_native_backend
- # Runs AFTER both per-engine go jobs (which contain the doc-engine-specific
- # server build + Python RESTFUL correctness runs). The DeepDoc native
- # backend is doc-engine-independent, so running it ONCE here — instead of
- # duplicated (and memory-competing) inside each go job — removes the
- # redundant second -race pass and the concurrent -race OOM risk. Running it
- # last also keeps a flaky -race OOM from ever obscuring the functional
- # verdict produced by the go jobs.
- needs: [ragflow_tests_infinity, ragflow_tests_elasticsearch]
- if: ${{ github.event_name != 'pull_request' || (github.event.pull_request.draft == false && contains(github.event.pull_request.labels.*.name, 'ci') && (github.event.action != 'labeled' || github.event.label.name == 'ci')) }}
- runs-on: [ "self-hosted", "ragflow-test" ]
- steps:
- - name: Ensure workspace ownership
- run: |
- echo "Workflow triggered by ${{ github.event_name }}"
- echo "chown -R ${USER} ${GITHUB_WORKSPACE}" && sudo chown -R ${USER} ${GITHUB_WORKSPACE}
-
- - name: Check out code
- uses: actions/checkout@v6
- with:
- ref: ${{ (github.event_name == 'pull_request' || github.event_name == 'pull_request_target') && format('refs/pull/{0}/merge', github.event.pull_request.number) || github.sha }}
- fetch-depth: 0
- fetch-tags: true
-
- - name: Build C++ static library (ragflow native backend)
- # Produces librag_tokenizer_c_api.a with the re2 symbols renamed into
- # the ragtokre2_ namespace (so it no longer collides with onnxruntime's
- # bundled re2 copy at link time). Only the .a is needed for the native
- # integration tests; the per-engine go jobs build the full server.
- run: |
- set -euo pipefail
- BUILDER_CONTAINER=ragflow_build_${GITHUB_RUN_ID}_native_$(od -An -N4 -tx4 /dev/urandom | tr -d ' ')
- cleanup_builder() {
- if [[ -n "${BUILDER_CONTAINER:-}" ]]; then
- sudo docker rm -f -v "${BUILDER_CONTAINER}" >/dev/null 2>&1 || true
- fi
- }
- trap cleanup_builder EXIT
-
- TZ=${TZ:-$(readlink -f /etc/localtime | awk -F '/zoneinfo/' '{print $2}')}
- sudo docker run --privileged -d --name "${BUILDER_CONTAINER}" \
- -e TZ="${TZ}" \
- -e UV_INDEX=https://mirrors.aliyun.com/pypi/simple \
- -v "${PWD}:/ragflow" \
- -v "${PWD}/internal/binding/cpp/resource:/usr/share/infinity/resource" \
- infiniflow/infinity_builder:ubuntu22_clang20
- sudo docker exec "${BUILDER_CONTAINER}" bash -c 'git config --global safe.directory "*" && cd /ragflow && ./build.sh --cpp'
-
- - name: Cache external DeepDoc testdata
- # The native backend tests fetch pinned test assets from
- # xugangqiang/ragflow-testdata (see scripts/fetch_deepdoc_testdata.sh).
- # Cache them across runs, keyed by the pinned ref recorded in
- # internal/deepdoc/native/testdata.ref so a ref bump invalidates the cache.
- uses: actions/cache@v4
- with:
- path: ${{ github.workspace }}/.cache/ragflow-testdata
- key: deepdoc-testdata-${{ hashFiles('internal/deepdoc/native/testdata.ref') }}
-
- - name: Run in-process DeepDoc backend integration tests (correctness + race)
- # Doc-engine-independent: the same native backend (det/layout/tsr/rec
- # ONNX models + onnxruntime) is exercised regardless of DOC_ENGINE, so
- # it is run ONCE here instead of duplicated inside each go job.
- #
- # Inside build.sh, the golden/comparison tests run WITHOUT -race (no
- # race-detection value, ~10x memory tax) and only the concurrency
- # correctness tests run WITH -race (see run_native_integration_tests).
- #
- # XDG_CACHE_HOME is pinned to a workspace path so the test fetch script
- # and this job's actions/cache step agree on a single, deterministic
- # location (the self-hosted runner's $HOME is not stable across steps).
- env:
- MODEL_DIR: /opt/ragflow-deepdoc-models
- CGO_ENABLED: '1'
- DEEPDOC_NATIVE_REQUIRED: '1'
- XDG_CACHE_HOME: ${{ github.workspace }}/.cache
- run: |
- set -euo pipefail
- ./build.sh --test-native
-
diff --git a/scripts/fetch_deepdoc_testdata.sh b/scripts/fetch_deepdoc_testdata.sh
index 17c5900b8f..2d3148ac6c 100644
--- a/scripts/fetch_deepdoc_testdata.sh
+++ b/scripts/fetch_deepdoc_testdata.sh
@@ -6,15 +6,15 @@
# (mirrors internal/deepdoc/<pkg>/testdata).
#
# The data lives in an external asset repository
-# (RAGFLOW_TESTDATA_REPO, default xugangqiang/ragflow-testdata — the fork used
-# until the data is merged into infiniflow/ragflow-testdata; see
-# deepdoc_native_testdata_handoff.md S8) and is pinned by a tag recorded in
+# (RAGFLOW_TESTDATA_REPO, default infiniflow/ragflow-testdata — the canonical
+# org-owned asset repository. The data was migrated out of the xugangqiang fork
+# (see deepdoc_native_testdata_handoff.md S8) and is pinned by a tag recorded in
# internal/deepdoc/<pkg>/testdata.ref. We sparse-clone only the relevant
# subtree into a content-addressed cache and symlink it into the package so
# existing tests (which read relative testdata/... paths) need no changes.
#
# Env:
-# RAGFLOW_TESTDATA_REPO repo "owner/name" (default xugangqiang/ragflow-testdata)
+# RAGFLOW_TESTDATA_REPO repo "owner/name" (default infiniflow/ragflow-testdata)
# RAGFLOW_TESTDATA_REF override the anchor tag/ref (else read testdata.ref)
# XDG_CACHE_HOME cache base (default ~/.cache)
#
@@ -50,7 +50,7 @@ if [ -z "$REF" ]; then
exit 1
fi
-REPO="${RAGFLOW_TESTDATA_REPO:-xugangqiang/ragflow-testdata}"
+REPO="${RAGFLOW_TESTDATA_REPO:-infiniflow/ragflow-testdata}"
CACHE_BASE="${XDG_CACHE_HOME:-$HOME/.cache}/ragflow-testdata"
CACHE="$CACHE_BASE/$REF"
SRC="$CACHE/deepdoc/$PKG/testdata"
@@ -79,11 +79,29 @@ rm -f "$TARGET" 2>/dev/null || true
if [ ! -e "$SRC" ] || [ -z "$(ls -A "$SRC" 2>/dev/null)" ]; then
echo "fetch_deepdoc_testdata: cloning $REPO @ $REF (subtree deepdoc/$PKG/testdata)"
- rm -rf "$CACHE"
mkdir -p "$CACHE_BASE"
- git clone --depth 1 --filter=blob:none --branch "$REF" --sparse \
- "https://github.com/$REPO.git" "$CACHE" >&2
- git -C "$CACHE" sparse-checkout set "deepdoc/$PKG/testdata" >&2
+ # Network clones are best-effort and occasionally fail with a transient TLS
+ # reset (seen on the self-hosted runner). Retry a few times before giving up
+ # so a CI blip does not redden the run.
+ attempt=0
+ max_attempts=3
+ fetched=0
+ until [ "$attempt" -ge "$max_attempts" ]; do
+ attempt=$((attempt + 1))
+ rm -rf "$CACHE"
+ if git clone --depth 1 --filter=blob:none --branch "$REF" --sparse \
+ "https://github.com/$REPO.git" "$CACHE" >&2 && \
+ git -C "$CACHE" sparse-checkout set "deepdoc/$PKG/testdata" >&2; then
+ fetched=1
+ break
+ fi
+ echo "fetch_deepdoc_testdata: clone attempt $attempt/$max_attempts failed, retrying in 3s" >&2
+ sleep 3
+ done
+ if [ "$fetched" -ne 1 ]; then
+ echo "fetch_deepdoc_testdata: clone failed after $max_attempts attempts" >&2
+ exit 1
+ fi
fi
if [ ! -e "$SRC" ] || [ -z "$(ls -A "$SRC" 2>/dev/null)" ]; then