Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
fix(ci): fix ragfs-python native extension build in CI pipelines (#1532)
kaisongli8d3983479532
diff --git a/.github/workflows/api_test.yml b/.github/workflows/api_test.ymlindex 6894e143d..5fcca89a9 100644--- a/.github/workflows/api_test.yml+++ b/.github/workflows/api_test.yml@@ -113,7 +113,12 @@ jobs:- name: Install build dependencies (uv pip)run: |echo "Installing build dependencies to uv environment..."- uv pip install setuptools setuptools_scm cmake wheel+ uv pip install setuptools setuptools_scm cmake wheel maturin++ - name: Set up Rust toolchain+ uses: dtolnay/rust-toolchain@v1+ with:+ toolchain: stable- name: Build C++ extensionsshell: bash@@ -132,6 +137,53 @@ jobs:uv run python setup.py build_ext --inplace+ - name: Build ragfs-python native extension+ shell: bash+ run: |+ RAGFS_LIB_DIR="openviking/lib"+ RAGFS_SO_COUNT=$(ls -1 "$RAGFS_LIB_DIR"/ragfs_python*.so "$RAGFS_LIB_DIR"/ragfs_python*.pyd "$RAGFS_LIB_DIR"/ragfs_python*.dylib 2>/dev/null | wc -l | xargs || echo "0")++ if [ "$RAGFS_SO_COUNT" -gt 0 ]; then+ echo "✅ ragfs_python native extension already exists:"+ ls -la "$RAGFS_LIB_DIR"/ragfs_python*+ exit 0+ fi++ echo "ragfs_python native lib not found after build_ext, building via maturin..."+ TMPDIR_RAGFS=$(mktemp -d)+ cd crates/ragfs-python+ uv run --no-project maturin build --release --features s3 --out "$TMPDIR_RAGFS"+ cd ../..++ WHL_FILE=$(ls -1 "$TMPDIR_RAGFS"/ragfs_python-*.whl 2>/dev/null | head -1)+ if [ -z "$WHL_FILE" ]; then+ echo "❌ ERROR: maturin build produced no wheel"+ ls -la "$TMPDIR_RAGFS"/+ exit 1+ fi++ echo "Extracting ragfs_python from wheel: $WHL_FILE"+ mkdir -p "$RAGFS_LIB_DIR"+ uv run python -c "+ import zipfile, os, sys, stat+ with zipfile.ZipFile('$WHL_FILE') as zf:+ for name in zf.namelist():+ bn = os.path.basename(name)+ if bn.startswith('ragfs_python') and (bn.endswith('.so') or bn.endswith('.pyd') or bn.endswith('.dylib')):+ dst = os.path.join('$RAGFS_LIB_DIR', bn)+ with zf.open(name) as src, open(dst, 'wb') as f:+ f.write(src.read())+ os.chmod(dst, 0o755)+ print(f' [OK] ragfs-python: extracted {bn} -> {dst}')+ sys.exit(0)+ print('[ERROR] No ragfs_python native library found in wheel')+ sys.exit(1)+ "+ rm -rf "$TMPDIR_RAGFS"++ echo "Verifying ragfs_python native extension:"+ ls -la "$RAGFS_LIB_DIR"/ragfs_python*+- name: Install API test dependenciesrun: |cd tests/api_test@@ -411,7 +463,7 @@ jobs:uv run python -m pytest . -v --html=api-test-report.html --self-contained-html \--ignore=retrieval/ --ignore=resources/test_pack.py --ignore=resources/test_wait_processed.py \--ignore=admin/ --ignore=skills/ --ignore=system/test_system_status.py --ignore=system/test_is_healthy.py --ignore=system/test_system_wait.py \- --ignore=scenarios/ -k "not test_observer"+ --ignore=scenarios/ --ignore=sessions/ -k "not test_observer"ficontinue-on-error: true@@ -429,7 +481,7 @@ jobs:uv run python -m pytest . -v --html=api-test-report.html --self-contained-html --ignore=filesystem/} else {Write-Host "Running basic tests only (no VLM/Embedding, Windows: skipping filesystem tests)"- uv run python -m pytest . -v --html=api-test-report.html --self-contained-html --ignore=retrieval/ --ignore=resources/test_pack.py --ignore=resources/test_wait_processed.py --ignore=admin/ --ignore=skills/ --ignore=system/test_system_status.py --ignore=system/test_is_healthy.py --ignore=system/test_system_wait.py --ignore=filesystem/ --ignore=scenarios/ -k "not test_observer"+ uv run python -m pytest . -v --html=api-test-report.html --self-contained-html --ignore=retrieval/ --ignore=resources/test_pack.py --ignore=resources/test_wait_processed.py --ignore=admin/ --ignore=skills/ --ignore=system/test_system_status.py --ignore=system/test_is_healthy.py --ignore=system/test_system_wait.py --ignore=filesystem/ --ignore=scenarios/ --ignore=sessions/ -k "not test_observer"}continue-on-error: truediff --git a/crates/ragfs-python/pyproject.toml b/crates/ragfs-python/pyproject.tomlindex eb3445e29..b2b51358a 100644--- a/crates/ragfs-python/pyproject.toml+++ b/crates/ragfs-python/pyproject.toml@@ -10,4 +10,3 @@ requires-python = ">=3.10"[tool.maturin]features = ["s3"]abi3 = true-python-source = "3.10"diff --git a/tests/oc2ov_test/upgrade_openviking.sh b/tests/oc2ov_test/upgrade_openviking.shindex 8ed44913a..b0c6f93f8 100644--- a/tests/oc2ov_test/upgrade_openviking.sh+++ b/tests/oc2ov_test/upgrade_openviking.sh@@ -222,6 +222,43 @@ elselog "⚠️ Rust toolchain setup failed, build may fail"fi+log "[5.55/8] Checking maturin for ragfs-python build..."+MATURIN_OK=false++if python -c "import maturin" 2>/dev/null; then+ MATURIN_VERSION=$(python -m maturin --version 2>/dev/null | awk '{print $2}' || echo "unknown")+ log "✅ maturin is already available (Python module): $MATURIN_VERSION"+ MATURIN_OK=true+elif command -v maturin &> /dev/null; then+ MATURIN_VERSION=$(maturin --version 2>/dev/null | awk '{print $2}' || echo "unknown")+ log "✅ maturin is already available (CLI): $MATURIN_VERSION"+ MATURIN_OK=true+fi++if [ "$MATURIN_OK" = false ]; then+ log "maturin not found, installing..."+ if command -v uv &> /dev/null && uv pip --help &> /dev/null; then+ if uv pip install maturin 2>&1 | tee -a "$LOG_FILE"; then+ log "✅ maturin installed via uv pip"+ MATURIN_OK=true+ fi+ fi++ if [ "$MATURIN_OK" = false ]; then+ if pip install maturin 2>&1 | tee -a "$LOG_FILE"; then+ log "✅ maturin installed via pip"+ MATURIN_OK=true+ else+ log "⚠️ Failed to install maturin, ragfs-python will not be built"+ fi+ fi+fi++if [ "$MATURIN_OK" = true ]; then+ MATURIN_VERSION=$(python -m maturin --version 2>/dev/null || maturin --version 2>/dev/null || echo "unknown")+ log "maturin version: $MATURIN_VERSION"+fi+log "[5.6/8] Checking Python build dependencies..."# Check if setuptools-scm is already installed@@ -280,7 +317,49 @@ for i in $(seq 1 $MAX_RETRIES); dolog "Build attempt $i/$MAX_RETRIES..."if python setup.py build_ext --inplace 2>&1 | tee -a "$LOG_FILE"; then- log "build_ext completed, installing dependencies..."+ log "build_ext completed"++ RAGFS_LIB_DIR="$PROJECT_DIR/openviking/lib"+ RAGFS_SO_COUNT=$(ls -1 "$RAGFS_LIB_DIR"/ragfs_python*.so "$RAGFS_LIB_DIR"/ragfs_python*.pyd "$RAGFS_LIB_DIR"/ragfs_python*.dylib 2>/dev/null | wc -l | xargs || echo "0")+ if [ "$RAGFS_SO_COUNT" -eq 0 ] && [ "$MATURIN_OK" = true ] && [ -d "$PROJECT_DIR/crates/ragfs-python" ]; then+ log "ragfs_python native lib not found after build_ext, building via maturin..."+ TMPDIR_RAGFS=$(mktemp -d)+ if (cd "$PROJECT_DIR/crates/ragfs-python" && python -m maturin build --release --features s3 --out "$TMPDIR_RAGFS" 2>&1 | tee -a "$LOG_FILE"); then+ WHL_FILE=$(ls -1 "$TMPDIR_RAGFS"/ragfs_python-*.whl 2>/dev/null | head -1)+ if [ -n "$WHL_FILE" ]; then+ mkdir -p "$RAGFS_LIB_DIR"+ python -c "+import zipfile, os, sys, stat+with zipfile.ZipFile('$WHL_FILE') as zf:+ for name in zf.namelist():+ bn = os.path.basename(name)+ if bn.startswith('ragfs_python') and (bn.endswith('.so') or bn.endswith('.pyd') or bn.endswith('.dylib')):+ dst = os.path.join('$RAGFS_LIB_DIR', bn)+ with zf.open(name) as src, open(dst, 'wb') as f:+ f.write(src.read())+ os.chmod(dst, stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH)+ print(f' [OK] ragfs-python: extracted {bn} -> {dst}')+ sys.exit(0)+print('[ERROR] No ragfs_python native library found in wheel')+sys.exit(1)+" 2>&1 | tee -a "$LOG_FILE"+ else+ log "⚠️ maturin produced no wheel"+ fi+ else+ log "⚠️ maturin build failed, ragfs-python may not be available"+ fi+ rm -rf "$TMPDIR_RAGFS"+ fi++ RAGFS_SO_FINAL=$(ls -1 "$RAGFS_LIB_DIR"/ragfs_python*.so "$RAGFS_LIB_DIR"/ragfs_python*.pyd "$RAGFS_LIB_DIR"/ragfs_python*.dylib 2>/dev/null | head -1 || true)+ if [ -n "$RAGFS_SO_FINAL" ]; then+ log "✅ ragfs_python native extension verified: $RAGFS_SO_FINAL"+ else+ log "⚠️ WARNING: ragfs_python native extension not found in $RAGFS_LIB_DIR, server may fail to start"+ fi++ log "Installing dependencies..."UV_EXTRA_ARGS=""if [ -n "$GITHUB_WORKSPACE" ]; thenUV_EXTRA_ARGS="--index-url https://pypi.tuna.tsinghua.edu.cn/simple"