Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python
14.6 KB
Raw Blame
#!/usr/bin/env bash
set -euo pipefail

suite="${1:-${TEST_SUITE:-core}}"
container_runtime="${CONTAINER_RUNTIME:-$(command -v docker >/dev/null 2>&1 && echo docker || echo podman)}"
env_file="${ENV_FILE:-.env}"

if [[ -f "$env_file" ]]; then
  compose_cmd=("$container_runtime" compose --env-file "$env_file")
else
  compose_cmd=("$container_runtime" compose)
fi

# Load variables from env file to find COMPOSE_PROJECT_NAME and OPENSEARCH_PORT
COMPOSE_PROJECT_NAME=""
OPENSEARCH_PORT=""
LANGFLOW_PORT=""
if [[ ${OPENRAG_FRONTEND_BASE_PATH+x} != x ]]; then
  OPENRAG_FRONTEND_BASE_PATH=""
  if [[ -f "$env_file" ]]; then
    OPENRAG_FRONTEND_BASE_PATH="$(
      { grep -E '^OPENRAG_FRONTEND_BASE_PATH=' "$env_file" || [[ $? -eq 1 ]]; } |
        cut -d= -f2- |
        tr -d '"'\'
    )"
  fi
fi
if [[ -f "$env_file" ]]; then
  COMPOSE_PROJECT_NAME="$(grep -E '^COMPOSE_PROJECT_NAME=' "$env_file" | cut -d= -f2- | tr -d '"'\')"
  OPENSEARCH_PORT="$(grep -E '^OPENSEARCH_PORT=' "$env_file" | cut -d= -f2- | tr -d '"'\')"
  LANGFLOW_PORT="$(grep -E '^LANGFLOW_PORT=' "$env_file" | cut -d= -f2- | tr -d '"'\')"
fi

COMPOSE_PROJECT_NAME="${COMPOSE_PROJECT_NAME:-openrag}"
OPENSEARCH_PORT="${OPENSEARCH_PORT:-9200}"
LANGFLOW_PORT="${LANGFLOW_PORT:-7860}"
export OPENRAG_FRONTEND_BASE_PATH
frontend_base_path="${OPENRAG_FRONTEND_BASE_PATH#/}"
frontend_base_path="${frontend_base_path%/}"
if [[ -n "$frontend_base_path" ]]; then
  FRONTEND_URL="http://localhost:3000/${frontend_base_path}"
else
  FRONTEND_URL="http://localhost:3000"
fi

compose_cmd+=("-p" "$COMPOSE_PROJECT_NAME")

red=$'\033[0;31m'
purple=$'\033[38;2;119;62;255m'
yellow=$'\033[1;33m'
cyan=$'\033[0;36m'
green=$'\033[0;32m'
nc=$'\033[0m'

test_result=0

wait_for_url() {
  local label="$1"
  local url="$2"
  local attempts="${3:-60}"

  echo "${yellow}Waiting for ${label}...${nc}"
  for _ in $(seq 1 "$attempts"); do
    if curl -sf "$url" >/dev/null 2>&1; then
      return 0
    fi
    sleep 2
  done

  echo "${red}Timed out waiting for ${label} at ${url}${nc}"
  return 1
}

test_jwt_opensearch() {
  echo "${cyan}=== JWT OpenSearch Authentication Test ===${nc}"
  echo "${yellow}Generating test JWT token...${nc}"
  test_token="$(uv run python -c 'from utils.logging_config import configure_logging; configure_logging(log_level="CRITICAL"); from src.session_manager import SessionManager, AnonymousUser; sm = SessionManager("test"); print(sm.create_jwt_token(AnonymousUser()).removeprefix("Bearer "))' 2>/dev/null)"
  if [[ -z "$test_token" ]]; then
    echo "${red}Failed to generate JWT token${nc}"
    return 1
  fi

  echo "${yellow}Testing JWT against OpenSearch...${nc}"
  response_file="$(mktemp /tmp/jwt-os-diag.XXXXXX)"
  if ! curl --fail-with-body -k -s \
    -o "$response_file" \
    -H "Authorization: Bearer $test_token" \
    -H "Content-Type: application/json" \
    https://localhost:${OPENSEARCH_PORT}/documents/_search \
    -d '{"query":{"match_all":{}}}'; then
    echo "${red}curl command failed (network error or HTTP 4xx/5xx)${nc}"
    head -c 400 "$response_file" 2>/dev/null || true
    rm -f "$response_file"
    return 1
  fi

  echo "${green}Success - OpenSearch accepted JWT${nc}"
  echo "Response preview:"
  head -c 200 "$response_file" | sed 's/^/  /' || true
  rm -f "$response_file"
  echo ""
}

dump_logs() {
  echo "${red}=== Tests failed, saving container logs to service-logs/ ===${nc}"
  mkdir -p service-logs

  redact() {
    local pw="${OPENSEARCH_PASSWORD:-__unset__}"
    pw=$(printf '%s\n' "$pw" | sed -E 's/([][\\/.*+?^$|()])/\\\1/g')
    sed -E -e 's/Bearer [A-Za-z0-9._-]+/Bearer **REDACTED**/g' \
           -e 's/token=[A-Za-z0-9._-]+/token=**REDACTED**/g' \
           -e 's/sk-[A-Za-z0-9._-]+/sk-**REDACTED**/g' \
           -e "s/${pw}/**REDACTED**/g"
  }

  "$container_runtime" logs --tail 10000 "${COMPOSE_PROJECT_NAME}-langflow" 2>&1 | redact > service-logs/langflow.log || echo "${red}Could not get Langflow logs${nc}"
  "$container_runtime" logs --tail 10000 "${COMPOSE_PROJECT_NAME}-backend" 2>&1 | redact > service-logs/backend.log || echo "${red}Could not get backend logs${nc}"
  "$container_runtime" logs --tail 10000 "${COMPOSE_PROJECT_NAME}-frontend" 2>&1 | redact > service-logs/frontend.log || echo "${red}Could not get frontend logs${nc}"
  "$container_runtime" logs --tail 10000 "${COMPOSE_PROJECT_NAME}-opensearch" 2>&1 | redact > service-logs/opensearch.log || echo "${red}Could not get OpenSearch logs${nc}"
  if [[ -f ~/.openrag/tui/docling-serve.log ]]; then
    redact < ~/.openrag/tui/docling-serve.log > service-logs/docling.log || echo "${red}Could not get Docling logs${nc}"
  fi
}

generate_report() {
  uv run python scripts/ci/generate_test_report.py service-logs || true
  if [[ -n "${GITHUB_STEP_SUMMARY:-}" && -f service-logs/test-failure-report.md ]]; then
    cat service-logs/test-failure-report.md >> "$GITHUB_STEP_SUMMARY"
  fi
}

clean_infra() {
  echo "${yellow}Tearing down infra and cleaning up files...${nc}"
  uv run python scripts/docling_ctl.py stop || true
  
  local exit_status=0
  if ! "${compose_cmd[@]}" down -v 2>/dev/null; then
    echo "${red}ERROR: docker compose down failed${nc}"
    exit_status=1
  fi
  
  # NOTE: flows/*.json are git-tracked flow definitions, not runtime state -- only
  # flows/backup is generated at runtime. Deleting the top-level flows/ directory here
  # would remove those tracked files for the remainder of this job (no re-checkout
  # happens before a retry), breaking every flow-dependent request afterwards.
  if command -v sudo >/dev/null 2>&1; then
    if ! sudo rm -rf langflow-data config data keys opensearch-data openrag-documents flows/backup; then
      echo "${red}ERROR: sudo rm -rf failed${nc}"
      exit_status=1
    fi
  else
    if ! rm -rf langflow-data config data keys opensearch-data openrag-documents flows/backup 2>/dev/null; then
      local docker_rm_success=false
      for i in 1 2 3; do
        if docker run --rm -v "$(pwd):/work" alpine sh -c "rm -rf /work/opensearch-data /work/config /work/langflow-data /work/keys /work/data /work/flows/backup /work/openrag-documents"; then
          docker_rm_success=true
          break
        fi
        echo "Attempt $i failed, retrying in 5s..."
        sleep 5
      done
      if [ "$docker_rm_success" = "false" ]; then
        echo "${red}ERROR: alpine rm -rf failed${nc}"
        exit_status=1
      fi
    fi
  fi
  
  return "$exit_status"
}

trap_cleanup() {
  echo "${red}Interrupted! Cleaning up...${nc}"
  clean_infra || true
  exit 1
}
trap trap_cleanup SIGINT SIGTERM

run_attempt() {
  local exit_code=0

  echo "::group::Start Infrastructure"
  echo "${yellow}Starting infra for suite '${suite}' with OpenRAG version '${OPENRAG_VERSION:-latest}'${nc}"
  if ! OPENSEARCH_HOST=opensearch "${compose_cmd[@]}" up -d opensearch langflow openrag-backend openrag-frontend; then
    echo "${red}ERROR: docker compose up failed${nc}"
    return 1
  fi

  echo "${cyan}Architecture: $(uname -m), Platform: $(uname -s)${nc}"

  # Normalize host architecture
  host_arch="$(uname -m)"
  host_arch_norm=""
  if [[ "$host_arch" == "x86_64" ]]; then
    host_arch_norm="amd64"
  elif [[ "$host_arch" == "aarch64" || "$host_arch" == "arm64" ]]; then
    host_arch_norm="arm64"
  else
    host_arch_norm="$host_arch"
  fi

  # Get image architecture
  image_name="langflowai/openrag-backend:${OPENRAG_VERSION:-latest}"
  image_arch=$("$container_runtime" inspect --format '{{.Architecture}}' "$image_name" 2>/dev/null || echo "")

  if [[ -n "$image_arch" && "$image_arch" != "$host_arch_norm" ]]; then
    echo "${red}WARNING: Architecture mismatch detected!${nc}"
    echo "${red}Host architecture is '${host_arch}' (${host_arch_norm}), but container image architecture is '${image_arch}'.${nc}"
    echo "${red}The backend will run under QEMU emulation, which is extremely slow and may cause timeouts.${nc}"
  fi

  echo "${yellow}Starting docling-serve...${nc}"
  docling_start_failed=0
  docling_start_output="$(uv run python scripts/docling_ctl.py start --port 5001 --timeout 180 2>&1)" || docling_start_failed=1
  echo "$docling_start_output"
  if [[ "$docling_start_failed" = "1" ]]; then
    echo "${red}ERROR: docling_ctl.py start failed. Output above.${nc}"
    uv run python scripts/docling_ctl.py status 2>&1 || true
    return 1
  fi

  docling_endpoint="$(echo "$docling_start_output" | grep "Endpoint:" | awk '{print $2}')"
  if [[ -z "$docling_endpoint" ]]; then
    echo "${red}WARNING: docling-serve did not report an endpoint. Defaulting to http://localhost:5001${nc}"
    docling_endpoint="http://localhost:5001"
  fi

  echo "${purple}Docling-serve started at ${docling_endpoint}${nc}"
  echo "${yellow}Docling-serve status check:${nc}"
  uv run python scripts/docling_ctl.py status 2>&1 || true

  echo "${yellow}Waiting for backend OIDC endpoint...${nc}"
  for i in $(seq 1 60); do
    if "${compose_cmd[@]}" exec -T openrag-backend curl -sf http://localhost:8000/.well-known/openid-configuration >/dev/null 2>&1; then
      break
    fi
    if [[ "$i" -eq 60 ]]; then
      echo "${red}Backend OIDC endpoint was not reachable in time${nc}"
      echo "${yellow}--- Last 50 lines of backend logs ---${nc}"
      "$container_runtime" logs --tail 50 "${COMPOSE_PROJECT_NAME}-backend" 2>&1 || true
      echo "${yellow}------------------------------------${nc}"
      return 1
    fi
    sleep 2
  done

  echo "${yellow}Fixing JWT key ownership for test runner (host UID $(id -u))...${nc}"
  "$container_runtime" run --rm -v "$(pwd)/keys:/keys" alpine sh -c "chown $(id -u):$(id -g) /keys/private_key.pem /keys/public_key.pem 2>/dev/null; chmod 600 /keys/private_key.pem; chmod 644 /keys/public_key.pem 2>/dev/null" 2>/dev/null || true

  echo "${yellow}Waiting for OpenSearch security config to be fully applied...${nc}"
  for i in $(seq 1 60); do
    if "${compose_cmd[@]}" logs opensearch 2>&1 | grep -q "Security configuration applied successfully"; then
      echo "${purple}Security configuration applied${nc}"
      break
    fi
    if [[ "$i" -eq 60 ]]; then
      echo "${red}OpenSearch security config was not applied in time${nc}"
      return 1
    fi
    sleep 2
  done

  echo "${yellow}Verifying OIDC authenticator is active in OpenSearch...${nc}"
  for i in $(seq 1 30); do
    authc_config="$(curl -k -s -u "admin:${OPENSEARCH_PASSWORD}" https://localhost:${OPENSEARCH_PORT}/_opendistro/_security/api/securityconfig 2>/dev/null || true)"
    if echo "$authc_config" | grep -q "openid_auth_domain"; then
      echo "${purple}OIDC authenticator configured${nc}"
      echo "$authc_config" | grep -A 5 "openid_auth_domain" || true
      break
    fi
    if [[ "$i" -eq 30 ]]; then
      echo "${red}OIDC authenticator NOT found or unreachable in time!${nc}"
      echo "Security config output: $authc_config"
      return 1
    fi
    sleep 2
  done

  if ! wait_for_url "Langflow" "http://localhost:${LANGFLOW_PORT}/" 60; then
    return 1
  fi
  if ! wait_for_url "docling-serve at ${docling_endpoint}" "${docling_endpoint}/health" 60; then
    return 1
  fi
  echo "::endgroup::"

  # Clear and recreate service logs directory to isolate diagnostic artifacts per attempt
  rm -rf service-logs
  mkdir -p service-logs

  case "$suite" in
    core)
      echo "::group::Core Integration Tests"
      echo "${cyan}════════════════════════════════════════${nc}"
      echo "${purple} Core Integration Tests${nc}"
      echo "${cyan}════════════════════════════════════════${nc}"
      LOG_LEVEL="${LOG_LEVEL:-DEBUG}" \
        GOOGLE_OAUTH_CLIENT_ID="" \
        GOOGLE_OAUTH_CLIENT_SECRET="" \
        OPENSEARCH_HOST=localhost OPENSEARCH_PORT=${OPENSEARCH_PORT} \
        OPENSEARCH_USERNAME=admin OPENSEARCH_PASSWORD="${OPENSEARCH_PASSWORD}" \
        DISABLE_STARTUP_INGEST="${DISABLE_STARTUP_INGEST:-true}" \
        uv run pytest tests/integration/core -vv -s --log-file=service-logs/pytest-core.log --log-file-level=DEBUG --junitxml=service-logs/junit-core.xml || exit_code=1
      echo "::endgroup::"
      test_jwt_opensearch || exit_code=1
      ;;
    sdk-python)
      if ! wait_for_url "frontend at ${FRONTEND_URL}" "${FRONTEND_URL}/" 60; then
        return 1
      fi
      echo "::group::SDK Integration Tests (Python)"
      echo "${cyan}════════════════════════════════════════${nc}"
      echo "${purple} SDK Integration Tests (Python)${nc}"
      echo "${cyan}════════════════════════════════════════${nc}"
      if ! uv pip install --quiet -e sdks/python; then
        echo "${red}ERROR: uv pip install failed${nc}"
        return 1
      fi
      SDK_TESTS_ONLY=true OPENRAG_URL="${FRONTEND_URL}" uv run pytest tests/integration/sdk/ -vv -s --log-file=service-logs/pytest-sdk.log --log-file-level=DEBUG --junitxml=service-logs/junit-sdk-python.xml || exit_code=1
      echo "::endgroup::"
      ;;
    sdk-typescript)
      if ! wait_for_url "frontend at ${FRONTEND_URL}" "${FRONTEND_URL}/" 60; then
        return 1
      fi
      echo "::group::SDK Integration Tests (TypeScript)"
      echo "${cyan}════════════════════════════════════════${nc}"
      echo "${purple} SDK Integration Tests (TypeScript)${nc}"
      echo "${cyan}════════════════════════════════════════${nc}"
      cd sdks/typescript
      npm install && npm run build && OPENRAG_URL="${FRONTEND_URL}" npm test -- --reporter=junit --outputFile=../../service-logs/junit-sdk-typescript.xml || exit_code=1
      cd ../..
      echo "::endgroup::"
      ;;
    *)
      echo "${red}Unknown integration suite: ${suite}${nc}"
      exit_code=1
      ;;
  esac

  return "$exit_code"
}

if [[ -z "${OPENSEARCH_PASSWORD:-}" ]]; then
  echo "${red}OPENSEARCH_PASSWORD is required${nc}"
  exit 1
fi

echo "${yellow}Installing test dependencies...${nc}"
uv sync --quiet --group dev

max_attempts=2
test_result=0

for attempt in $(seq 1 "$max_attempts"); do
  echo "${yellow}=== Starting Integration Suite '${suite}' - Attempt $attempt of $max_attempts ===${nc}"
  
  if run_attempt; then
    echo "${green}Attempt $attempt succeeded!${nc}"
    test_result=0
    break
  else
    echo "${red}Attempt $attempt failed!${nc}"
    test_result=1
    dump_logs || true
    
    if [ "$attempt" -lt "$max_attempts" ]; then
      if ! clean_infra; then
        echo "${red}ERROR: Cleanup failed. Aborting further attempts.${nc}"
        test_result=1
        break
      fi
      echo "${yellow}Retrying in 5 seconds...${nc}"
      sleep 5
    fi
  fi
done

generate_report || true
exit "$test_result"