Enterprise agent platform for building, deploying, and operating AI teammates.
"""
End-to-end tests for KB lifecycle management.
This module tests the complete knowledge base lifecycle:
1. Create new KB (collection)
2. Ingest multiple documents
3. Search and verify content
4. Delete documents
5. Delete KB (collection)
"""
from __future__ import annotations
import sys
import tempfile
from pathlib import Path
import pytest
from fastapi.testclient import TestClient
from tests.web_integration.http_helpers import eventually, http_detail
from xagent.core.tools.core.RAG_tools.kb.collection_handle import deployment_kb_backend
from xagent.core.tools.core.RAG_tools.storage.factory import get_vector_index_store
pytestmark = [
pytest.mark.e2e,
pytest.mark.contract_stub,
pytest.mark.usefixtures("kb_engine"),
]
def _log(msg: str) -> None:
"""Helper to log messages immediately."""
sys.stdout.write(f"{msg}\n")
sys.stdout.flush()
def _reupload_helpers(
client: TestClient, auth_headers: dict[str, str], collection: str
):
"""Return ``upload(text)`` for one file name and ``found(word)`` for a collection."""
def upload(text: str) -> dict:
response = client.post(
"/api/kb/ingest",
files={"file": ("notes.txt", text.encode(), "text/plain")},
data={"collection": collection, "chunk_size": 100, "chunk_overlap": 0},
headers=auth_headers,
)
assert response.status_code == 200, http_detail(response)
assert response.json()["status"] == "success", http_detail(response)
return response.json()
def found(query: str) -> bool:
response = client.post(
"/api/kb/search",
data={
"collection": collection,
"query_text": query,
"embedding_model_id": "e2e-test-embedding",
"search_type": "sparse",
},
headers=auth_headers,
)
assert response.status_code == 200, http_detail(response)
return bool(response.json()["results"])
return upload, found
# ==========================================
# TEST-SPECIFIC FIXTURES
# ==========================================
# Note: _StubEmbeddingAdapter, stub_embedding_adapter, and mock_rag_pipeline
# are provided by conftest.py with autouse=True
@pytest.fixture
def sample_lifecycle_files():
"""Create sample test files for lifecycle testing."""
files = {}
with tempfile.TemporaryDirectory() as temp_dir:
# Create diverse test files
test_files = {
"document1.txt": "First document with important information about testing.",
"document2.md": "# Second Document\n\nThis is a markdown document with **bold** text.",
"document3.json": '{"title": "Third Document", "content": "JSON data for testing"}',
}
for filename, content in test_files.items():
file_path = Path(temp_dir) / filename
file_path.write_text(content, encoding="utf-8")
files[filename] = str(file_path)
yield files, temp_dir
# ==========================================
# E2E TEST CLASSES
# ==========================================
class TestKBLifecycleE2E:
"""
End-to-end tests for complete KB lifecycle.
These tests simulate the complete user workflow from creating
a knowledge base through to deletion, ensuring all operations
work correctly together.
"""
@pytest.mark.e2e
@pytest.mark.slow
def test_complete_kb_lifecycle_single_document(
self,
client: TestClient,
auth_headers: dict[str, str],
sample_lifecycle_files: tuple[dict[str, str], str],
):
"""Test complete KB lifecycle: create → ingest → search → delete."""
_log("\n=== START test_complete_kb_lifecycle_single_document ===")
files, temp_dir = sample_lifecycle_files
collection_name = "e2e_lifecycle_single"
file_path = files["document1.txt"]
_log(f"Collection name: {collection_name}")
_log(f"File path: {file_path}")
# Step 1: Create collection and ingest document
_log("Step 1: Ingesting document...")
with open(file_path, "rb") as f:
ingest_response = client.post(
"/api/kb/ingest",
files={"file": ("document1.txt", f, "text/plain")},
data={
"collection": collection_name,
"parse_method": "default",
},
headers=auth_headers,
)
_log(f"Ingest response status: {ingest_response.status_code}")
# Verify ingestion succeeded
assert ingest_response.status_code == 200
ingest_result = ingest_response.json()
assert ingest_result["status"] in ["success", "partial"]
_log(f"Ingest result status: {ingest_result.get('status')}")
# Step 2: List collections to verify KB exists
_log("Step 2: Listing collections...")
list_response = client.get("/api/kb/collections", headers=auth_headers)
assert list_response.status_code == 200
collections = list_response.json()
assert "collections" in collections
_log(f"Collections count: {len(collections.get('collections', []))}")
# Step 3: Try to delete the document
_log("Step 3: Deleting document...")
delete_doc_response = client.delete(
f"/api/kb/collections/{collection_name}/documents/document1.txt",
headers=auth_headers,
)
# Deleting just-uploaded document should succeed
assert delete_doc_response.status_code == 200
_log(f"Delete response status: {delete_doc_response.status_code}")
_log("=== END test_complete_kb_lifecycle_single_document ===\n")
@pytest.mark.e2e
@pytest.mark.slow
def test_search_stops_finding_a_deleted_document_and_keeps_the_others(
self,
client: TestClient,
auth_headers: dict[str, str],
sample_lifecycle_files: tuple[dict[str, str], str],
kb_engine: str,
):
"""Keyword search finds each ingested text; a delete removes only its own."""
assert deployment_kb_backend().value == kb_engine
files, _ = sample_lifecycle_files
collection_name = "e2e_lifecycle_search"
for filename in ("document1.txt", "document2.md"):
with open(files[filename], "rb") as f:
ingest_response = client.post(
"/api/kb/ingest",
files={"file": (filename, f, "text/plain")},
data={"collection": collection_name},
headers=auth_headers,
)
assert ingest_response.status_code == 200
assert ingest_response.json()["status"] == "success"
def found(query: str) -> list[str]:
response = client.post(
"/api/kb/search",
data={
"collection": collection_name,
"query_text": query,
"embedding_model_id": "e2e-test-embedding",
"search_type": "sparse",
},
headers=auth_headers,
)
assert response.status_code == 200, http_detail(response)
assert response.json()["status"] == "success", http_detail(response)
return [hit["text"] for hit in response.json()["results"]]
eventually(lambda: bool(found("important information")))
eventually(lambda: bool(found("markdown document")))
assert found("important information") == [
"First document with important information about testing."
]
delete_response = client.delete(
f"/api/kb/collections/{collection_name}/documents/document1.txt",
headers=auth_headers,
)
assert delete_response.status_code == 200
eventually(lambda: not found("important information"))
assert found("markdown document")
@pytest.mark.e2e
@pytest.mark.slow
def test_reupload_with_changed_content_replaces_the_old_content(
self,
client: TestClient,
auth_headers: dict[str, str],
kb_engine: str,
):
"""A same-name upload with new text is searchable and drops the old text."""
assert deployment_kb_backend().value == kb_engine
upload, found = _reupload_helpers(client, auth_headers, "e2e_reupload_changed")
upload("alpha zebra lantern")
eventually(lambda: found("zebra"))
unchanged = upload("alpha zebra lantern")
assert "no pending embeddings" in unchanged["message"]
assert found("zebra")
changed = upload("gamma quokka prism")
assert "no pending embeddings" not in changed["message"]
eventually(lambda: found("quokka"))
eventually(lambda: not found("zebra"))
@pytest.mark.e2e
@pytest.mark.slow
def test_failed_reupload_keeps_the_old_content_and_a_retry_replaces_it(
self,
client: TestClient,
auth_headers: dict[str, str],
stub_embedding_adapter,
monkeypatch: pytest.MonkeyPatch,
kb_engine: str,
):
"""An embedder failure leaves the old text searchable; the retry swaps it."""
assert deployment_kb_backend().value == kb_engine
upload, found = _reupload_helpers(client, auth_headers, "e2e_reupload_failed")
upload("alpha zebra lantern")
eventually(lambda: found("zebra"))
real_encode = stub_embedding_adapter.encode
def broken(*args, **kwargs):
raise RuntimeError("embedder down")
monkeypatch.setattr(stub_embedding_adapter, "encode", broken)
response = client.post(
"/api/kb/ingest",
files={"file": ("notes.txt", b"gamma quokka prism", "text/plain")},
data={"collection": "e2e_reupload_failed", "max_retries": 0},
headers=auth_headers,
)
assert response.status_code == 500, http_detail(response)
assert found("zebra")
assert not found("quokka")
monkeypatch.setattr(stub_embedding_adapter, "encode", real_encode)
retried = upload("gamma quokka prism")
assert "no pending embeddings" not in retried["message"]
eventually(lambda: found("quokka"))
eventually(lambda: not found("zebra"))
@pytest.mark.e2e
@pytest.mark.slow
def test_reverting_to_the_old_content_after_a_failed_upload_drops_the_failed_text(
self,
client: TestClient,
auth_headers: dict[str, str],
stub_embedding_adapter,
monkeypatch: pytest.MonkeyPatch,
kb_engine: str,
):
"""v1, a v2 whose embedding fails, then v1 again: v2 must not stay searchable."""
assert deployment_kb_backend().value == kb_engine
upload, found = _reupload_helpers(client, auth_headers, "e2e_reupload_revert")
upload("alpha zebra lantern")
eventually(lambda: found("zebra"))
real_encode = stub_embedding_adapter.encode
monkeypatch.setattr(
stub_embedding_adapter,
"encode",
lambda *args, **kwargs: (_ for _ in ()).throw(RuntimeError("down")),
)
response = client.post(
"/api/kb/ingest",
files={"file": ("notes.txt", b"gamma quokka prism", "text/plain")},
data={"collection": "e2e_reupload_revert", "max_retries": 0},
headers=auth_headers,
)
assert response.status_code == 500, http_detail(response)
encoded: list[str] = []
def recording(*args, **kwargs):
encoded.append(repr((args, kwargs)))
return real_encode(*args, **kwargs)
monkeypatch.setattr(stub_embedding_adapter, "encode", recording)
def chunk_texts() -> list[str]:
return [
row["text"]
for batch in get_vector_index_store().iter_batches(
table_name="chunks",
filters={"collection": "e2e_reupload_revert"},
is_admin=True,
)
for row in batch.to_pylist()
]
assert any("quokka" in text for text in chunk_texts())
upload("alpha zebra lantern")
eventually(lambda: found("zebra"))
eventually(lambda: not found("quokka"))
assert not any("quokka" in call for call in encoded)
assert not any("quokka" in text for text in chunk_texts())
@pytest.mark.e2e
@pytest.mark.slow
def test_reupload_with_new_content_and_new_parse_method_drops_the_old_text(
self,
client: TestClient,
auth_headers: dict[str, str],
kb_engine: str,
):
"""A changed parse method gives a new parse_hash; the old chunks still go."""
assert deployment_kb_backend().value == kb_engine
upload, found = _reupload_helpers(client, auth_headers, "e2e_reupload_parse")
upload("alpha zebra lantern")
eventually(lambda: found("zebra"))
response = client.post(
"/api/kb/ingest",
files={"file": ("notes.txt", b"gamma quokka prism", "text/plain")},
data={"collection": "e2e_reupload_parse", "parse_method": "deepdoc"},
headers=auth_headers,
)
assert response.status_code == 200, http_detail(response)
eventually(lambda: found("quokka"))
eventually(lambda: not found("zebra"))
@pytest.mark.e2e
@pytest.mark.slow
def test_reupload_replaces_every_chunk_of_a_multi_chunk_document(
self,
client: TestClient,
auth_headers: dict[str, str],
kb_engine: str,
):
"""No chunk of the old text survives when the new text also has several."""
assert deployment_kb_backend().value == kb_engine
upload, found = _reupload_helpers(client, auth_headers, "e2e_reupload_multi")
old_words = ["zebra", "lantern", "walnut", "harbor"]
new_words = ["quokka", "prism", "meadow", "falcon"]
def text(words: list[str]) -> str:
return "\n\n".join(f"{word} " * 20 for word in words)
assert upload(text(old_words))["chunk_count"] > 1
eventually(lambda: all(found(word) for word in old_words))
assert upload(text(new_words))["chunk_count"] > 1
eventually(lambda: all(found(word) for word in new_words))
eventually(lambda: not any(found(word) for word in old_words))
@pytest.mark.e2e
@pytest.mark.slow
def test_complete_kb_lifecycle_multiple_documents(
self,
client: TestClient,
auth_headers: dict[str, str],
sample_lifecycle_files: tuple[dict[str, str], str],
):
"""Test KB lifecycle with multiple documents of different formats."""
_log("\n=== START test_complete_kb_lifecycle_multiple_documents ===")
files, temp_dir = sample_lifecycle_files
collection_name = "e2e_lifecycle_multi"
_log(f"Collection name: {collection_name}")
# Step 1: Ingest multiple documents
_log("Step 1: Ingesting multiple documents...")
uploaded_docs = []
for filename in ["document1.txt", "document2.md", "document3.json"]:
print(f" Ingesting {filename}...")
file_path = files[filename]
with open(file_path, "rb") as f:
response = client.post(
"/api/kb/ingest",
files={"file": (filename, f, "text/plain")},
data={"collection": collection_name},
headers=auth_headers,
)
print(f" Response status: {response.status_code}")
if response.status_code == 200:
uploaded_docs.append(filename)
# Verify at least some documents were ingested
assert len(uploaded_docs) >= 1
print(f"Uploaded {len(uploaded_docs)} documents")
# Step 2: List collections
_log("Step 2: Listing collections...")
list_response = client.get("/api/kb/collections", headers=auth_headers)
assert list_response.status_code == 200
print(f"List response status: {list_response.status_code}")
# Step 3: Try to delete each document
_log("Step 3: Deleting documents...")
for doc_name in uploaded_docs:
print(f" Deleting {doc_name}...")
delete_response = client.delete(
f"/api/kb/collections/{collection_name}/documents/{doc_name}",
headers=auth_headers,
)
# Deleting just-uploaded documents should succeed
assert delete_response.status_code == 200
print(f" Delete response: {delete_response.status_code}")
_log("=== END test_complete_kb_lifecycle_multiple_documents ===\n")
@pytest.mark.e2e
@pytest.mark.slow
def test_delete_collection_after_ingestion(
self,
client: TestClient,
auth_headers: dict[str, str],
sample_lifecycle_files: tuple[dict[str, str], str],
):
"""Test deleting an entire collection after document ingestion."""
_log("\n=== START test_delete_collection_after_ingestion ===")
files, temp_dir = sample_lifecycle_files
collection_name = "e2e_delete_collection"
file_path = files["document1.txt"]
print(f"Collection name: {collection_name}")
# Step 1: Ingest a document
_log("Step 1: Ingesting document...")
with open(file_path, "rb") as f:
ingest_response = client.post(
"/api/kb/ingest",
files={"file": ("document1.txt", f, "text/plain")},
data={"collection": collection_name},
headers=auth_headers,
)
print(f"Ingest response: {ingest_response.status_code}")
if ingest_response.status_code == 200:
# Step 2: Delete the entire collection
_log("Step 2: Deleting collection...")
delete_response = client.delete(
f"/api/kb/collections/{collection_name}",
headers=auth_headers,
)
# Deleting a collection that exists should succeed
assert delete_response.status_code == 200
print(f"Delete response: {delete_response.status_code}")
_log("=== END test_delete_collection_after_ingestion ===\n")
@pytest.mark.e2e
@pytest.mark.slow
def test_collection_isolation(
self,
client: TestClient,
auth_headers: dict[str, str],
sample_lifecycle_files: tuple[dict[str, str], str],
):
"""Test that different collections maintain document isolation."""
_log("\n=== START test_collection_isolation ===")
files, temp_dir = sample_lifecycle_files
# Create two separate collections
collection_a = "e2e_isolation_a"
collection_b = "e2e_isolation_b"
# Ingest into collection A
file_path_a = files["document1.txt"]
with open(file_path_a, "rb") as f:
response_a = client.post(
"/api/kb/ingest",
files={"file": ("doc_a.txt", f, "text/plain")},
data={"collection": collection_a},
headers=auth_headers,
)
# Ingest into collection B
file_path_b = files["document2.md"]
with open(file_path_b, "rb") as f:
response_b = client.post(
"/api/kb/ingest",
files={"file": ("doc_b.md", f, "text/plain")},
data={"collection": collection_b},
headers=auth_headers,
)
# At least one should succeed
assert response_a.status_code == 200 or response_b.status_code == 200
# List collections should show both if they were created
list_response = client.get("/api/kb/collections", headers=auth_headers)
assert list_response.status_code == 200
@pytest.mark.e2e
@pytest.mark.slow
def test_collection_config_management(
self,
client: TestClient,
auth_headers: dict[str, str],
):
"""Test saving and retrieving collection configuration."""
_log("\n=== START test_collection_config_management ===")
collection_name = "e2e_config_test"
# Save collection config
_log("Saving config...")
config_response = client.post(
f"/api/kb/collections/{collection_name}/config",
json={
"parse_method": "deepdoc",
"chunk_strategy": "fixed_size",
"chunk_size": 500,
"chunk_overlap": 50,
"embedding_model_id": "text-embedding-v3",
},
headers=auth_headers,
)
print(f"Config response: {config_response.status_code}")
# Config save should succeed
assert config_response.status_code == 200
_log("=== END test_collection_config_management ===\n")
@pytest.mark.e2e
@pytest.mark.slow
def test_ingest_with_custom_config(
self,
client: TestClient,
auth_headers: dict[str, str],
sample_lifecycle_files: tuple[dict[str, str], str],
):
"""Test ingesting documents with custom ingestion configuration."""
files, temp_dir = sample_lifecycle_files
collection_name = "e2e_custom_config"
file_path = files["document1.txt"]
# Ingest with custom chunking configuration
with open(file_path, "rb") as f:
response = client.post(
"/api/kb/ingest",
files={"file": ("document1.txt", f, "text/plain")},
data={
"collection": collection_name,
"parse_method": "deepdoc",
"chunk_strategy": "fixed_size",
"chunk_size": "300",
"chunk_overlap": "100",
"separators": '["\\n\\n", "\\n", " ", ""]',
},
headers=auth_headers,
)
assert response.status_code == 200
result = response.json()
assert result["status"] in ["success", "partial"]
@pytest.mark.e2e
@pytest.mark.slow
def test_document_listing_after_multiple_ingestions(
self,
client: TestClient,
auth_headers: dict[str, str],
sample_lifecycle_files: tuple[dict[str, str], str],
):
"""Test that documents are correctly listed after multiple ingestions."""
files, temp_dir = sample_lifecycle_files
collection_name = "e2e_listing_test"
# Ingest multiple documents
uploaded = []
for filename in ["document1.txt", "document2.md"]:
file_path = files[filename]
with open(file_path, "rb") as f:
response = client.post(
"/api/kb/ingest",
files={"file": (filename, f, "text/plain")},
data={"collection": collection_name},
headers=auth_headers,
)
if response.status_code == 200:
uploaded.append(filename)
if len(uploaded) > 0:
# No collection-detail GET; verify filenames via documents/check.
check_response = client.post(
f"/api/kb/collections/{collection_name}/documents/check",
json={"filenames": uploaded},
headers=auth_headers,
)
assert check_response.status_code == 200
existing = set(check_response.json().get("existing_filenames", []))
assert existing.issuperset(set(uploaded))
@pytest.mark.e2e
@pytest.mark.slow
def test_error_handling_invalid_collection_name(
self,
client: TestClient,
auth_headers: dict[str, str],
sample_lifecycle_files: tuple[dict[str, str], str],
):
"""Test error handling with invalid collection names."""
files, temp_dir = sample_lifecycle_files
file_path = files["document1.txt"]
# Try various invalid collection names
invalid_names = [
"../../../etc",
"collection/../other",
"collection with spaces",
"collection/with/slashes",
]
for invalid_name in invalid_names:
with open(file_path, "rb") as f:
response = client.post(
"/api/kb/ingest",
files={"file": ("test.txt", f, "text/plain")},
data={"collection": invalid_name},
headers=auth_headers,
)
# Path traversal names are rejected because basename != original
# Only "collection with spaces" passes validation
if invalid_name == "collection with spaces":
assert response.status_code == 200
else:
assert response.status_code == 422
@pytest.mark.e2e
@pytest.mark.slow
def test_multiple_document_ingestion(
self,
client: TestClient,
auth_headers: dict[str, str],
sample_lifecycle_files: tuple[dict[str, str], str],
):
"""Test handling multiple ingestion requests sequentially."""
files, temp_dir = sample_lifecycle_files
collection_name = "e2e_multiple_ingestion"
# Ingest multiple files sequentially
results = []
for filename in ["document1.txt", "document2.md", "document3.json"]:
file_path = files[filename]
with open(file_path, "rb") as f:
response = client.post(
"/api/kb/ingest",
files={"file": (filename, f, "text/plain")},
data={"collection": collection_name},
headers=auth_headers,
)
results.append(response.status_code)
# All ingestions should succeed
assert all(code == 200 for code in results)