Enterprise agent platform for building, deploying, and operating AI teammates.
"""References and cleanup claims serialize through the production boundaries."""
import asyncio
import hashlib
from concurrent.futures import ThreadPoolExecutor
from contextlib import ExitStack
from datetime import UTC, datetime, timedelta
from threading import Event
import pytest
import sqlalchemy as sa
from filelock import FileLock
from sqlalchemy.orm import sessionmaker
from tests.shared.async_waits import DB_PROGRESS_TIMEOUT
from xagent.core.tools.core.RAG_tools.storage.lancedb_stores import (
LanceDBVectorIndexStore,
)
from xagent.providers.vector_store.lancedb import clear_connection_cache
from xagent.web.models.database import Base, configure_db, get_engine, get_session_local
from xagent.web.models.kb_ingest_target import KBIngestTarget
from xagent.web.models.uploaded_file import UploadedFile
from xagent.web.models.uploaded_file_cleanup_fence import UploadedFileCleanupFence
from xagent.web.models.user import User
from xagent.web.services.kb_ingest_targets import (
admit_kb_ingest_target,
release_kb_ingest_target_generation,
tombstone_kb_ingest_target,
tombstone_kb_ingest_targets_for_collection,
)
from xagent.web.services.kb_reference_protection import FileReferenceConflict
from xagent.web.services.orphan_upload_gc import (
TASKLESS_SHARE_UPLOAD_SOURCE,
cleanup_orphaned_taskless_uploads,
)
@pytest.fixture(
params=["sqlite", pytest.param("postgresql", marks=pytest.mark.postgresql)]
)
def boundary(request, tmp_path, monkeypatch):
from tests.shared.postgres_disposable import disposable_database_factory
from xagent.web.models import database
from xagent.web.services.kb_reference_protection import (
install_file_reference_validator,
)
previous = database._SessionLocal, database._engine
monkeypatch.setenv("LANCEDB_DIR", str(tmp_path / "lance"))
monkeypatch.setenv("XAGENT_UPLOADS_DIR", str(tmp_path / "uploads"))
monkeypatch.setenv("XAGENT_FILE_MATERIALIZE_DIR", str(tmp_path / "materialized"))
clear_connection_cache()
stack = ExitStack()
if request.param == "sqlite":
configure_db(f"sqlite:///{tmp_path / 'metadata.db'}")
engine = get_engine()
else:
make = stack.enter_context(disposable_database_factory("kb_reference"))
engine = make("boundary")
database._engine = engine
database._SessionLocal = sessionmaker(bind=engine, autoflush=False)
install_file_reference_validator()
Base.metadata.create_all(engine)
sessions = get_session_local()
path = tmp_path / "uploads" / "user_1" / "source.txt"
path.parent.mkdir(parents=True)
path.write_text("retained source")
with sessions.begin() as db:
db.add_all(
[
User(id=1, username="owner", password_hash="unused"),
User(id=2, username="other", password_hash="unused"),
]
)
db.flush()
db.add(
UploadedFile(
user_id=1,
file_id="source",
filename=path.name,
storage_path=str(path),
storage_key="users/1/uploads/source/source.txt",
checksum=hashlib.sha256(b"retained source").hexdigest(),
storage_status="available",
upload_source=TASKLESS_SHARE_UPLOAD_SOURCE,
created_at=datetime.now(UTC) - timedelta(days=10),
)
)
monkeypatch.setattr(
"xagent.web.services.orphan_upload_gc.delete_uploaded_file_compensation_object",
lambda **_kwargs: "absent",
)
yield sessions, LanceDBVectorIndexStore(), path
clear_connection_cache()
engine.dispose()
database._SessionLocal, database._engine = previous
stack.close()
def document(*, owner=1, file_id="source", doc_id="doc"):
return {
"collection": "kb",
"doc_id": doc_id,
"file_id": file_id,
"source_path": "/source",
"file_type": "txt",
"content_hash": "hash",
"uploaded_at": datetime.now(UTC),
"title": None,
"language": None,
"user_id": owner,
}
@pytest.mark.parametrize("owner", [1, 2])
def test_existing_document_prevents_cleanup_and_local_unlink(boundary, owner):
sessions, store, path = boundary
store.upsert_documents([document(owner=owner)])
with sessions() as db:
result = cleanup_orphaned_taskless_uploads(db, older_than_seconds=1)
assert result.deleted == 0
assert db.query(UploadedFile).one().storage_status == "available"
assert path.read_text() == "retained source"
def claim(sessions):
with sessions() as db:
return cleanup_orphaned_taskless_uploads(db, older_than_seconds=1)
def handle(store):
from xagent.core.tools.core.RAG_tools.kb.collection_handle import (
LanceDBCollectionHandle,
)
from xagent.core.tools.core.RAG_tools.kb.models import (
KBAccessMode,
KBBackendCapabilities,
KBCollectionContext,
KBStorageBackend,
KBUserScope,
)
return LanceDBCollectionHandle(
KBCollectionContext(
collection="kb",
user_scope=KBUserScope(1, False),
access_mode=KBAccessMode.WRITE,
allow_create=True,
hide_missing=False,
metadata_store=None,
vector_index_store=store,
ingestion_status_store=None,
main_pointer_store=None,
backend=KBStorageBackend.LANCEDB,
capabilities=KBBackendCapabilities.lancedb(),
)
)
def write(store, mode):
from xagent.core.tools.core.RAG_tools.core.schemas import DocumentRecordDetail
from xagent.core.tools.core.RAG_tools.kb.models import KBDocumentRowsSnapshot
row = document()
if mode == "sync":
store.upsert_documents([row])
elif mode == "async":
asyncio.run(store.upsert_documents_async([row]))
elif mode == "restore":
handle(store).restore_document(DocumentRecordDetail.from_row(row))
else:
handle(store).restore_document_rows(
KBDocumentRowsSnapshot("kb", ("doc",), {"documents": [row]}),
user_id=1,
is_admin=False,
)
@pytest.mark.parametrize("mode", ["sync", "async", "restore", "snapshot"])
def test_claimed_and_settled_identity_rejects_late_production_writes(boundary, mode):
sessions, store, _ = boundary
assert claim(sessions).deleted == 1
with pytest.raises(FileReferenceConflict):
write(store, mode)
assert store.list_document_records_by_file_ids(["source"]) == []
def test_new_standalone_and_republished_uploads_are_accepted(boundary):
from xagent.web.services.uploaded_file_store import UploadedFileStore
sessions, store, path = boundary
assert claim(sessions).deleted == 1
store.upsert_documents([document(file_id="standalone", doc_id="standalone")])
path.write_text("new publication")
with sessions.begin() as db:
UploadedFileStore(db).add_already_durable(
UploadedFile(
user_id=1,
file_id="source",
filename=path.name,
storage_path=str(path),
storage_status="available",
storage_key="durable/new-generation",
checksum="new",
)
)
store.upsert_documents([document()])
assert {
row.file_id
for row in store.list_document_records_by_file_ids(["source", "standalone"])
} == {"source", "standalone"}
def test_standalone_without_sql_rejects_retired_id_but_accepts_new_source(
boundary, monkeypatch
):
from xagent.core.tools.core.RAG_tools.storage import file_reference
from xagent.web.services.uploaded_file_store import UploadedFileStore
sessions, store, path = boundary
assert claim(sessions).deleted == 1
monkeypatch.setattr(file_reference, "_validator", None)
with pytest.raises(RuntimeError, match="Source file was claimed"):
store.upsert_documents([document()])
store.upsert_documents([document(file_id="independent", doc_id="independent")])
path.write_text("new publication")
with sessions.begin() as db:
UploadedFileStore(db).add_already_durable(
UploadedFile(
user_id=1,
file_id="source",
filename=path.name,
storage_path=str(path),
storage_status="available",
storage_key="durable/new-generation",
checksum="new",
)
)
store.upsert_documents([document()])
assert len(store.list_document_records_by_file_ids(["source"])) == 1
def test_republication_rollback_keeps_standalone_fence(boundary, monkeypatch):
from xagent.core.tools.core.RAG_tools.storage import file_reference
from xagent.web.services.uploaded_file_store import UploadedFileStore
sessions, store, path = boundary
assert claim(sessions).deleted == 1
with sessions() as db:
UploadedFileStore(db).add_already_durable(
UploadedFile(
user_id=1,
file_id="source",
filename=path.name,
storage_path=str(path),
storage_status="available",
storage_key="durable/new",
checksum="new",
)
)
db.rollback()
monkeypatch.setattr(file_reference, "_validator", None)
with pytest.raises(RuntimeError, match="Source file was claimed"):
store.upsert_documents([document()])
def admit(sessions, *, generation="one", file_id="source", job_id=None):
from xagent.web.services.background_jobs import create_background_job
with sessions() as db:
if job_id is None:
job = create_background_job(
db,
user_id=2,
job_type="kb.ingest.document",
payload={"file_id": file_id},
)
job_id = str(job.id)
admit_kb_ingest_target(
db,
user_id=2,
collection="kb",
target_path="canonical/source",
file_id=file_id,
generation_id=generation,
job_id=job_id,
file_sha256="hash",
)
return job_id
@pytest.mark.parametrize("release", ["release", "tombstone", "collection"])
def test_pending_target_protects_until_actual_release(boundary, release):
sessions, _, path = boundary
admit(sessions)
assert claim(sessions).deleted == 0
assert path.exists()
with sessions() as db:
if release == "release":
assert release_kb_ingest_target_generation(
db,
user_id=2,
collection="kb",
target_path="canonical/source",
generation_id="one",
)
elif release == "tombstone":
tombstone_kb_ingest_target(
db,
user_id=2,
collection="kb",
target_path="canonical/source",
)
else:
assert (
tombstone_kb_ingest_targets_for_collection(
db, user_id=2, collection="kb"
)
== 1
)
assert claim(sessions).deleted == 1
def test_generation_replacement_and_stale_release_do_not_drop_protection(boundary):
sessions, _, _ = boundary
admit(sessions)
admit(sessions, generation="two")
with sessions() as db:
assert not release_kb_ingest_target_generation(
db,
user_id=2,
collection="kb",
target_path="canonical/source",
generation_id="one",
)
assert claim(sessions).deleted == 0
admit(sessions, generation="three", file_id="new-source")
assert claim(sessions).deleted == 1
def test_document_deletion_releases_protection(boundary):
sessions, store, _ = boundary
store.upsert_documents([document(owner=2)])
assert claim(sessions).deleted == 0
store.delete_document_record(
collection_name="kb", doc_id="doc", user_id=2, is_admin=False
)
assert claim(sessions).deleted == 1
def test_reference_query_failure_prevents_claim_and_unlink(boundary, monkeypatch):
sessions, _, path = boundary
def unavailable(_ids):
raise OSError("reference query unavailable")
monkeypatch.setattr(
"xagent.web.services.kb_file_service.find_referenced_file_ids", unavailable
)
assert claim(sessions).deleted == 0
assert path.exists()
with sessions() as db:
assert db.query(UploadedFile).one().storage_status == "available"
assert db.query(UploadedFileCleanupFence).count() == 0
@pytest.mark.parametrize("mode", ["sync", "async", "restore", "snapshot", "target"])
@pytest.mark.parametrize("first", ["reference", "claim"])
def test_independent_connections_race_with_both_winners(
boundary, monkeypatch, mode, first
):
from xagent.core.tools.core.RAG_tools.storage import file_reference
sessions, store, path = boundary
entered, proceed, blocked = Event(), Event(), Event()
job_id = None
if mode == "target":
from xagent.web.services.background_jobs import create_background_job
with sessions() as db:
job_id = str(
create_background_job(
db,
user_id=2,
job_type="kb.ingest.document",
payload={"file_id": "source"},
).id
)
original_acquire = FileLock._acquire
def acquire(lock):
original_acquire(lock)
if lock._context.lock_file_fd is None:
blocked.set()
monkeypatch.setattr(FileLock, "_acquire", acquire)
validator = file_reference._validator
def pause_reference(ids):
validator(ids)
if first == "reference":
entered.set()
assert proceed.wait(DB_PROGRESS_TIMEOUT)
monkeypatch.setattr(file_reference, "_validator", pause_reference)
@sa.event.listens_for(sessions.kw["bind"], "after_cursor_execute")
def pause_claim(_conn, _cursor, statement, _parameters, _context, _many):
if (
first == "claim"
and statement.startswith("UPDATE uploaded_files SET storage_status")
) or (
first == "reference"
and mode == "target"
and statement.startswith("INSERT INTO kb_ingest_targets")
):
entered.set()
assert proceed.wait(DB_PROGRESS_TIMEOUT)
def establish():
if mode == "target":
# Admission uses the caller's independent connection and checks the
# same state as document publication, without the global validator.
admit(sessions, job_id=job_id)
else:
write(store, mode)
actions = {"reference": establish, "claim": lambda: claim(sessions)}
def winning():
return actions[first]()
with ThreadPoolExecutor(max_workers=2) as executor:
winner = executor.submit(winning)
try:
assert entered.wait(DB_PROGRESS_TIMEOUT)
loser = executor.submit(
actions["claim" if first == "reference" else "reference"]
)
assert blocked.wait(DB_PROGRESS_TIMEOUT), (
"Contender did not actually wait on the winner's lock"
)
finally:
proceed.set()
result = winner.result(timeout=8)
if first == "claim":
assert result.deleted == 1
with pytest.raises(FileReferenceConflict):
loser.result(timeout=8)
assert store.list_document_records_by_file_ids(["source"]) == []
else:
assert loser.result(timeout=8).deleted == 0
assert path.exists()
sa.event.remove(sessions.kw["bind"], "after_cursor_execute", pause_claim)
def test_reference_write_failure_releases_lock_and_does_not_pin(boundary, monkeypatch):
sessions, store, _ = boundary
original = store._get_connection
def unavailable():
raise OSError("publication failed")
monkeypatch.setattr(store, "_get_connection", unavailable)
with pytest.raises(OSError):
store.upsert_documents([document()])
monkeypatch.setattr(store, "_get_connection", original)
assert claim(sessions).deleted == 1
def test_claim_commit_failure_rolls_back_fence_and_keeps_reference_possible(
boundary, monkeypatch
):
from xagent.web.services.orphan_upload_gc import _claim_orphan, _orphan_candidates
sessions, store, _ = boundary
with sessions() as db:
candidate = _orphan_candidates(
db, cutoff=datetime.now(UTC), limit=1, after=None
)[0]
def fail():
raise OSError("commit failed")
monkeypatch.setattr(db, "commit", fail)
with pytest.raises(OSError):
_claim_orphan(db, candidate)
db.rollback()
store.upsert_documents([document()])
with sessions() as db:
assert db.query(UploadedFile).one().storage_status == "available"
assert db.query(UploadedFileCleanupFence).count() == 0
def test_async_cancellation_waits_for_native_publication(boundary, monkeypatch):
sessions, store, _ = boundary
original = store._get_async_connection
async def scenario():
started, proceed = asyncio.Event(), asyncio.Event()
class Connection:
async def open_table(self, name):
table = await (await original()).open_table(name)
class Table:
def merge_insert(self, keys):
builder = table.merge_insert(keys)
class Builder:
def when_matched_update_all(self):
builder.when_matched_update_all()
return self
def when_not_matched_insert_all(self):
builder.when_not_matched_insert_all()
return self
async def execute(self, rows):
started.set()
await proceed.wait()
await builder.execute(rows)
return Builder()
def close(self):
table.close()
return Table()
async def connection():
return Connection()
monkeypatch.setattr(store, "_get_async_connection", connection)
publication = asyncio.create_task(store.upsert_documents_async([document()]))
await asyncio.wait_for(started.wait(), DB_PROGRESS_TIMEOUT)
publication.cancel()
await asyncio.sleep(0)
cleanup = asyncio.create_task(asyncio.to_thread(claim, sessions))
proceed.set()
with pytest.raises(asyncio.CancelledError):
await publication
assert (await cleanup).deleted == 0
assert len(store.list_document_records_by_file_ids(["source"])) == 1
asyncio.run(scenario())
@pytest.mark.parametrize("stage", ["before", "after", "claim_before_commit"])
def test_process_exit_releases_lock_without_losing_actual_reference(
boundary, stage, monkeypatch
):
import os
import subprocess
import sys
from sqlalchemy.engine import make_url
sessions, store, _ = boundary
engine = sessions.kw["bind"]
if engine.dialect.name == "postgresql":
with engine.connect() as conn:
name = conn.execute(sa.text("SELECT current_database()")).scalar()
url = make_url(os.environ["XAGENT_TEST_POSTGRES_URL"]).set(database=name)
else:
url = engine.url
monkeypatch.setenv(
"REFERENCE_TEST_DATABASE", url.render_as_string(hide_password=False)
)
script = """
import os, sys
from datetime import UTC, datetime
from xagent.web.models.database import configure_db
from xagent.core.tools.core.RAG_tools.storage import file_reference
from xagent.core.tools.core.RAG_tools.storage.lancedb_stores import LanceDBVectorIndexStore
configure_db(os.environ["REFERENCE_TEST_DATABASE"])
if sys.argv[1] == "claim_before_commit":
from xagent.web.models.database import get_session_local
from xagent.web.services.orphan_upload_gc import _claim_orphan, _orphan_candidates
with get_session_local()() as db:
candidate = _orphan_candidates(db, cutoff=datetime.now(UTC), limit=1, after=None)[0]
db.commit = lambda: os._exit(37)
_claim_orphan(db, candidate)
store = LanceDBVectorIndexStore()
if sys.argv[1] == "before":
original = file_reference._validator
def exit_before(ids):
original(ids)
os._exit(37)
file_reference._validator = exit_before
else:
store.invalidate_table_cache = lambda *_args: os._exit(37)
store.upsert_documents([{
"collection": "kb", "doc_id": "doc", "file_id": "source",
"source_path": "/source", "file_type": "txt", "content_hash": "hash",
"uploaded_at": datetime.now(UTC), "title": None, "language": None, "user_id": 1,
}])
"""
process = subprocess.run(
[sys.executable, "-c", script, stage], timeout=45, capture_output=True
)
assert process.returncode == 37, process.stderr.decode()
if stage == "claim_before_commit":
store.upsert_documents([document()])
with sessions() as db:
assert db.query(UploadedFileCleanupFence).count() == 0
assert claim(sessions).deleted == (1 if stage == "before" else 0)
def test_sql_connections_are_returned_before_lancedb_writes(boundary, monkeypatch):
sessions, store, _ = boundary
active = set()
engine = sessions.kw["bind"]
def checkout(connection, _record, _proxy):
active.add(id(connection))
def returned(connection, _record):
active.discard(id(connection))
sa.event.listen(engine, "checkout", checkout)
sa.event.listen(engine, "checkin", returned) # codespell:ignore checkin
original = store._get_connection
def connection():
assert not active, "SQL connection held during LanceDB publication"
return original()
monkeypatch.setattr(store, "_get_connection", connection)
try:
write(store, "sync")
write(store, "async")
write(store, "restore")
write(store, "snapshot")
finally:
sa.event.remove(engine, "checkout", checkout)
sa.event.remove(engine, "checkin", returned) # codespell:ignore checkin
def test_released_reference_preserves_detachment_window(boundary):
sessions, store, path = boundary
detached = datetime.now(UTC) - timedelta(hours=1)
with sessions.begin() as db:
row = db.query(UploadedFile).one()
row.detached_reason = "task_deleted"
row.detached_at = detached
store.upsert_documents([document()])
store.delete_document_record(
collection_name="kb", doc_id="doc", user_id=1, is_admin=False
)
assert claim(sessions).scanned == 0
with sessions() as db:
row = db.query(UploadedFile).one()
assert row.detached_at.replace(tzinfo=UTC) == detached
assert path.exists()
def test_registered_upload_compensation_also_protects_documents_and_targets(boundary):
from xagent.web.services.uploaded_file_store import (
RegisteredUploadCompensationClaim,
compensate_registered_uploads_sync,
)
sessions, store, path = boundary
store.upsert_documents([document(owner=2)])
compensation = RegisteredUploadCompensationClaim(
user_id=1,
file_id="source",
expected_task_id=None,
expected_storage_key="users/1/uploads/source/source.txt",
)
compensate_registered_uploads_sync([compensation])
store.delete_document_record(
collection_name="kb", doc_id="doc", user_id=2, is_admin=False
)
admit(sessions)
compensate_registered_uploads_sync([compensation])
with sessions() as db:
assert db.query(UploadedFile).one().storage_status == "available"
assert path.exists()
def test_schema_upgrade_preserves_data_and_matches_fresh_install(boundary):
from pathlib import Path
from alembic.migration import MigrationContext
from alembic.operations import Operations
from tests.shared.postgres_disposable import load_migration_module
sessions, _, _ = boundary
engine = sessions.kw["bind"]
migration = load_migration_module(
Path("src/xagent/migrations/versions/20261005_uploaded_file_cleanup_fences.py")
)
fresh = sa.inspect(engine).get_columns("uploaded_file_cleanup_fences")
UploadedFileCleanupFence.__table__.drop(engine)
for _ in range(2):
with engine.begin() as conn:
context = MigrationContext.configure(conn)
with Operations.context(context):
migration.upgrade()
upgraded = sa.inspect(engine).get_columns("uploaded_file_cleanup_fences")
assert [(c["name"], str(c["type"]), c["nullable"]) for c in upgraded] == [
(c["name"], str(c["type"]), c["nullable"]) for c in fresh
]
with sessions() as db:
assert db.query(UploadedFile).one().file_id == "source"
assert claim(sessions).deleted == 1
with engine.begin() as conn, Operations.context(MigrationContext.configure(conn)):
migration.upgrade()
with pytest.raises(RuntimeError, match="Retain cleanup fences"):
migration.downgrade()
with sessions() as db:
assert db.query(UploadedFileCleanupFence).one().file_id == "source"
def test_schema_failure_retry_leaves_other_tables_intact(boundary):
from pathlib import Path
from alembic.migration import MigrationContext
from alembic.operations import Operations
from tests.shared.postgres_disposable import load_migration_module
sessions, _, _ = boundary
engine = sessions.kw["bind"]
migration = load_migration_module(
Path("src/xagent/migrations/versions/20261005_uploaded_file_cleanup_fences.py")
)
UploadedFileCleanupFence.__table__.drop(engine)
with pytest.raises(OSError), engine.begin() as conn:
if engine.dialect.name == "sqlite":
conn.exec_driver_sql("BEGIN IMMEDIATE")
with Operations.context(MigrationContext.configure(conn)):
migration.upgrade()
raise OSError("interrupted migration")
assert not sa.inspect(engine).has_table("uploaded_file_cleanup_fences")
with engine.begin() as conn, Operations.context(MigrationContext.configure(conn)):
migration.upgrade()
with sessions() as db:
assert db.query(UploadedFile).one().file_id == "source"
@pytest.mark.parametrize("failure", [False, True])
@pytest.mark.parametrize("release_failure", [False, True])
def test_real_staged_ingest_releases_final_target_and_preserves_actual_documents(
boundary,
tmp_path,
monkeypatch,
failure,
release_failure,
):
from xagent.core.file_storage.factory import get_unscoped_file_storage
from xagent.core.model.model import EmbeddingModelConfig
from xagent.core.storage.manager import initialize_storage_manager
from xagent.core.tools.core.RAG_tools.core.schemas import IngestionConfig
from xagent.core.tools.core.RAG_tools.pipelines import document_ingestion
from xagent.web.jobs.exceptions import BackgroundJobHandlerError
from xagent.web.jobs.kb_tasks import handle_kb_ingest_document
from xagent.web.models import database
from xagent.web.services import kb_ingest_targets
from xagent.web.services.background_jobs import create_background_job
sessions, store, path = boundary
foreign_engine = sa.create_engine(f"sqlite:///{tmp_path / 'foreign.db'}")
Base.metadata.create_all(foreign_engine)
foreign_sessions = sessionmaker(bind=foreign_engine)
with foreign_sessions.begin() as foreign:
foreign.add(User(id=1, username="foreign", password_hash="unused"))
foreign.flush()
foreign.add(
UploadedFile(
user_id=1,
file_id="source",
filename="foreign.txt",
storage_path="/foreign",
storage_status="compensating",
)
)
release = kb_ingest_targets.release_kb_ingest_target_generation
def unavailable(*_args, **_kwargs):
raise RuntimeError("target release unavailable")
if release_failure:
monkeypatch.setattr(
kb_ingest_targets, "release_kb_ingest_target_generation", unavailable
)
monkeypatch.setenv("XAGENT_FILE_STORAGE_URI", (tmp_path / "durable").as_uri())
get_unscoped_file_storage.cache_clear()
initialize_storage_manager(str(tmp_path / "workspace"), str(tmp_path / "uploads"))
class Embeddings:
abilities = ["embedding"]
def get_dimension(self):
return 2
def encode(self, text, **_kwargs):
if failure:
raise RuntimeError("embedding provider failed")
if isinstance(text, str):
return [float(len(text)), 0.0]
return [[float(len(item)), 0.0] for item in text]
config = EmbeddingModelConfig(
id="test", model_name="test", model_provider="test", dimension=2
)
monkeypatch.setattr(
document_ingestion,
"_resolve_embedding_adapter",
lambda _cfg: (config, Embeddings()),
)
monkeypatch.setattr(
"xagent.core.tools.core.RAG_tools.management.collection_manager.resolve_embedding_adapter",
lambda *_args, **_kwargs: (config, Embeddings()),
)
staged = tmp_path / "stage.txt"
staged.write_text("The real staged ingestion source.")
with sessions() as db:
job = create_background_job(
db,
user_id=1,
job_type="kb.ingest.document",
payload={
"collection": "kb",
"source_path": str(staged),
"target_path": str(path),
"file_id": "source",
"generation_id": "one",
"user_id": 1,
"is_admin": False,
"filename": path.name,
"mime_type": "text/plain",
"file_size": staged.stat().st_size,
"ingestion_config": IngestionConfig(
embedding_model_id="test"
).model_dump(mode="json"),
"collection_existed_before": True,
"document_existed_before": False,
},
)
job_id = str(job.id)
admit_kb_ingest_target(
db,
user_id=1,
collection="kb",
target_path=str(path),
file_id="source",
generation_id="one",
job_id=job_id,
file_sha256="hash",
)
job = db.get(type(job), job_id)
job.attempts = job.max_attempts
db.commit()
with monkeypatch.context() as global_override:
global_override.setattr(database, "_SessionLocal", foreign_sessions)
if failure:
with pytest.raises(
BackgroundJobHandlerError, match="embedding provider failed"
):
handle_kb_ingest_document(db, job)
else:
assert handle_kb_ingest_document(db, job)["status"] == "success"
assert (db.query(KBIngestTarget).one().deleted_at is None) is release_failure
assert bool(store.list_document_records_by_file_ids(["source"])) is not failure
if release_failure:
assert claim(sessions).deleted == 0
with sessions() as db:
assert release(
db,
user_id=1,
collection="kb",
target_path=str(path),
generation_id="one",
)
assert claim(sessions).deleted == (1 if failure else 0)
foreign_engine.dispose()
get_unscoped_file_storage.cache_clear()
@pytest.fixture
def submit_ingest(boundary, tmp_path, monkeypatch):
from fastapi import FastAPI
from fastapi.testclient import TestClient
from xagent.web.api import kb
from xagent.web.auth_dependencies import get_current_user
from xagent.web.models.database import get_db
sessions, _store, path = boundary
app = FastAPI()
app.include_router(kb.kb_router)
def db_dependency():
with sessions() as db:
yield db
app.dependency_overrides[get_db] = db_dependency
app.dependency_overrides[get_current_user] = lambda: User(
id=1, username="owner", is_admin=False
)
async def available(*_args, **_kwargs):
return None
async def enqueue(_db, job):
return job
monkeypatch.setattr(kb, "_ensure_collection_access", available)
monkeypatch.setattr(kb, "_ensure_background_job_queue_available_async", available)
monkeypatch.setattr(kb, "_enqueue_background_job_or_503_async", enqueue)
monkeypatch.setattr(kb, "get_upload_path", lambda *_args, **_kwargs: str(path))
monkeypatch.setattr(
kb,
"_build_background_ingest_staging_path",
lambda **_kwargs: tmp_path / "staged.txt",
)
client = TestClient(app, raise_server_exceptions=False)
def submit():
return client.post(
"/api/kb/ingest/jobs",
files={"file": ("source.txt", b"fresh upload", "text/plain")},
data={"collection": "kb"},
)
return submit
@pytest.mark.parametrize("retired", [False, True])
def test_real_upload_admission_preserves_unretired_id_and_renews_retired_id(
boundary, submit_ingest, retired
):
from xagent.web.api.kb import _background_ingest_file_id
from xagent.web.models.background_job import BackgroundJob
sessions, store, path = boundary
identity = _background_ingest_file_id(user_id=1, storage_path=path)
with sessions.begin() as db:
db.query(UploadedFile).update({UploadedFile.file_id: identity})
if not retired:
db.query(UploadedFile).delete()
if retired:
assert claim(sessions).deleted == 1
response = submit_ingest()
assert response.status_code == 202, response.text
with sessions() as db:
job = db.get(BackgroundJob, response.json()["id"])
chosen = job.payload["file_id"]
assert (chosen != identity) is retired
assert db.query(KBIngestTarget).one().file_id == chosen
store.upsert_documents([document(file_id=chosen)])
assert store.list_document_records_by_file_ids([chosen])
def test_admission_lock_failure_marks_job_failed_and_allows_fresh_retry(
boundary, submit_ingest, monkeypatch, tmp_path
):
from filelock import Timeout
from xagent.web.models.background_job import BackgroundJob
from xagent.web.services import kb_reference_protection
sessions, _store, _path = boundary
original = kb_reference_protection.file_reference_lock
def unavailable(_ids):
raise Timeout(str(tmp_path / "private-reference.lock"))
monkeypatch.setattr(kb_reference_protection, "file_reference_lock", unavailable)
response = submit_ingest()
assert response.status_code == 503
assert response.json() == {"detail": "KB source is busy; retry the upload"}
assert response.headers["retry-after"] == "15"
assert str(tmp_path) not in response.text
with sessions() as db:
failed = db.query(BackgroundJob).one()
assert failed.status == "failed"
assert failed.error_message == "KB source is busy; retry the upload"
assert db.query(KBIngestTarget).count() == 0
assert not (tmp_path / "staged.txt").exists()
monkeypatch.setattr(kb_reference_protection, "file_reference_lock", original)
response = submit_ingest()
assert response.status_code == 202, response.text
assert response.json()["id"] != failed.id
def test_retryable_worker_failure_keeps_generation_and_staged_source(
boundary, tmp_path, monkeypatch
):
from xagent.core.tools.core.RAG_tools.core.schemas import IngestionConfig
from xagent.web.jobs import kb_tasks
from xagent.web.jobs.exceptions import BackgroundJobHandlerError
from xagent.web.services.background_jobs import create_background_job
sessions, store, path = boundary
staged = tmp_path / "retry.txt"
staged.write_text("retryable source")
error = BackgroundJobHandlerError("temporary ingest failure", retryable=True)
def temporarily_unavailable(**_kwargs):
raise error
monkeypatch.setattr(kb_tasks, "run_document_ingestion", temporarily_unavailable)
with sessions() as db:
job = create_background_job(
db,
user_id=1,
job_type="kb.ingest.document",
payload={
"user_id": 2,
"collection": "kb",
"file_id": "source",
"target_path": "canonical/source",
"source_path": str(staged),
"generation_id": "one",
"collection_existed_before": True,
"ingestion_config": IngestionConfig().model_dump(mode="json"),
},
)
job_id = str(job.id)
admit(sessions, job_id=job_id)
with sessions() as db:
job = db.get(type(job), job_id)
job.attempts = 1
db.commit()
with pytest.raises(BackgroundJobHandlerError) as caught:
kb_tasks.handle_kb_ingest_document(db, job)
assert caught.value is error
assert db.query(KBIngestTarget).one().deleted_at is None
assert staged.read_text() == "retryable source"
assert not store.list_document_records_by_file_ids(["source"])
assert claim(sessions).deleted == 0
assert path.exists()
def test_compensating_canonical_upload_rejects_admission_with_safe_conflict(
boundary, submit_ingest, tmp_path
):
from xagent.web.models.background_job import BackgroundJob
sessions, store, _path = boundary
with sessions.begin() as db:
db.query(UploadedFile).update({UploadedFile.storage_status: "compensating"})
response = submit_ingest()
assert response.status_code == 409
message = "KB source is unavailable or the ingest was superseded"
assert response.json() == {"detail": message}
with sessions() as db:
failed = db.query(BackgroundJob).one()
assert failed.status == "failed"
assert failed.error_message == message
assert db.query(KBIngestTarget).count() == 0
assert db.query(UploadedFile).one().file_id == "source"
assert not store.list_document_records_by_file_ids(["source"])
assert not (tmp_path / "staged.txt").exists()
def test_upload_admission_wait_keeps_request_event_loop_responsive(
boundary, submit_ingest, monkeypatch
):
from contextlib import contextmanager
from xagent.web.api import kb
from xagent.web.services import kb_reference_protection
tick = Event()
request_loops = []
original = kb_reference_protection.file_reference_lock
async def access(*_args, **_kwargs):
request_loops.append(asyncio.get_running_loop())
@contextmanager
def delayed_lock(ids):
request_loops[0].call_soon_threadsafe(tick.set)
assert tick.wait(3), "Admission blocked the request event loop"
with original(ids):
yield
monkeypatch.setattr(kb, "_ensure_collection_access", access)
monkeypatch.setattr(kb_reference_protection, "file_reference_lock", delayed_lock)
response = submit_ingest()
assert response.status_code == 202, response.text
assert tick.is_set()