Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.
Python agentagentic-aiagentic-frameworkagentic-workflowaiautomationclaudedeepseekgenaigptllmlow-codemcpnextjsno-codeopenaipythonskillsworkflow
import uuid
from datetime import UTC, datetime, timedelta
from unittest.mock import patch
import pytest
from sqlalchemy.orm import Session
import tasks.batch_clean_document_task as task_module
from extensions.storage.storage_type import StorageType
from models.dataset import Dataset, DocumentSegment, SegmentAttachmentBinding
from models.enums import CreatorUserRole, DataSourceType
from models.knowledge_fs import KnowledgeFSUpgradeFileLease, KnowledgeFSUpgradeJob
from models.model import UploadFile
from tasks.batch_clean_document_task import batch_clean_document_task
@pytest.fixture
def cleanup_rows(sqlite_session: Session, monkeypatch: pytest.MonkeyPatch) -> tuple[str, str, str]:
tenant_id = str(uuid.uuid4())
dataset_id = str(uuid.uuid4())
document_id = str(uuid.uuid4())
created_by = str(uuid.uuid4())
dataset = Dataset(
id=dataset_id,
tenant_id=tenant_id,
name="Batch cleanup dataset",
data_source_type=DataSourceType.UPLOAD_FILE,
created_by=created_by,
)
segment = DocumentSegment(
tenant_id=tenant_id,
dataset_id=dataset_id,
document_id=document_id,
position=1,
content="content",
word_count=1,
tokens=1,
created_by=created_by,
index_node_id="node-1",
)
sqlite_session.add_all([dataset, segment])
sqlite_session.commit()
engine = sqlite_session.get_bind()
monkeypatch.setattr(
task_module.session_factory,
"create_session",
lambda: Session(engine, expire_on_commit=False),
)
return dataset_id, document_id, tenant_id
def test_successful_vector_cleanup_schedules_billing_refresh(cleanup_rows: tuple[str, str, str]) -> None:
dataset_id, document_id, tenant_id = cleanup_rows
with (
patch("tasks.batch_clean_document_task.get_image_upload_file_ids", return_value=[]),
patch("tasks.batch_clean_document_task.clean_document_indexes", return_value=tenant_id) as index_cleanup,
patch("tasks.batch_clean_document_task.schedule_billing_vector_space_refresh") as schedule_refresh,
):
batch_clean_document_task(
document_ids=[document_id],
dataset_id=dataset_id,
doc_form="paragraph",
file_ids=[],
)
index_cleanup.assert_called_once()
schedule_refresh.assert_called_once_with(tenant_id)
def test_failed_vector_cleanup_does_not_schedule_billing_refresh(cleanup_rows: tuple[str, str, str]) -> None:
dataset_id, document_id, _tenant_id = cleanup_rows
with (
patch("tasks.batch_clean_document_task.get_image_upload_file_ids", return_value=[]),
patch("tasks.batch_clean_document_task.clean_document_indexes") as index_cleanup,
patch("tasks.batch_clean_document_task.schedule_billing_vector_space_refresh") as schedule_refresh,
):
index_cleanup.side_effect = RuntimeError("vector cleanup failed")
batch_clean_document_task(
document_ids=[document_id],
dataset_id=dataset_id,
doc_form="paragraph",
file_ids=[],
)
schedule_refresh.assert_not_called()
def test_batch_cleanup_keeps_only_the_leased_legacy_source_file(
cleanup_rows: tuple[str, str, str], sqlite_session: Session
) -> None:
dataset_id, document_id, tenant_id = cleanup_rows
account_id = str(uuid.uuid4())
now = datetime.now(UTC).replace(tzinfo=None)
leased_file = UploadFile(
tenant_id=tenant_id,
storage_type=StorageType.LOCAL,
key=f"upload_files/{tenant_id}/leased.txt",
name="leased.txt",
size=10,
extension="txt",
mime_type="text/plain",
created_by_role=CreatorUserRole.ACCOUNT,
created_by=account_id,
created_at=now,
used=False,
)
deletable_file = UploadFile(
tenant_id=tenant_id,
storage_type=StorageType.LOCAL,
key=f"upload_files/{tenant_id}/deletable.txt",
name="deletable.txt",
size=10,
extension="txt",
mime_type="text/plain",
created_by_role=CreatorUserRole.ACCOUNT,
created_by=account_id,
created_at=now,
used=False,
)
job = KnowledgeFSUpgradeJob(
tenant_id=tenant_id,
old_dataset_id=dataset_id,
requested_by_account_id=account_id,
owner_account_id=account_id,
idempotency_key="batch-cleanup-lease-test",
snapshot_at=now,
config_snapshot={},
permission_snapshot={},
app_binding_snapshot=[],
tag_ids_snapshot=[],
)
sqlite_session.add_all([leased_file, deletable_file, job])
sqlite_session.flush()
sqlite_session.add(
KnowledgeFSUpgradeFileLease(
job_id=job.id,
old_upload_file_id=leased_file.id,
expires_at=now + timedelta(hours=1),
)
)
sqlite_session.commit()
leased_file_id = leased_file.id
deletable_file_id = deletable_file.id
deletable_file_key = deletable_file.key
with (
patch("tasks.batch_clean_document_task.get_image_upload_file_ids", return_value=[]),
patch("tasks.batch_clean_document_task.clean_document_indexes", return_value=tenant_id),
patch("tasks.batch_clean_document_task.schedule_billing_vector_space_refresh"),
patch("tasks.batch_clean_document_task.storage") as storage,
):
batch_clean_document_task(
document_ids=[document_id],
dataset_id=dataset_id,
doc_form="paragraph",
file_ids=[leased_file_id, deletable_file_id],
)
sqlite_session.expire_all()
assert sqlite_session.get(UploadFile, leased_file_id) is not None
assert sqlite_session.get(UploadFile, deletable_file_id) is None
storage.delete.assert_called_once_with(deletable_file_key)
def test_cleans_segment_attachment_bindings_and_files(
cleanup_rows: tuple[str, str, str], sqlite_session: Session
) -> None:
dataset_id, document_id, tenant_id = cleanup_rows
segment = sqlite_session.query(DocumentSegment).filter_by(document_id=document_id).one()
attachment = UploadFile(
tenant_id=tenant_id,
storage_type=StorageType.LOCAL,
key="attachments/image.png",
name="image.png",
size=10,
extension="png",
mime_type="image/png",
created_by_role=CreatorUserRole.ACCOUNT,
created_by=segment.created_by,
created_at=datetime.now(UTC),
used=True,
)
binding = SegmentAttachmentBinding(
tenant_id=tenant_id,
dataset_id=dataset_id,
document_id=document_id,
segment_id=segment.id,
attachment_id=attachment.id,
)
sqlite_session.add_all([attachment, binding])
sqlite_session.commit()
attachment_id = attachment.id
attachment_key = attachment.key
binding_id = binding.id
with (
patch("tasks.batch_clean_document_task.get_image_upload_file_ids", return_value=[]),
patch("tasks.batch_clean_document_task.clean_document_indexes"),
patch("tasks.batch_clean_document_task.schedule_billing_vector_space_refresh"),
patch("tasks.batch_clean_document_task.storage.delete") as storage_delete,
):
batch_clean_document_task(
document_ids=[document_id],
dataset_id=dataset_id,
doc_form="paragraph",
file_ids=[],
)
sqlite_session.expire_all()
assert sqlite_session.get(SegmentAttachmentBinding, binding_id) is None
assert sqlite_session.get(UploadFile, attachment_id) is None
storage_delete.assert_called_once_with(attachment_key)