Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

23.0 KB
Raw Blame
from __future__ import annotations

import hashlib
import json
from datetime import UTC, datetime, timedelta
from typing import cast
from unittest.mock import patch

import pytest
import sqlalchemy as sa
from pydantic import ValidationError
from sqlalchemy import event, select
from sqlalchemy.engine import Engine
from sqlalchemy.orm import Session, sessionmaker

from libs.datetime_utils import ensure_naive_utc
from models.base import TypeBase
from models.knowledge_fs_cleanup import (
    KnowledgeFSCleanupAuthorization,
    KnowledgeFSCleanupAuthorizationStatus,
    KnowledgeFSCleanupTarget,
)
from models.knowledge_fs_cutover import (
    KnowledgeFSMigrationIssue,
    KnowledgeFSShadowAuthorizationDiff,
    KnowledgeFSWorkspaceCutoverLedger,
    KnowledgeFSWorkspaceCutoverPhase,
)
from services.knowledge_fs.cleanup import (
    CleanupApprovalInput,
    CleanupCompletionEvidenceInput,
    CleanupReadinessEvidenceInput,
    CleanupStartInput,
    KnowledgeFSCleanupConflictError,
    KnowledgeFSCleanupGateBlockedError,
    KnowledgeFSCleanupService,
    cleanup_workspace_cohort_digest,
)
from services.knowledge_fs.cutover import KnowledgeFSCutoverGateBlockedError, KnowledgeFSWorkspaceCutoverService

_REQUESTER_ID = "00000000-0000-0000-0000-000000000001"
_APPROVER_ID = "00000000-0000-0000-0000-000000000002"
_STARTER_ID = "00000000-0000-0000-0000-000000000003"
_TENANT_ID = "00000000-0000-0000-0000-000000000004"
_REQUEST_ID = "00000000-0000-0000-0000-000000000005"
_CONTROL_SPACE_ID = "00000000-0000-0000-0000-000000000006"
_SECOND_TENANT_ID = "00000000-0000-0000-0000-000000000007"
_PLAN_DIGEST = f"sha256:{'a' * 64}"
_NOW = datetime(2026, 7, 10, 12, 0, tzinfo=UTC)
_WATERMARK = {
    "membership_epoch": 2,
    "space_acl_epoch": 2,
    "external_access_epoch": 2,
    "content_policy_revision": 2,
}


def _sha256(payload: object) -> str:
    canonical = json.dumps(payload, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode()
    return f"sha256:{hashlib.sha256(canonical).hexdigest()}"


_SOURCE_DIGEST = _sha256(_WATERMARK)
_FREEZE_REVISION = 7
_ACTIVATION_DIGEST = _sha256(
    {
        "final_revision_watermark": _WATERMARK,
        "final_task_watermark": 12,
    }
)
_ACTIVATION_REVISION = 8
_SMOKE = {
    "schema_version": "knowledge-fs-p8-cutover-smoke/v1",
    "tenant_id": _TENANT_ID,
    "environment": "production",
    "operator": "knowledge-fs-cutover",
    "operator_account_id": _REQUESTER_ID,
    "observed_at": "2026-07-01T10:05:00+00:00",
    "checks": {
        "authorization": True,
        "list_spaces": True,
        "create_space": True,
        "query": True,
        "upload": True,
        "stream": True,
        "deletion": True,
    },
    "evidence_references": {
        "authorization": "evidence://smoke/authorization",
        "list_spaces": "evidence://smoke/list-spaces",
        "create_space": "evidence://smoke/create-space",
        "query": "evidence://smoke/query",
        "upload": "evidence://smoke/upload",
        "stream": "evidence://smoke/stream",
        "deletion": "evidence://smoke/deletion",
    },
}
_MODELS = (
    KnowledgeFSWorkspaceCutoverLedger,
    KnowledgeFSMigrationIssue,
    KnowledgeFSShadowAuthorizationDiff,
    KnowledgeFSCleanupAuthorization,
)


def _ready_ledger(tenant_id: str, *, cas_version: int = 9) -> KnowledgeFSWorkspaceCutoverLedger:
    freeze_id = _sha256(
        {
            "freezeRevision": _FREEZE_REVISION,
            "namespaceId": tenant_id,
            "sourceRevisionDigest": _SOURCE_DIGEST,
            "sourceTaskWatermark": 10,
        }
    )
    activation_id = _sha256(
        {
            "activationRevision": _ACTIVATION_REVISION,
            "namespaceId": tenant_id,
            "sourceRevisionDigest": _ACTIVATION_DIGEST,
        }
    )
    return KnowledgeFSWorkspaceCutoverLedger(
        tenant_id=tenant_id,
        source_revision_watermark=_WATERMARK,
        applied_revision_watermark=_WATERMARK,
        phase=KnowledgeFSWorkspaceCutoverPhase.READY_FOR_CLEANUP,
        final_revision_watermark=_WATERMARK,
        source_task_watermark=10,
        final_task_watermark=12,
        applied_task_watermark=12,
        remote_freeze_id=freeze_id,
        remote_freeze_revision=_FREEZE_REVISION,
        remote_freeze_digest=_SOURCE_DIGEST,
        remote_freeze_task_watermark=10,
        remote_freeze_control_space_id=_CONTROL_SPACE_ID,
        remote_freeze_frozen_at=datetime(2026, 7, 1, 9, 0),
        remote_freeze_updated_at=datetime(2026, 7, 1, 9, 0),
        remote_freeze_acknowledged_at=datetime(2026, 7, 1, 9, 1),
        remote_freeze_applied=True,
        remote_freeze_replayed=False,
        remote_activation_id=activation_id,
        remote_activation_revision=_ACTIVATION_REVISION,
        remote_activation_digest=_ACTIVATION_DIGEST,
        remote_activation_control_space_id=_CONTROL_SPACE_ID,
        remote_activation_activated_at=datetime(2026, 7, 1, 10, 0),
        remote_activation_updated_at=datetime(2026, 7, 1, 10, 0),
        remote_activation_acknowledged_at=datetime(2026, 7, 1, 10, 1),
        remote_activation_applied=True,
        remote_activation_replayed=False,
        freeze_at=datetime(2026, 7, 1, 9, 0),
        cutover_at=datetime(2026, 7, 1, 10, 0),
        rollback_cutoff_at=datetime(2026, 7, 3, 10, 0),
        observation_started_at=datetime(2026, 7, 1, 11, 0),
        observation_window_ends_at=datetime(2026, 7, 4, 11, 0),
        maximum_task_expires_at=datetime(2026, 7, 5, 11, 0),
        observation_completed_at=datetime(2026, 7, 5, 11, 0),
        product_routes_enabled=True,
        capability_v2_enabled=True,
        integrated_mode_enabled=True,
        legacy_acl_read_only=True,
        smoke_results={**_SMOKE, "tenant_id": tenant_id},
        legacy_dependency_report=[],
        legacy_dependency_checked_at=datetime(2026, 7, 5, 11, 0),
        legacy_dependency_ready=True,
        cas_version=cas_version,
    )


class _Clock:
    def __init__(self, value: datetime) -> None:
        self.value = value

    def __call__(self) -> datetime:
        return self.value


@pytest.fixture
def cleanup_context(
    sqlite_engine: Engine,
) -> tuple[KnowledgeFSCleanupService, sessionmaker[Session], _Clock]:
    tables = [model.metadata.tables[model.__tablename__] for model in _MODELS]
    TypeBase.metadata.create_all(sqlite_engine, tables=tables)
    session_maker = sessionmaker(bind=sqlite_engine, expire_on_commit=False)
    clock = _Clock(_NOW)
    with session_maker.begin() as session:
        session.add(_ready_ledger(_TENANT_ID))
    return KnowledgeFSCleanupService(session_maker, clock=clock), session_maker, clock


def _readiness_payload(**updates: object) -> CleanupReadinessEvidenceInput:
    values: dict[str, object] = {
        "schema_version": "knowledge-fs-p9-cleanup/v1",
        "tenant_id": _TENANT_ID,
        "request_id": _REQUEST_ID,
        "expected_cas_version": 9,
        "plan_digest": _PLAN_DIGEST,
        "targets": [target.value for target in KnowledgeFSCleanupTarget],
        "expected_workspace_count": 1,
        "workspace_tenant_ids": [_TENANT_ID],
        "workspace_cohort_digest": cleanup_workspace_cohort_digest([_TENANT_ID]),
        "workspace_inventory_reference": "inventory://production/kfs/workspaces/2026-07-09",
        "evidence_environment": "production",
        "observed_at": datetime(2026, 7, 9, 12, 0, tzinfo=UTC),
        "requested_at": datetime(2026, 7, 9, 13, 0, tzinfo=UTC),
        "legacy_route_zero_window_started_at": datetime(2026, 7, 3, 10, 0, tzinfo=UTC),
        "legacy_route_zero_window_ends_at": datetime(2026, 7, 9, 12, 0, tzinfo=UTC),
        "rollback_window_seconds": 3 * 24 * 60 * 60,
        "legacy_route_calls": 0,
        "legacy_access_route_calls": 0,
        "legacy_member_route_calls": 0,
        "legacy_api_key_route_calls": 0,
        "legacy_route_metric_reference": "metrics://production/kfs-legacy-routes/window-17",
        "maximum_token_expires_at": datetime(2026, 7, 6, 12, 0, tzinfo=UTC),
        "backup_reference": "backup://production/kfs/2026-07-08",
        "backup_verified_at": datetime(2026, 7, 8, 10, 0, tzinfo=UTC),
        "restore_drill_reference": "restore-drill://production/kfs/2026-07-08",
        "restore_drill_verified_at": datetime(2026, 7, 8, 11, 0, tzinfo=UTC),
        "change_window_approval_reference": "change://CAB-2026-0710",
        "requested_by_account_id": _REQUESTER_ID,
    }
    values.update(updates)
    return CleanupReadinessEvidenceInput.model_validate(values)


def _approval_payload(**updates: object) -> CleanupApprovalInput:
    values: dict[str, object] = {
        "schema_version": "knowledge-fs-p9-cleanup-approval/v1",
        "tenant_id": _TENANT_ID,
        "request_id": _REQUEST_ID,
        "expected_cas_version": 10,
        "plan_digest": _PLAN_DIGEST,
        "approved_by_account_id": _APPROVER_ID,
        "approved_at": _NOW - timedelta(hours=1),
        "approval_expires_at": _NOW + timedelta(hours=1),
    }
    values.update(updates)
    return CleanupApprovalInput.model_validate(values)


def _start_payload(**updates: object) -> CleanupStartInput:
    values: dict[str, object] = {
        "schema_version": "knowledge-fs-p9-cleanup-start/v1",
        "tenant_id": _TENANT_ID,
        "request_id": _REQUEST_ID,
        "expected_cas_version": 11,
        "plan_digest": _PLAN_DIGEST,
        "started_by_account_id": _STARTER_ID,
        "started_at": _NOW,
        "confirmation": "START-KNOWLEDGE-FS-IRREVERSIBLE-CLEANUP",
    }
    values.update(updates)
    return CleanupStartInput.model_validate(values)


def _completion_payload(**updates: object) -> CleanupCompletionEvidenceInput:
    values: dict[str, object] = {
        "schema_version": "knowledge-fs-p9-cleanup-completion/v1",
        "tenant_id": _TENANT_ID,
        "request_id": _REQUEST_ID,
        "expected_cas_version": 12,
        "plan_digest": _PLAN_DIGEST,
        "migration_bundle_digest": _PLAN_DIGEST,
        "database_engine": "postgresql",
        "migration_revision": "knowledge-fs-p9-removal/v1",
        "archived_row_counts": {
            "knowledge_space_members": 10,
            "knowledge_space_access_policies": 2,
            "knowledge_space_access_policy_members": 4,
            "knowledge_space_api_access": 2,
            "knowledge_space_api_keys": 3,
            "knowledge_space_permission_snapshots": 12,
        },
        "checks": {
            "legacy_foreign_keys_remaining": 0,
            "legacy_tables_remaining": 0,
            "legacy_routes_registered": 0,
            "legacy_v1_auth_acceptances": 0,
            "raw_proxy_routes_registered": 0,
            "post_cleanup_smoke_passed": True,
            "recovery_material_verified": True,
        },
        "archive_reference": "archive://production/kfs/p9-2026-07-10",
        "catalog_verification_reference": "catalog://production/kfs/p9-2026-07-10",
        "route_metric_reference": "metrics://production/kfs/post-p9",
        "post_cleanup_smoke_reference": "smoke://production/kfs/post-p9",
        "recovery_material_reference": "recovery://production/kfs/p9-2026-07-10",
        "completed_by_account_id": _STARTER_ID,
        "completed_at": _NOW,
    }
    values.update(updates)
    return CleanupCompletionEvidenceInput.model_validate(values)


def _request_and_approve(service: KnowledgeFSCleanupService) -> None:
    service.request_cleanup(_readiness_payload(), apply=True)
    service.approve_cleanup(_approval_payload(), apply=True)


def test_cleanup_workflow_is_dry_run_by_default_and_never_executes_destructive_sql(
    cleanup_context: tuple[KnowledgeFSCleanupService, sessionmaker[Session], _Clock],
) -> None:
    service, session_maker, _ = cleanup_context
    readiness = _readiness_payload()

    dry_run = service.request_cleanup(readiness, apply=False)

    assert dry_run.ready is True
    assert dry_run.status == "dry_run"
    with session_maker() as session:
        assert session.scalar(select(sa.func.count()).select_from(KnowledgeFSCleanupAuthorization)) == 0
        ledger = session.scalar(select(KnowledgeFSWorkspaceCutoverLedger))
    assert ledger is not None
    assert ledger.cas_version == 9

    requested = service.request_cleanup(readiness, apply=True)
    assert requested.status == "requested"
    approval_dry_run = service.approve_cleanup(_approval_payload(), apply=False)
    assert approval_dry_run.approvable is True
    with session_maker() as session:
        authorization = session.scalar(select(KnowledgeFSCleanupAuthorization))
    assert authorization is not None
    assert authorization.status is KnowledgeFSCleanupAuthorizationStatus.REQUESTED

    approved = service.approve_cleanup(_approval_payload(), apply=True)
    assert approved.status == "approved"
    start_dry_run = service.start_cleanup(_start_payload(), apply=False)
    assert start_dry_run.startable is True
    assert start_dry_run.irreversible_cleanup_at is None

    statements: list[str] = []

    def capture_statement(
        _connection: object,
        _cursor: object,
        statement: str,
        _parameters: object,
        _context: object,
        _executemany: bool,
    ) -> None:
        statements.append(statement.strip().lower())

    engine = cast(Engine, session_maker.kw["bind"])
    event.listen(engine, "before_cursor_execute", capture_statement)
    try:
        started = service.start_cleanup(_start_payload(), apply=True)
    finally:
        event.remove(engine, "before_cursor_execute", capture_statement)

    assert started.status == "started"
    assert started.destructive_actions_executed is False
    assert not any(statement.startswith(("drop ", "delete ", "truncate ")) for statement in statements)
    status = service.status(tenant_id=_TENANT_ID, request_id=_REQUEST_ID)
    assert status["status"] == "started"
    assert status["destructive_actions_executed"] is False
    assert status["irreversible_cleanup_at"] is not None


def test_cleanup_readiness_requires_persisted_production_windows_and_zero_legacy_calls(
    cleanup_context: tuple[KnowledgeFSCleanupService, sessionmaker[Session], _Clock],
) -> None:
    service, session_maker, _ = cleanup_context
    with session_maker.begin() as session:
        session.execute(
            sa.update(KnowledgeFSWorkspaceCutoverLedger).values(
                phase=KnowledgeFSWorkspaceCutoverPhase.OBSERVING,
                observation_completed_at=None,
            )
        )
    nonzero_calls = _readiness_payload(legacy_route_calls=4, legacy_access_route_calls=4)
    report = service.request_cleanup(nonzero_calls, apply=False)

    assert report.ready is False
    assert "workspace is not ready_for_cleanup" in report.reasons
    assert "production observation is not persisted as complete" in report.reasons
    assert "legacy route calls are nonzero" in report.reasons
    with pytest.raises(KnowledgeFSCleanupGateBlockedError):
        service.request_cleanup(nonzero_calls, apply=True)


def test_cleanup_schema_rejects_nonproduction_or_incomplete_target_evidence() -> None:
    with pytest.raises(ValidationError, match="production"):
        _readiness_payload(evidence_environment="local")
    with pytest.raises(ValidationError, match="every destructive target"):
        _readiness_payload(targets=[KnowledgeFSCleanupTarget.LEGACY_ACL_SCHEMA.value] * 6)
    with pytest.raises(ValidationError, match="explicit timezone"):
        _readiness_payload(observed_at=datetime(2026, 7, 9, 12, 0))


def test_cleanup_approval_requires_four_eyes_current_cas_and_bounded_expiry(
    cleanup_context: tuple[KnowledgeFSCleanupService, sessionmaker[Session], _Clock],
) -> None:
    service, _, _ = cleanup_context
    service.request_cleanup(_readiness_payload(), apply=True)

    same_actor = service.approve_cleanup(
        _approval_payload(approved_by_account_id=_REQUESTER_ID),
        apply=False,
    )
    assert same_actor.approvable is False
    assert "cleanup requires a distinct approver" in same_actor.reasons
    with pytest.raises(KnowledgeFSCleanupGateBlockedError, match="distinct approver"):
        service.approve_cleanup(
            _approval_payload(approved_by_account_id=_REQUESTER_ID),
            apply=True,
        )
    with pytest.raises(KnowledgeFSCleanupGateBlockedError, match="24 hours"):
        service.approve_cleanup(
            _approval_payload(approval_expires_at=_NOW + timedelta(days=2)),
            apply=True,
        )


def test_irreversible_start_closes_rollback_even_with_backdated_operator_timestamp(
    cleanup_context: tuple[KnowledgeFSCleanupService, sessionmaker[Session], _Clock],
) -> None:
    service, session_maker, clock = cleanup_context
    _request_and_approve(service)
    service.start_cleanup(_start_payload(), apply=True)
    cutover_service = KnowledgeFSWorkspaceCutoverService(session_maker, clock=clock)

    with pytest.raises(KnowledgeFSCutoverGateBlockedError, match="cutoff"):
        cutover_service.rollback(
            tenant_id=_TENANT_ID,
            expected_cas_version=12,
            rolled_back_at=datetime(2026, 7, 2, 12, 0, tzinfo=UTC),
        )


def test_cleanup_start_rejects_expired_approval_and_plan_digest_drift(
    cleanup_context: tuple[KnowledgeFSCleanupService, sessionmaker[Session], _Clock],
) -> None:
    service, _, clock = cleanup_context
    _request_and_approve(service)
    clock.value = _NOW + timedelta(hours=2)

    expired = service.start_cleanup(
        _start_payload(started_at=_NOW + timedelta(hours=2)),
        apply=False,
    )
    assert expired.startable is False
    assert "cleanup approval has expired" in expired.reasons
    with pytest.raises(KnowledgeFSCleanupGateBlockedError, match="expired"):
        service.start_cleanup(
            _start_payload(started_at=_NOW + timedelta(hours=2)),
            apply=True,
        )
    with pytest.raises(KnowledgeFSCleanupGateBlockedError, match="plan digest"):
        service.start_cleanup(
            _start_payload(
                plan_digest=f"sha256:{'b' * 64}",
                started_at=_NOW + timedelta(hours=2),
            ),
            apply=True,
        )


def test_cleanup_start_rolls_back_irreversible_fence_when_audit_cas_loses_race(
    cleanup_context: tuple[KnowledgeFSCleanupService, sessionmaker[Session], _Clock],
) -> None:
    service, session_maker, _ = cleanup_context
    _request_and_approve(service)

    with (
        patch(
            "services.knowledge_fs.cleanup.SQLAlchemyKnowledgeFSCleanupAuthorizationRepository.compare_and_set",
            return_value=False,
        ),
        pytest.raises(KnowledgeFSCleanupConflictError, match="changed during start"),
    ):
        service.start_cleanup(_start_payload(), apply=True)

    with session_maker() as session:
        ledger = session.scalar(select(KnowledgeFSWorkspaceCutoverLedger))
        authorization = session.scalar(select(KnowledgeFSCleanupAuthorization))
    assert ledger is not None
    assert ledger.irreversible_cleanup_at is None
    assert ledger.cas_version == 11
    assert authorization is not None
    assert authorization.status is KnowledgeFSCleanupAuthorizationStatus.APPROVED


def test_cleanup_requires_the_exact_global_workspace_cohort_and_fences_every_ledger(
    cleanup_context: tuple[KnowledgeFSCleanupService, sessionmaker[Session], _Clock],
) -> None:
    service, session_maker, _ = cleanup_context
    with session_maker.begin() as session:
        session.add(_ready_ledger(_SECOND_TENANT_ID))

    incomplete = service.request_cleanup(_readiness_payload(), apply=False)
    assert incomplete.ready is False
    assert "persisted Workspace count does not match the cleanup cohort" in incomplete.reasons
    assert "persisted Workspace cohort does not exactly match the cleanup inventory" in incomplete.reasons

    tenant_ids = [_TENANT_ID, _SECOND_TENANT_ID]
    readiness = _readiness_payload(
        expected_workspace_count=2,
        workspace_tenant_ids=tenant_ids,
        workspace_cohort_digest=cleanup_workspace_cohort_digest(tenant_ids),
    )
    service.request_cleanup(readiness, apply=True)
    service.approve_cleanup(_approval_payload(), apply=True)
    service.start_cleanup(_start_payload(), apply=True)

    with session_maker() as session:
        ledgers = tuple(
            session.scalars(
                select(KnowledgeFSWorkspaceCutoverLedger).order_by(KnowledgeFSWorkspaceCutoverLedger.tenant_id)
            )
        )
    assert len(ledgers) == 2
    assert {candidate.irreversible_cleanup_at for candidate in ledgers} == {ensure_naive_utc(_NOW)}
    assert {candidate.cas_version for candidate in ledgers} == {10, 12}


def test_cleanup_readiness_requires_exact_remote_freeze_and_activation_evidence(
    cleanup_context: tuple[KnowledgeFSCleanupService, sessionmaker[Session], _Clock],
) -> None:
    service, session_maker, _ = cleanup_context
    with session_maker.begin() as session:
        session.execute(
            sa.update(KnowledgeFSWorkspaceCutoverLedger).values(
                remote_activation_id=None,
                remote_activation_revision=None,
                remote_activation_digest=None,
                remote_activation_control_space_id=None,
                remote_activation_activated_at=None,
                remote_activation_updated_at=None,
                remote_activation_acknowledged_at=None,
                remote_activation_applied=None,
                remote_activation_replayed=None,
            )
        )

    report = service.request_cleanup(_readiness_payload(), apply=False)

    assert report.ready is False
    assert "remote Workspace activation evidence is incomplete or inconsistent" in report.reasons


def test_cleanup_completion_is_strict_replay_safe_and_marks_external_actions_executed(
    cleanup_context: tuple[KnowledgeFSCleanupService, sessionmaker[Session], _Clock],
) -> None:
    service, session_maker, _ = cleanup_context
    _request_and_approve(service)
    service.start_cleanup(_start_payload(), apply=True)

    dry_run = service.complete_cleanup(_completion_payload(), apply=False)
    assert dry_run.completable is True
    assert dry_run.destructive_actions_executed is False

    completed = service.complete_cleanup(_completion_payload(), apply=True)
    assert completed.status == "completed"
    assert completed.destructive_actions_executed is True
    replayed = service.complete_cleanup(_completion_payload(), apply=True)
    assert replayed.replayed is True
    assert replayed.destructive_actions_executed is True

    status = service.status(tenant_id=_TENANT_ID, request_id=_REQUEST_ID)
    assert status["status"] == "completed"
    assert status["destructive_actions_executed"] is True
    assert status["completion_evidence"] == _completion_payload().model_dump(mode="json")
    with session_maker() as session:
        authorization = session.scalar(select(KnowledgeFSCleanupAuthorization))
    assert authorization is not None
    assert authorization.status is KnowledgeFSCleanupAuthorizationStatus.COMPLETED
    assert authorization.completed_ledger_cas_version == 13


def test_cleanup_completion_schema_rejects_unapproved_bundle_or_incomplete_checks() -> None:
    with pytest.raises(ValidationError, match="must match the approved plan"):
        _completion_payload(migration_bundle_digest=f"sha256:{'b' * 64}")
    checks = _completion_payload().checks.model_dump()
    checks["legacy_tables_remaining"] = 1
    with pytest.raises(ValidationError, match="Input should be 0"):
        _completion_payload(checks=checks)