Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

fix(knowledge-fs): settle revokes after deleted Spaces (#45692)

FFXNccbc37868458
diff --git a/api/services/knowledge_fs/lifecycle_remote_http.py b/api/services/knowledge_fs/lifecycle_remote_http.py
index aa91dca2605..a9a7fa99601 100644
--- a/api/services/knowledge_fs/lifecycle_remote_http.py
+++ b/api/services/knowledge_fs/lifecycle_remote_http.py
@@ -223,6 +223,13 @@ class HTTPKnowledgeFSLifecycleRemoteClient:
def revoke_capability_grant(
self, request: KnowledgeFSCapabilityGrantRevokeRequest
) -> KnowledgeFSCapabilityGrantRevokeAck:
+ """Send the bound grant revoke and return only a validated 200 ACK.
+
+ The caller supplies a lifecycle revoke request for one Space and event. A typed
+ deleted-Space conflict remains an error for the saga to verify locally; other
+ transport and response failures also raise without settling the command.
+ """
+
token = self._issue_capability(
namespace_id=request.namespace_id,
control_space_id=request.control_space_id,
@@ -242,6 +249,7 @@ class HTTPKnowledgeFSLifecycleRemoteClient:
"revokeSequence": request.revoke_sequence,
},
expected_statuses=(HTTPStatus.OK,),
+ terminal_conflict_code="CAPABILITY_SPACE_DELETED",
)
try:
ack = _CapabilityGrantRevokeResponse.model_validate(response)
@@ -496,7 +504,15 @@ class HTTPKnowledgeFSLifecycleRemoteClient:
payload: JsonValue | None,
expected_statuses: tuple[HTTPStatus, ...],
query: tuple[tuple[str, str], ...] = (),
+ terminal_conflict_code: str | None = None,
) -> JsonValue:
+ """Send an authenticated lifecycle request and return bounded JSON on allowed statuses.
+
+ The caller supplies a signed token and expected statuses. Every other response or
+ transport failure raises a remote error. A configured terminal conflict code only
+ identifies the remote condition; callers must verify local deletion before settling.
+ """
+
headers = {
"Accept": "application/json",
"Accept-Encoding": "identity",
@@ -534,6 +550,16 @@ class HTTPKnowledgeFSLifecycleRemoteClient:
"KnowledgeFS returned an unsupported lifecycle media type",
)
if response.status_code not in {int(status) for status in expected_statuses}:
+ if response.status_code == HTTPStatus.CONFLICT and terminal_conflict_code is not None:
+ try:
+ error_payload = response.json()
+ except ValueError:
+ error_payload = None
+ if isinstance(error_payload, dict) and error_payload.get("code") == terminal_conflict_code:
+ raise KnowledgeFSLifecycleRemoteError(
+ terminal_conflict_code,
+ "KnowledgeFS reported a completed Space deletion",
+ )
raise KnowledgeFSLifecycleRemoteError(
f"KNOWLEDGE_FS_HTTP_{response.status_code}",
f"KnowledgeFS lifecycle request returned HTTP {response.status_code}",
diff --git a/api/services/knowledge_fs/lifecycle_saga.py b/api/services/knowledge_fs/lifecycle_saga.py
index 91abd8cd81e..3524b10ac56 100644
--- a/api/services/knowledge_fs/lifecycle_saga.py
+++ b/api/services/knowledge_fs/lifecycle_saga.py
@@ -137,7 +137,20 @@ class KnowledgeFSLifecycleSagaRunner:
completed = self._settle_deletion(claimed=claimed, progress=progress, settled_at=now)
return self._result(claimed, completed=completed, settled_at=now)
if claimed.operation is KnowledgeFSLifecycleOperation.REVOKE:
- acknowledgment = self._remote.revoke_capability_grant(_revoke_request(claimed))
+ if self._settle_revoke_after_deletion(claimed=claimed, settled_at=now):
+ return self._result(claimed, completed=True, settled_at=now)
+ try:
+ acknowledgment = self._remote.revoke_capability_grant(_revoke_request(claimed))
+ except KnowledgeFSLifecycleRemoteError as exc:
+ # DELETE can finish while this request is in flight. Recheck its durable
+ # terminal state before persisting another retry of an obsolete command.
+ if self._settle_revoke_after_deletion(
+ claimed=claimed,
+ settled_at=now,
+ remote_deleted=exc.code == "CAPABILITY_SPACE_DELETED",
+ ):
+ return self._result(claimed, completed=True, settled_at=now)
+ raise
completed = self._settle_revoke(
claimed=claimed,
acknowledgment=acknowledgment,
@@ -474,6 +487,61 @@ class KnowledgeFSLifecycleSagaRunner:
raise KnowledgeFSLifecycleLeaseLostError("Delete command lease was lost before retry scheduling")
return False
+ def _settle_revoke_after_deletion(
+ self, *, claimed: _ClaimedCommand, settled_at: datetime, remote_deleted: bool = False
+ ) -> bool:
+ """ACK a bound REVOKE only after local or authenticated remote deletion proof.
+
+ The claimed outbox must retain its original event, key, grant and positive sequence,
+ and the registered Space must match this tenant's DELETE intent. A local DELETED state
+ requires a succeeded DELETE outbox; an in-flight DELETING state additionally requires
+ KFS's typed completed-deletion response. Success changes only the leased REVOKE row;
+ a mismatch leaves it untouched for normal retry.
+ """
+
+ payload = claimed.command_payload
+ sequence = payload.get("revoke_sequence")
+ grant_id = payload.get("grant_id")
+ if (
+ payload.get("event_id") != claimed.operation_id
+ or payload.get("idempotency_key") != claimed.idempotency_key
+ or not isinstance(grant_id, str)
+ or not grant_id
+ or not isinstance(sequence, int)
+ or isinstance(sequence, bool)
+ or sequence < 1
+ ):
+ return False
+ with self._session_maker.begin() as session:
+ control_space = SQLAlchemyKnowledgeFSControlSpaceRepository(session).get(
+ tenant_id=claimed.tenant_id,
+ control_space_id=claimed.control_space_id,
+ )
+ if (
+ control_space is None
+ or control_space.knowledge_space_id is None
+ or control_space.knowledge_space_id != payload.get("knowledge_space_id")
+ or control_space.lifecycle_operation_id is None
+ ):
+ return False
+ outbox_repository = SQLAlchemyKnowledgeFSLifecycleOutboxRepository(session)
+ deletion = outbox_repository.find_latest_for_control_space(
+ tenant_id=claimed.tenant_id,
+ control_space_id=claimed.control_space_id,
+ operation=KnowledgeFSLifecycleOperation.DELETE,
+ )
+ if deletion is None or deletion.operation_id != control_space.lifecycle_operation_id:
+ return False
+ locally_completed = (
+ control_space.state is KnowledgeFSControlSpaceState.DELETED
+ and deletion.status is KnowledgeFSLifecycleOutboxStatus.SUCCEEDED
+ )
+ remotely_completed = remote_deleted and control_space.state is KnowledgeFSControlSpaceState.DELETING
+ if not (locally_completed or remotely_completed):
+ return False
+ self._ack(outbox_repository, claimed=claimed, completed_at=settled_at)
+ return True
+
def _settle_revoke(
self,
*,
diff --git a/api/tests/unit_tests/services/knowledge_fs/test_lifecycle_remote_http.py b/api/tests/unit_tests/services/knowledge_fs/test_lifecycle_remote_http.py
index fb5d5fa58b9..c329f5d67ef 100644
--- a/api/tests/unit_tests/services/knowledge_fs/test_lifecycle_remote_http.py
+++ b/api/tests/unit_tests/services/knowledge_fs/test_lifecycle_remote_http.py
@@ -538,6 +538,55 @@ def test_revoke_capability_grant_sends_monotonic_event_and_maps_ack(
assert issued.resource.id == request.knowledge_space_id
+ ("response_code", "invalid_json", "expected_error_code"),
+ [
+ ("CAPABILITY_SPACE_DELETED", False, "CAPABILITY_SPACE_DELETED"),
+ ("CAPABILITY_REVOKE_CONFLICT", False, "KNOWLEDGE_FS_HTTP_409"),
+ ("CAPABILITY_SPACE_DELETED", True, "KNOWLEDGE_FS_HTTP_409"),
+ ],
+)
+def test_revoke_distinguishes_completed_space_deletion_from_other_conflicts(
+ monkeypatch: pytest.MonkeyPatch, response_code: str, invalid_json: bool, expected_error_code: str
+) -> None:
+ """Only valid JSON carrying the typed deleted-Space code receives a distinct retry reason."""
+
+ issuer = RecordingCapabilityIssuer(token="revoke-capability")
+ response = (
+ httpx.Response(409, text="{", headers={"Content-Type": "application/json"})
+ if invalid_json
+ else httpx.Response(
+ 409,
+ json={"code": response_code, "error": "Capability revoke conflicts with durable state"},
+ headers={"Content-Type": "application/json"},
+ )
+ )
+ monkeypatch.setattr(ssrf_proxy, "make_request", lambda **_: response)
+ monkeypatch.setattr(ssrf_proxy, "buffer_response", lambda response, **_: response)
+ remote = HTTPKnowledgeFSLifecycleRemoteClient(
+ base_url="https://knowledge-fs.test",
+ issuer=issuer.as_issuer(),
+ timeout_seconds=3,
+ )
+ request = KnowledgeFSCapabilityGrantRevokeRequest(
+ namespace_id="tenant-1",
+ control_space_id="30000000-0000-4000-8000-000000000001",
+ operation_id="40000000-0000-4000-8000-000000000002",
+ idempotency_key="revoke-grant-once",
+ knowledge_space_id="10000000-0000-4000-8000-000000000001",
+ grant_id="50000000-0000-4000-8000-000000000001",
+ event_id="60000000-0000-4000-8000-000000000001",
+ reason_code="permission_revoked",
+ revoke_sequence=7,
+ expected_revision=4,
+ )
+
+ with pytest.raises(KnowledgeFSLifecycleRemoteError) as error:
+ remote.revoke_capability_grant(request)
+
+ assert error.value.code == expected_error_code
+
+
def test_list_paginates_with_one_namespace_capability_and_reconstructs_provisioning_keys(
monkeypatch: pytest.MonkeyPatch,
) -> None:
diff --git a/api/tests/unit_tests/services/knowledge_fs/test_lifecycle_saga.py b/api/tests/unit_tests/services/knowledge_fs/test_lifecycle_saga.py
index 44f5d601c17..b0b4102634a 100644
--- a/api/tests/unit_tests/services/knowledge_fs/test_lifecycle_saga.py
+++ b/api/tests/unit_tests/services/knowledge_fs/test_lifecycle_saga.py
@@ -12,8 +12,10 @@ from models.knowledge_fs import (
KnowledgeFSControlSpace,
KnowledgeFSControlSpacePermission,
KnowledgeFSControlSpaceState,
+ KnowledgeFSLifecycleOperation,
KnowledgeFSLifecycleOutbox,
KnowledgeFSLifecycleOutboxStatus,
+ KnowledgeFSRevokeCommandPayload,
)
from services.knowledge_fs.control_space_commands import (
KnowledgeFSControlSpaceCommandService,
@@ -26,6 +28,7 @@ from services.knowledge_fs.lifecycle_port import (
KnowledgeFSDeletionProgress,
KnowledgeFSIntegratedDeletionRequest,
KnowledgeFSIntegratedProvisionRequest,
+ KnowledgeFSLifecycleRemoteError,
KnowledgeFSRemoteSpace,
)
from services.knowledge_fs.lifecycle_saga import KnowledgeFSLifecycleSagaRunner
@@ -268,6 +271,207 @@ def test_deletion_saga_crosses_irreversible_point_and_remains_retryable_when_pro
]
+ "sqlite_session",
+ [(KnowledgeFSControlSpace, KnowledgeFSLifecycleOutbox)],
+ indirect=True,
+)
[email protected]("queued_before_deletion", [True, False])
+def test_revoke_is_acknowledged_after_matching_space_deletion_without_remote_call(
+ sqlite_session: Session, queued_before_deletion: bool
+) -> None:
+ """Both queued and newly produced revokes are covered by the same terminal DELETE."""
+
+ session_maker = _session_maker(sqlite_session)
+ now = datetime(2026, 7, 21, 12, 0)
+ control_space = KnowledgeFSControlSpace(
+ tenant_id="tenant-1",
+ owner_account_id="account-1",
+ provisioning_key="provision-1",
+ knowledge_space_id="space-1",
+ knowledge_space_revision=2,
+ state=KnowledgeFSControlSpaceState.ACTIVE,
+ )
+ sqlite_session.add(control_space)
+ sqlite_session.commit()
+
+ def enqueue_revoke(*, deferred: bool) -> str:
+ event_id = "30000000-0000-4000-8000-000000000001"
+ grant_id = "20000000-0000-4000-8000-000000000001"
+ idempotency_key = f"kfs-revoke:{control_space.id}:4:{grant_id}"
+ with session_maker.begin() as session:
+ command = KnowledgeFSLifecycleOutbox(
+ tenant_id="tenant-1",
+ control_space_id=control_space.id,
+ operation_id=event_id,
+ idempotency_key=idempotency_key,
+ operation=KnowledgeFSLifecycleOperation.REVOKE,
+ command_payload=KnowledgeFSRevokeCommandPayload(
+ schema_version=1,
+ idempotency_key=idempotency_key,
+ expected_revision=2,
+ event_id=event_id,
+ grant_id=grant_id,
+ knowledge_space_id="space-1",
+ principal="dify-account:member-1",
+ reason_code="workspace_rbac_role_changed",
+ revoke_sequence=4,
+ ),
+ expected_control_space_version=control_space.resource_version,
+ next_attempt_at=now + timedelta(minutes=1) if deferred else None,
+ )
+ session.add(command)
+ session.flush()
+ return command.id
+
+ revoke_id = enqueue_revoke(deferred=True) if queued_before_deletion else None
+ deletion = KnowledgeFSControlSpaceCommandService(session_maker).request_deletion(
+ tenant_id="tenant-1",
+ control_space_id=control_space.id,
+ operation_id="delete-1",
+ idempotency_key="delete-idempotency-1",
+ )
+ remote = FakeKnowledgeFSLifecycleRemote()
+ remote.deletion_progress.append(KnowledgeFSDeletionProgress(KnowledgeFSDeletionPhase.COMPLETED, 3, now))
+ runner = KnowledgeFSLifecycleSagaRunner(session_maker, remote)
+ deleted = runner.dispatch_one(
+ worker_id="worker-1",
+ now=now,
+ lease_duration=timedelta(minutes=1),
+ product_enabled=True,
+ )
+ if revoke_id is None:
+ revoke_id = enqueue_revoke(deferred=False)
+ revoked = runner.dispatch_one(
+ worker_id="worker-1",
+ now=now + timedelta(minutes=1),
+ lease_duration=timedelta(minutes=1),
+ product_enabled=True,
+ )
+
+ with session_maker() as session:
+ revoke = session.get(KnowledgeFSLifecycleOutbox, revoke_id)
+ delete = session.get(KnowledgeFSLifecycleOutbox, deletion.outbox.id if deletion.outbox else "")
+ assert deleted.completed is True
+ assert delete is not None
+ assert delete.status is KnowledgeFSLifecycleOutboxStatus.SUCCEEDED
+ assert revoked.completed is True
+ assert revoke is not None
+ assert revoke.status is KnowledgeFSLifecycleOutboxStatus.SUCCEEDED
+
+
+ "sqlite_session",
+ [(KnowledgeFSControlSpace, KnowledgeFSLifecycleOutbox)],
+ indirect=True,
+)
+ ("remote_error_code", "proof_case", "expected_completed", "expected_status"),
+ [
+ ("CAPABILITY_SPACE_DELETED", "valid", True, KnowledgeFSLifecycleOutboxStatus.SUCCEEDED),
+ ("KNOWLEDGE_FS_HTTP_409", "valid", False, KnowledgeFSLifecycleOutboxStatus.RETRY),
+ ("CAPABILITY_SPACE_DELETED", "event_mismatch", False, KnowledgeFSLifecycleOutboxStatus.RETRY),
+ ("CAPABILITY_SPACE_DELETED", "space_mismatch", False, KnowledgeFSLifecycleOutboxStatus.RETRY),
+ ("CAPABILITY_SPACE_DELETED", "delete_mismatch", False, KnowledgeFSLifecycleOutboxStatus.RETRY),
+ ],
+)
+def test_only_typed_remote_deletion_settles_revoke_while_local_delete_is_pending(
+ sqlite_session: Session,
+ remote_error_code: str,
+ proof_case: str,
+ expected_completed: bool,
+ expected_status: KnowledgeFSLifecycleOutboxStatus,
+) -> None:
+ """A typed KFS response settles only a REVOKE bound to the original DELETE intent."""
+
+ session_maker = _session_maker(sqlite_session)
+ now = datetime(2026, 7, 21, 12, 0)
+ control_space = KnowledgeFSControlSpace(
+ tenant_id="tenant-1",
+ owner_account_id="account-1",
+ provisioning_key="provision-1",
+ knowledge_space_id="space-1",
+ knowledge_space_revision=2,
+ state=KnowledgeFSControlSpaceState.ACTIVE,
+ )
+ sqlite_session.add(control_space)
+ sqlite_session.commit()
+ deletion = KnowledgeFSControlSpaceCommandService(session_maker).request_deletion(
+ tenant_id="tenant-1",
+ control_space_id=control_space.id,
+ operation_id="delete-1",
+ idempotency_key="delete-idempotency-1",
+ )
+ assert deletion.outbox is not None
+ with session_maker.begin() as session:
+ delete = session.get(KnowledgeFSLifecycleOutbox, deletion.outbox.id)
+ assert delete is not None
+ delete.next_attempt_at = now + timedelta(hours=1)
+ if proof_case == "delete_mismatch":
+ stored_space = session.get(KnowledgeFSControlSpace, control_space.id)
+ assert stored_space is not None
+ stored_space.lifecycle_operation_id = "different-delete-operation"
+ revoke = KnowledgeFSLifecycleOutbox(
+ tenant_id="tenant-1",
+ control_space_id=control_space.id,
+ operation_id="30000000-0000-4000-8000-000000000001",
+ idempotency_key="revoke-1",
+ operation=KnowledgeFSLifecycleOperation.REVOKE,
+ command_payload=KnowledgeFSRevokeCommandPayload(
+ schema_version=1,
+ idempotency_key="revoke-1",
+ expected_revision=2,
+ event_id=(
+ "30000000-0000-4000-8000-000000000002"
+ if proof_case == "event_mismatch"
+ else "30000000-0000-4000-8000-000000000001"
+ ),
+ grant_id="20000000-0000-4000-8000-000000000001",
+ knowledge_space_id="space-2" if proof_case == "space_mismatch" else "space-1",
+ principal="dify-account:member-1",
+ reason_code="workspace_rbac_role_changed",
+ revoke_sequence=4,
+ ),
+ expected_control_space_version=control_space.resource_version,
+ )
+ session.add(revoke)
+ session.flush()
+ revoke_id = revoke.id
+
+ class DeletedSpaceRemote(FakeKnowledgeFSLifecycleRemote):
+ def __init__(self) -> None:
+ super().__init__()
+ self.revoke_requests: list[KnowledgeFSCapabilityGrantRevokeRequest] = []
+
+ @override
+ def revoke_capability_grant(
+ self, request: KnowledgeFSCapabilityGrantRevokeRequest
+ ) -> KnowledgeFSCapabilityGrantRevokeAck:
+ self.revoke_requests.append(request)
+ raise KnowledgeFSLifecycleRemoteError(remote_error_code, "Capability revoke failed")
+
+ remote = DeletedSpaceRemote()
+ result = KnowledgeFSLifecycleSagaRunner(session_maker, remote).dispatch_one(
+ worker_id="worker-1",
+ now=now,
+ lease_duration=timedelta(minutes=1),
+ product_enabled=True,
+ )
+
+ with session_maker() as session:
+ persisted_revoke = session.get(KnowledgeFSLifecycleOutbox, revoke_id)
+ persisted_delete = session.get(KnowledgeFSLifecycleOutbox, deletion.outbox.id)
+ persisted_space = session.get(KnowledgeFSControlSpace, control_space.id)
+ assert result.completed is expected_completed
+ assert len(remote.revoke_requests) == 1
+ assert persisted_revoke is not None
+ assert persisted_revoke.status is expected_status
+ assert persisted_delete is not None
+ assert persisted_delete.status is KnowledgeFSLifecycleOutboxStatus.PENDING
+ assert persisted_space is not None
+ assert persisted_space.state is KnowledgeFSControlSpaceState.DELETING
+
+
@pytest.mark.parametrize(
"sqlite_session",
[