Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

11.5 KB
Raw Blame
"""Server-owned commit boundary for durable Add source imports."""

from __future__ import annotations

from services.knowledge_fs.product_dto import (
    KnowledgeFSAsyncCrawlPreviewImportPayload,
    KnowledgeFSAsyncOnlineDocumentImportPayload,
    KnowledgeFSAsyncOnlineDriveImportPayload,
    KnowledgeFSAsyncSourceImport,
    KnowledgeFSAsyncWebsiteCrawlImportPayload,
    KnowledgeFSCrawlImportPayload,
    KnowledgeFSCrawlPreviewSelectionPayload,
    KnowledgeFSOnlineDocumentWorkflowImportPayload,
    KnowledgeFSOnlineDriveWorkflowImportPayload,
    KnowledgeFSSourceEditSyncPolicyPayload,
    KnowledgeFSSourceImportConfigurationPayload,
    KnowledgeFSSourceUpdatePayload,
    KnowledgeFSSourceWorkflowImportPayload,
    KnowledgeFSSourceWorkflowResponse,
)
from services.knowledge_fs.product_remote import KnowledgeFSProductRequestRejectedError

_ASYNC_IMPORT_KINDS = {
    "crawl-preview-selection",
    "website-crawl-import",
    "online-document-import",
    "online-drive-import",
}
_PENDING_IMPORT_KEY = "pendingImport"
_INITIAL_IMPORT_KEY = "initialImport"
_MANUAL_SYNC_POLICY = {"enabled": False, "mode": "manual"}


def commit_source_import(
    *,
    facade,
    tenant_id: str,
    account_id: str,
    control_space_id: str,
    source_id: str,
    payload: KnowledgeFSAsyncSourceImport,
    idempotency_key: str,
    source_update: KnowledgeFSSourceImportConfigurationPayload | None = None,
) -> KnowledgeFSSourceWorkflowResponse:
    """Start an import and transfer reconciliation ownership to the backend."""

    desired_sync_policy = (
        KnowledgeFSSourceEditSyncPolicyPayload.model_validate(payload.sync_policy.model_dump(by_alias=True))
        if source_update is not None
        else None
    )
    preview_workflow_id: str | None = None
    if isinstance(payload, KnowledgeFSAsyncCrawlPreviewImportPayload):
        if source_update is not None:
            raise ValueError("Atomic source edits require crawl-import or workflow-imports")
        preview_workflow_id = payload.preview_workflow_id
        preview_workflow = facade.get_source_workflow(
            tenant_id=tenant_id,
            account_id=account_id,
            control_space_id=control_space_id,
            run_id=preview_workflow_id,
        )
        if preview_workflow.source_id != source_id:
            raise ValueError("KnowledgeFS crawl preview workflow does not belong to this Source")
        import_workflow = facade.select_crawl_preview_pages(
            tenant_id=tenant_id,
            account_id=account_id,
            control_space_id=control_space_id,
            run_id=preview_workflow_id,
            payload=KnowledgeFSCrawlPreviewSelectionPayload(pageIds=payload.page_ids),
            idempotency_key=idempotency_key,
        )
    elif isinstance(payload, KnowledgeFSAsyncWebsiteCrawlImportPayload):
        import_workflow = facade.import_selected_source_crawl(
            tenant_id=tenant_id,
            account_id=account_id,
            control_space_id=control_space_id,
            source_id=source_id,
            payload=KnowledgeFSCrawlImportPayload(
                sourceUrls=payload.source_urls,
                replaceExistingSelection=True,
                sourceUpdate=source_update,
                desiredSyncPolicy=desired_sync_policy,
            ),
            idempotency_key=idempotency_key,
        )
    elif isinstance(payload, KnowledgeFSAsyncOnlineDocumentImportPayload):
        import_workflow = facade.import_source_workflow(
            tenant_id=tenant_id,
            account_id=account_id,
            control_space_id=control_space_id,
            source_id=source_id,
            payload=KnowledgeFSSourceWorkflowImportPayload(
                KnowledgeFSOnlineDocumentWorkflowImportPayload(
                    kind="online-document-import",
                    items=payload.items,
                    sourceUpdate=source_update,
                    desiredSyncPolicy=desired_sync_policy,
                )
            ),
            idempotency_key=idempotency_key,
        )
    elif isinstance(payload, KnowledgeFSAsyncOnlineDriveImportPayload):
        import_workflow = facade.import_source_workflow(
            tenant_id=tenant_id,
            account_id=account_id,
            control_space_id=control_space_id,
            source_id=source_id,
            payload=KnowledgeFSSourceWorkflowImportPayload(
                KnowledgeFSOnlineDriveWorkflowImportPayload(
                    kind="online-drive-import",
                    items=payload.items,
                    sourceUpdate=source_update,
                    desiredSyncPolicy=desired_sync_policy,
                )
            ),
            idempotency_key=idempotency_key,
        )
    else:
        raise TypeError("Unsupported KnowledgeFS async Source import")

    if import_workflow.source_id != source_id:
        raise RuntimeError("KnowledgeFS import workflow returned a different Source")
    if source_update is None:
        source = facade.get_source(
            tenant_id=tenant_id,
            account_id=account_id,
            control_space_id=control_space_id,
            source_id=source_id,
        )
        pending_import = {
            "kind": payload.kind,
            **({"previewWorkflowId": preview_workflow_id} if preview_workflow_id is not None else {}),
            "workflowId": import_workflow.id,
            "syncPolicy": payload.sync_policy.model_dump(mode="json", by_alias=True),
        }
        if source.metadata.get(_PENDING_IMPORT_KEY) != pending_import or source.status != "syncing":
            facade.update_source(
                tenant_id=tenant_id,
                account_id=account_id,
                control_space_id=control_space_id,
                source_id=source.id,
                payload=KnowledgeFSSourceUpdatePayload(
                    expectedVersion=source.version,
                    metadata={"preview": False, _PENDING_IMPORT_KEY: pending_import},
                    status="syncing",
                ),
            )

    from tasks.knowledge_fs_source_import_tasks import finalize_source_import

    finalize_source_import.delay(
        tenant_id=tenant_id,
        account_id=account_id,
        control_space_id=control_space_id,
        source_id=source_id,
        workflow_id=import_workflow.id,
    )
    return import_workflow


def resume_committed_source_import(
    *, facade, tenant_id: str, account_id: str, control_space_id: str, workflow: KnowledgeFSSourceWorkflowResponse
) -> None:
    """Restore server reconciliation when a committed import is retried or reconciled."""

    if workflow.source_id is None:
        return
    source = facade.get_source(
        tenant_id=tenant_id,
        account_id=account_id,
        control_space_id=control_space_id,
        source_id=workflow.source_id,
    )
    last_import = _matching_import_marker(source.metadata.get("lastImport"), workflow.id)
    current_pending_import = _matching_import_marker(source.metadata.get(_PENDING_IMPORT_KEY), workflow.id)
    initial_import = _matching_initial_website_import_marker(
        source.metadata.get(_INITIAL_IMPORT_KEY),
        workflow.id,
        source.metadata.get("providerKind"),
    )
    import_marker = last_import or current_pending_import or initial_import
    if import_marker is None:
        return
    pending_import = {
        "kind": import_marker.get("kind"),
        **(
            {"previewWorkflowId": import_marker.get("previewWorkflowId")}
            if import_marker.get("previewWorkflowId") is not None
            else {}
        ),
        "workflowId": workflow.id,
        "syncPolicy": import_marker.get("syncPolicy"),
    }
    from tasks.knowledge_fs_source_import_tasks import finalize_source_import

    try:
        if last_import is not None or current_pending_import != pending_import or source.status != "syncing":
            # updateSource merges metadata; null explicitly supersedes the terminal marker while retrying.
            try:
                facade.update_source(
                    tenant_id=tenant_id,
                    account_id=account_id,
                    control_space_id=control_space_id,
                    source_id=source.id,
                    payload=KnowledgeFSSourceUpdatePayload(
                        expectedVersion=source.version,
                        metadata={
                            _INITIAL_IMPORT_KEY: None,
                            "lastImport": None,
                            "preview": False,
                            _PENDING_IMPORT_KEY: pending_import,
                        },
                        status="syncing",
                    ),
                )
            except KnowledgeFSProductRequestRejectedError as error:
                if error.status_code != 409:
                    raise
                # A prior reconciler can finish its stale terminal write while retry is accepted.
                # The newly dispatched reconciler below re-reads both authorities and repairs it.
    finally:
        # The remote Source update may commit before its follow-up sync-policy bind fails.
        # Always restore reconciliation ownership once this workflow marker is authoritative.
        finalize_source_import.delay(
            tenant_id=tenant_id,
            account_id=account_id,
            control_space_id=control_space_id,
            source_id=source.id,
            workflow_id=workflow.id,
        )


def retry_or_resume_source_workflow(
    *, facade, tenant_id: str, account_id: str, control_space_id: str, run_id: str
) -> KnowledgeFSSourceWorkflowResponse:
    """Retry a failed run, or reconcile an already-completed import idempotently."""

    workflow = facade.get_source_workflow(
        tenant_id=tenant_id,
        account_id=account_id,
        control_space_id=control_space_id,
        run_id=run_id,
    )
    if workflow.state != "completed":
        try:
            workflow = facade.retry_source_workflow(
                tenant_id=tenant_id,
                account_id=account_id,
                control_space_id=control_space_id,
                run_id=run_id,
            )
        except KnowledgeFSProductRequestRejectedError as error:
            if error.status_code != 409:
                raise
            # A concurrent retry may have completed between the authoritative read and mutation.
            workflow = facade.get_source_workflow(
                tenant_id=tenant_id,
                account_id=account_id,
                control_space_id=control_space_id,
                run_id=run_id,
            )
            if workflow.state != "completed":
                raise
    resume_committed_source_import(
        facade=facade,
        tenant_id=tenant_id,
        account_id=account_id,
        control_space_id=control_space_id,
        workflow=workflow,
    )
    return workflow


def _matching_import_marker(value: object, workflow_id: str) -> dict[str, object] | None:
    if (
        not isinstance(value, dict)
        or value.get("kind") not in _ASYNC_IMPORT_KINDS
        or value.get("workflowId") != workflow_id
    ):
        return None
    return value


def _matching_initial_website_import_marker(
    value: object, workflow_id: str, provider_kind: object
) -> dict[str, object] | None:
    if not isinstance(value, dict) or value.get("workflowId") != workflow_id or provider_kind != "website":
        return None
    sync_policy = value.get("syncPolicy")
    return {
        "kind": "website-crawl-import",
        "syncPolicy": sync_policy if isinstance(sync_policy, dict) else _MANUAL_SYNC_POLICY,
        "workflowId": workflow_id,
    }


__all__ = ["commit_source_import", "resume_committed_source_import", "retry_or_resume_source_workflow"]