Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

16.9 KB
Raw Blame
"""Trusted KnowledgeFS gateway to Dify's configured object-storage backend."""

import json
from base64 import b64decode
from binascii import Error as BinasciiError
from http import HTTPStatus
from typing import NoReturn

from flask import Response, request
from flask_restx import Resource
from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, ValidationError
from pydantic.alias_generators import to_camel

from controllers.common.schema import query_params_from_model, register_response_schema_models
from controllers.inner_api import inner_api_ns
from controllers.inner_api.wraps import knowledge_fs_inner_api_only
from fields.base import ResponseModel
from libs.exception import BaseHTTPException
from libs.helper import dump_response
from services.knowledge_fs.object_storage import (
    KNOWLEDGE_FS_OBJECT_MAX_BYTES,
    KnowledgeFSObjectList,
    KnowledgeFSObjectMetadata,
    KnowledgeFSObjectStorageChecksumError,
    KnowledgeFSObjectStorageCorruptError,
    KnowledgeFSObjectStorageError,
    KnowledgeFSObjectStorageInvalidInputError,
    KnowledgeFSObjectStorageService,
    KnowledgeFSObjectStorageTooLargeError,
    KnowledgeFSObjectStorageUnavailableError,
)
from services.knowledge_fs.query_images import KnowledgeFSQueryImageError, load_query_image
from services.knowledge_fs.remote_images import (
    KnowledgeFSRemoteImageError,
    load_remote_image,
)

_METADATA_HEADER = "X-Knowledge-FS-Metadata"
_CHECKSUM_HEADER = "X-Knowledge-FS-Checksum-Sha256"
_CONTENT_TYPE_HEADER = "X-Knowledge-FS-Content-Type"
_MAX_ENCODED_METADATA_BYTES = 128 * 1024
_metadata_adapter = TypeAdapter(dict[str, str])


class KnowledgeFSObjectStorageHttpError(BaseHTTPException):
    """Safe HTTP representation of a KnowledgeFS storage boundary error."""

    error_code = "knowledge_fs_object_storage_failed"
    description = "KnowledgeFS object storage request failed."
    code = HTTPStatus.INTERNAL_SERVER_ERROR

    def __init__(self, *, error_code: str, description: str, status_code: HTTPStatus) -> None:
        self.error_code = error_code
        self.description = description
        self.code = status_code
        super().__init__(description)


class _CamelCaseResponse(ResponseModel):
    model_config = ConfigDict(alias_generator=to_camel)


class KnowledgeFSObjectQuery(BaseModel):
    model_config = ConfigDict(extra="forbid")

    key: str = Field(description="Logical KnowledgeFS object key")


class KnowledgeFSObjectListQuery(BaseModel):
    model_config = ConfigDict(extra="forbid")

    prefix: str = Field(default="", description="Logical object-key prefix")
    cursor: str | None = Field(default=None, description="Exclusive lexical key cursor")
    limit: int = Field(default=100, ge=1, le=100, description="Maximum objects to return")


class KnowledgeFSObjectMetadataResponse(_CamelCaseResponse):
    checksum_sha256_base64: str
    content_type: str | None = None
    key: str
    metadata: dict[str, str]
    size_bytes: int


class KnowledgeFSObjectListResponse(_CamelCaseResponse):
    objects: list[KnowledgeFSObjectMetadataResponse]
    next_cursor: str | None = None


class KnowledgeFSObjectHealthResponse(ResponseModel):
    ok: bool


class KnowledgeFSQueryImageQuery(BaseModel):
    model_config = ConfigDict(extra="forbid", alias_generator=to_camel, populate_by_name=True)

    upload_file_id: str = Field(min_length=1)
    tenant_id: str = Field(min_length=1)
    subject_id: str = Field(min_length=1)


class KnowledgeFSRemoteImageQuery(BaseModel):
    model_config = ConfigDict(extra="forbid")

    url: str = Field(min_length=1, max_length=8_192)


register_response_schema_models(
    inner_api_ns,
    KnowledgeFSObjectMetadataResponse,
    KnowledgeFSObjectListResponse,
    KnowledgeFSObjectHealthResponse,
)


@inner_api_ns.route("/knowledge-fs/query-image")
class KnowledgeFSQueryImageApi(Resource):
    """Resolve one actor-owned or workflow-granted Dify file for a bounded KFS query run."""

    @inner_api_ns.doc(params=query_params_from_model(KnowledgeFSQueryImageQuery))
    @inner_api_ns.produces(["image/gif", "image/jpeg", "image/png", "image/webp"])
    @knowledge_fs_inner_api_only
    def get(self) -> Response:
        try:
            query = KnowledgeFSQueryImageQuery.model_validate(request.args.to_dict(flat=True))
            access_grant = request.headers.get("X-Knowledge-FS-Query-Image-Grant")
            if access_grant is not None and (not access_grant or len(access_grant) > 2_048):
                raise KnowledgeFSQueryImageError("QUERY_IMAGE_GRANT_INVALID", "Workflow query image grant is invalid")
            account_id = None if access_grant else _query_image_account_id(query.subject_id)
            image = load_query_image(
                tenant_id=query.tenant_id,
                subject_id=query.subject_id,
                account_id=account_id,
                access_grant=access_grant,
                upload_file_id=query.upload_file_id,
            )
        except ValidationError as exc:
            raise _invalid_request_error() from exc
        except KnowledgeFSQueryImageError as exc:
            status = (
                HTTPStatus.NOT_FOUND
                if exc.code == "QUERY_IMAGE_NOT_FOUND"
                else HTTPStatus.REQUEST_ENTITY_TOO_LARGE
                if exc.code in {"QUERY_IMAGE_TOO_LARGE", "QUERY_IMAGE_TOTAL_TOO_LARGE"}
                else HTTPStatus.BAD_REQUEST
            )
            raise KnowledgeFSObjectStorageHttpError(
                error_code=exc.code.lower(),
                description=str(exc),
                status_code=status,
            ) from exc

        response = Response(image.body, content_type=image.mime_type)
        response.content_length = image.byte_size
        response.headers["X-Query-Image-Sha256"] = image.sha256
        response.headers["X-Query-Image-Upload-File-Id"] = image.upload_file_id
        return response


@inner_api_ns.route("/knowledge-fs/remote-image")
class KnowledgeFSRemoteImageApi(Resource):
    """Resolve one bounded document image through Dify's SSRF-protected network client."""

    @inner_api_ns.doc(params=query_params_from_model(KnowledgeFSRemoteImageQuery))
    @inner_api_ns.produces(["image/gif", "image/jpeg", "image/png", "image/webp"])
    @knowledge_fs_inner_api_only
    def get(self) -> Response:
        try:
            query = KnowledgeFSRemoteImageQuery.model_validate(request.args.to_dict(flat=True))
            image = load_remote_image(query.url)
        except ValidationError as exc:
            raise _invalid_request_error() from exc
        except KnowledgeFSRemoteImageError as exc:
            raise _remote_image_http_error(exc) from exc

        response = Response(image.body, content_type=image.mime_type)
        response.content_length = image.byte_size
        response.headers["X-Knowledge-FS-Remote-Image-Sha256"] = image.sha256
        return response


@inner_api_ns.route("/knowledge-fs/storage/object")
class KnowledgeFSObjectApi(Resource):
    """Read, write, or delete one logical KnowledgeFS object."""

    @inner_api_ns.doc(params=query_params_from_model(KnowledgeFSObjectQuery))
    @inner_api_ns.response(
        HTTPStatus.OK,
        "Object stored",
        inner_api_ns.models[KnowledgeFSObjectMetadataResponse.__name__],
    )
    @knowledge_fs_inner_api_only
    def put(self) -> dict[str, object]:
        try:
            query = KnowledgeFSObjectQuery.model_validate(request.args.to_dict(flat=True))
            metadata = _decode_metadata_header(request.headers.get(_METADATA_HEADER))
            body = request.stream.read(KNOWLEDGE_FS_OBJECT_MAX_BYTES + 1)
            if len(body) > KNOWLEDGE_FS_OBJECT_MAX_BYTES:
                raise KnowledgeFSObjectStorageTooLargeError(f"object exceeds max bytes {KNOWLEDGE_FS_OBJECT_MAX_BYTES}")
            result = KnowledgeFSObjectStorageService().put_object(
                body=body,
                checksum_sha256_base64=request.headers.get(_CHECKSUM_HEADER),
                content_type=request.headers.get(_CONTENT_TYPE_HEADER),
                key=query.key,
                metadata=metadata,
            )
        except ValidationError as exc:
            raise _invalid_request_error() from exc
        except KnowledgeFSObjectStorageError as exc:
            _raise_http_error(exc)
        return _metadata_response(result)

    @inner_api_ns.doc(params=query_params_from_model(KnowledgeFSObjectQuery))
    @inner_api_ns.produces(["application/octet-stream"])
    @knowledge_fs_inner_api_only
    def get(self) -> Response:
        try:
            query = KnowledgeFSObjectQuery.model_validate(request.args.to_dict(flat=True))
            service = KnowledgeFSObjectStorageService()
            metadata = service.head_object(key=query.key)
            if metadata is None:
                raise _not_found_error()
            body = service.load_stream(key=query.key)
            if body is None:
                raise _not_found_error()
        except ValidationError as exc:
            raise _invalid_request_error() from exc
        except KnowledgeFSObjectStorageError as exc:
            _raise_http_error(exc)

        response = Response(
            body,
            content_type=metadata.content_type or "application/octet-stream",
        )
        response.content_length = metadata.size_bytes
        response.headers[_CHECKSUM_HEADER] = metadata.checksum_sha256_base64
        return response

    @inner_api_ns.doc(params=query_params_from_model(KnowledgeFSObjectQuery))
    @inner_api_ns.response(HTTPStatus.NO_CONTENT, "Object deleted")
    @knowledge_fs_inner_api_only
    def delete(self) -> tuple[str, int]:
        try:
            query = KnowledgeFSObjectQuery.model_validate(request.args.to_dict(flat=True))
            KnowledgeFSObjectStorageService().delete_object(key=query.key)
        except ValidationError as exc:
            raise _invalid_request_error() from exc
        except KnowledgeFSObjectStorageError as exc:
            _raise_http_error(exc)
        return "", HTTPStatus.NO_CONTENT


@inner_api_ns.route("/knowledge-fs/storage/object/metadata")
class KnowledgeFSObjectMetadataApi(Resource):
    """Read portable metadata for one logical KnowledgeFS object."""

    @inner_api_ns.doc(params=query_params_from_model(KnowledgeFSObjectQuery))
    @inner_api_ns.response(
        HTTPStatus.OK,
        "Object metadata",
        inner_api_ns.models[KnowledgeFSObjectMetadataResponse.__name__],
    )
    @knowledge_fs_inner_api_only
    def get(self) -> dict[str, object]:
        try:
            query = KnowledgeFSObjectQuery.model_validate(request.args.to_dict(flat=True))
            result = KnowledgeFSObjectStorageService().head_object(key=query.key)
            if result is None:
                raise _not_found_error()
        except ValidationError as exc:
            raise _invalid_request_error() from exc
        except KnowledgeFSObjectStorageError as exc:
            _raise_http_error(exc)
        return _metadata_response(result)


@inner_api_ns.route("/knowledge-fs/storage/objects")
class KnowledgeFSObjectListApi(Resource):
    """List logical KnowledgeFS objects with bounded keyset pagination."""

    @inner_api_ns.doc(params=query_params_from_model(KnowledgeFSObjectListQuery))
    @inner_api_ns.response(
        HTTPStatus.OK,
        "Object page",
        inner_api_ns.models[KnowledgeFSObjectListResponse.__name__],
    )
    @knowledge_fs_inner_api_only
    def get(self) -> dict[str, object]:
        try:
            query = KnowledgeFSObjectListQuery.model_validate(request.args.to_dict(flat=True))
            result = KnowledgeFSObjectStorageService().list_objects(
                cursor=query.cursor,
                limit=query.limit,
                prefix=query.prefix,
            )
        except ValidationError as exc:
            raise _invalid_request_error() from exc
        except KnowledgeFSObjectStorageError as exc:
            _raise_http_error(exc)
        return _list_response(result)


@inner_api_ns.route("/knowledge-fs/storage/health")
class KnowledgeFSObjectHealthApi(Resource):
    """Report whether Dify storage satisfies KnowledgeFS portable requirements."""

    @inner_api_ns.response(
        HTTPStatus.OK,
        "Storage available",
        inner_api_ns.models[KnowledgeFSObjectHealthResponse.__name__],
    )
    @inner_api_ns.response(HTTPStatus.SERVICE_UNAVAILABLE, "Storage unavailable")
    @knowledge_fs_inner_api_only
    def get(self) -> dict[str, bool] | tuple[dict[str, bool], int]:
        if KnowledgeFSObjectStorageService().health():
            return {"ok": True}
        return {"ok": False}, HTTPStatus.SERVICE_UNAVAILABLE


def _decode_metadata_header(value: str | None) -> dict[str, str]:
    if value is None:
        return {}
    if len(value.encode()) > _MAX_ENCODED_METADATA_BYTES:
        raise KnowledgeFSObjectStorageInvalidInputError("object metadata header is too large")
    try:
        padding = "=" * (-len(value) % 4)
        decoded = b64decode(value + padding, altchars=b"-_", validate=True)
        return _metadata_adapter.validate_json(decoded)
    except (BinasciiError, UnicodeEncodeError, ValidationError, json.JSONDecodeError) as exc:
        raise KnowledgeFSObjectStorageInvalidInputError("object metadata header is invalid") from exc


def _query_image_account_id(subject_id: str) -> str:
    prefix = "dify-account:"
    account_id = subject_id[len(prefix) :].strip() if subject_id.startswith(prefix) else subject_id.strip()
    if not account_id:
        raise KnowledgeFSQueryImageError("QUERY_IMAGE_SUBJECT_INVALID", "Query image subject is invalid")
    return account_id


def _metadata_response(metadata: KnowledgeFSObjectMetadata) -> dict[str, object]:
    return dump_response(KnowledgeFSObjectMetadataResponse, metadata)


def _list_response(result: KnowledgeFSObjectList) -> dict[str, object]:
    return dump_response(KnowledgeFSObjectListResponse, result)


def _raise_http_error(error: KnowledgeFSObjectStorageError) -> NoReturn:
    if isinstance(error, KnowledgeFSObjectStorageTooLargeError):
        raise KnowledgeFSObjectStorageHttpError(
            error_code="knowledge_fs_object_too_large",
            description="KnowledgeFS object exceeds the configured size limit.",
            status_code=HTTPStatus.REQUEST_ENTITY_TOO_LARGE,
        ) from error
    if isinstance(error, KnowledgeFSObjectStorageChecksumError):
        raise KnowledgeFSObjectStorageHttpError(
            error_code="knowledge_fs_object_checksum_mismatch",
            description="KnowledgeFS object checksum does not match the request body.",
            status_code=HTTPStatus.UNPROCESSABLE_ENTITY,
        ) from error
    if isinstance(error, KnowledgeFSObjectStorageInvalidInputError):
        raise _invalid_request_error() from error
    if isinstance(error, KnowledgeFSObjectStorageCorruptError):
        raise KnowledgeFSObjectStorageHttpError(
            error_code="knowledge_fs_object_corrupt",
            description="KnowledgeFS object metadata is inconsistent.",
            status_code=HTTPStatus.BAD_GATEWAY,
        ) from error
    if isinstance(error, KnowledgeFSObjectStorageUnavailableError):
        raise KnowledgeFSObjectStorageHttpError(
            error_code="knowledge_fs_object_storage_unavailable",
            description="Dify object storage is unavailable for KnowledgeFS.",
            status_code=HTTPStatus.SERVICE_UNAVAILABLE,
        ) from error
    raise KnowledgeFSObjectStorageHttpError(
        error_code="knowledge_fs_object_storage_failed",
        description="KnowledgeFS object storage request failed.",
        status_code=HTTPStatus.INTERNAL_SERVER_ERROR,
    ) from error


def _invalid_request_error() -> KnowledgeFSObjectStorageHttpError:
    return KnowledgeFSObjectStorageHttpError(
        error_code="knowledge_fs_object_storage_invalid_request",
        description="KnowledgeFS object storage request is invalid.",
        status_code=HTTPStatus.BAD_REQUEST,
    )


def _not_found_error() -> KnowledgeFSObjectStorageHttpError:
    return KnowledgeFSObjectStorageHttpError(
        error_code="knowledge_fs_object_not_found",
        description="KnowledgeFS object was not found.",
        status_code=HTTPStatus.NOT_FOUND,
    )


def _remote_image_http_error(error: KnowledgeFSRemoteImageError) -> KnowledgeFSObjectStorageHttpError:
    status_by_code = {
        "REMOTE_IMAGE_BLOCKED": HTTPStatus.FORBIDDEN,
        "REMOTE_IMAGE_CONTENT_UNSUPPORTED": HTTPStatus.UNSUPPORTED_MEDIA_TYPE,
        "REMOTE_IMAGE_EMPTY": HTTPStatus.UNPROCESSABLE_ENTITY,
        "REMOTE_IMAGE_HOST_NOT_FOUND": HTTPStatus.NOT_FOUND,
        "REMOTE_IMAGE_NOT_FOUND": HTTPStatus.NOT_FOUND,
        "REMOTE_IMAGE_RATE_LIMITED": HTTPStatus.TOO_MANY_REQUESTS,
        "REMOTE_IMAGE_REQUEST_REJECTED": HTTPStatus.UNPROCESSABLE_ENTITY,
        "REMOTE_IMAGE_TOO_LARGE": HTTPStatus.REQUEST_ENTITY_TOO_LARGE,
        "REMOTE_IMAGE_UPSTREAM_UNAVAILABLE": HTTPStatus.BAD_GATEWAY,
        "REMOTE_IMAGE_URL_INVALID": HTTPStatus.BAD_REQUEST,
    }
    return KnowledgeFSObjectStorageHttpError(
        error_code=error.code.lower(),
        description=str(error),
        status_code=status_by_code.get(error.code, HTTPStatus.BAD_GATEWAY),
    )