Skip to content

armgit / zulip

Public Imported from GitHub
0

Zulip server and web application. Open-source team chat that helps teams stay productive and focused.

23.7 KB
Raw Blame
import inspect
import json
import types
from collections.abc import Callable, Sequence
from dataclasses import dataclass
from enum import Enum, auto
from functools import wraps
from typing import (
    Annotated,
    Concatenate,
    Generic,
    TypeAlias,
    TypeVar,
    Union,
    get_args,
    get_origin,
    get_type_hints,
)

from django.http import HttpRequest
from django.utils.translation import gettext as _
from pydantic import Json, StringConstraints, TypeAdapter, ValidationError
from typing_extensions import ParamSpec

from zerver.lib.exceptions import ApiParamValidationError, JsonableError
from zerver.lib.request import (
    RequestConfusingParamsError,
    RequestNotes,
    RequestVariableMissingError,
    arguments_map,
)
from zerver.lib.response import MutableJsonResponse

T = TypeVar("T")
ParamT = ParamSpec("ParamT")
ReturnT = TypeVar("ReturnT")


class DocumentationStatus(Enum):
    DOCUMENTED = auto()
    INTENTIONALLY_UNDOCUMENTED = auto()
    DOCUMENTATION_PENDING = auto()


DOCUMENTED = DocumentationStatus.DOCUMENTED
INTENTIONALLY_UNDOCUMENTED = DocumentationStatus.INTENTIONALLY_UNDOCUMENTED
DOCUMENTATION_PENDING = DocumentationStatus.DOCUMENTATION_PENDING


@dataclass(frozen=True)
class ApiParamConfig:
    """The metadata associated with a view function parameter as an annotation
    to configure how the typed_endpoint decorator should process it.

    It should be used with Annotated as the type annotation of a parameter
    in a @typed_endpoint-decorated function:
    ```
    @typed_endpoint
    def view(
        request: HttpRequest,
        *,
        flag_value: Annotated[Json[bool], ApiParamConfig(
            whence="flag",
            documentation_status=INTENTIONALLY_UNDOCUMENTED,
        )]
    ) -> HttpResponse:
        ...
    ```

    For a parameter that is not annotated with ApiParamConfig, typed_endpoint
    will construct a configuration using the defaults.

    whence:
    The name of the request variable that should be used for this parameter.
    If None, it is set to the name of the function parameter.

    path_only:
    Used for parameters included in the URL.

    argument_type_is_body:
    When set to true, the value of the parameter will be extracted from the
    request body instead of a single query parameter.

    documentation_status:
    The OpenAPI documentation status of this parameter. Unless it is set to
    INTENTIONALLY_UNDOCUMENTED or DOCUMENTATION_PENDING, the test suite is
    configured to raise an error when its documentation cannot be found.

    aliases:
    The names allowed for the request variable other than that specified with
    "whence".
    """

    whence: str | None = None
    path_only: bool = False
    argument_type_is_body: bool = False
    documentation_status: DocumentationStatus = DOCUMENTED
    aliases: tuple[str, ...] = ()


# TypeAliases for common Annotated types

# Commonly used for webhook views, where the payload has a content type of
# application/json. It reads the data from request body and parse it from JSON.
JsonBodyPayload: TypeAlias = Annotated[Json[T], ApiParamConfig(argument_type_is_body=True)]
# A shorthand to declare path only variables that should not be parsed from the
# request by the @typed_endpoint decorator.
PathOnly: TypeAlias = Annotated[T, ApiParamConfig(path_only=True)]
OptionalTopic: TypeAlias = Annotated[
    str | None,
    StringConstraints(strip_whitespace=True),
    ApiParamConfig(whence="topic", aliases=("subject",)),
]
ApnsAppId: TypeAlias = Annotated[str, StringConstraints(pattern="^[.a-zA-Z0-9-]+$")]

# Reusable annotation metadata for Annotated types

# This disallows strings of length 0 after stripping.
# Example usage: Annotated[T, StringRequiredConstraint()]
RequiredStringConstraint = lambda: StringConstraints(strip_whitespace=True, min_length=1)

# Implementation


class _NotSpecified:
    pass


NotSpecified = _NotSpecified()


# For performance reasons, attributes needed from ApiParamConfig are copied to
# FuncParam. We should use slotted dataclass once the entire codebase is
# switched to Python 3.10+
@dataclass(frozen=True)
class FuncParam(Generic[T]):
    # Default value of the parameter.
    default: T | _NotSpecified
    # Name of the function parameter as defined in the original function.
    param_name: str
    # Inspected the underlying type of the parameter by unwrapping the Annotated
    # type if there is one.
    param_type: type[T]
    # The Pydantic TypeAdapter used to parse arbitrary input to the desired type.
    # We store it on the FuncParam object as soon as the view function is
    # decorated because it is expensive to construct.
    # See also: https://docs.pydantic.dev/latest/usage/type_adapter/
    type_adapter: TypeAdapter[T]

    # The following group of attributes are computed from the ApiParamConfig
    # annotation associated with this param:
    # Name of the corresponding variable in the request data to look
    # for. When argument_type_is_body is True, this is set to "request".
    aliases: tuple[str, ...]
    argument_type_is_body: bool
    documentation_status: DocumentationStatus
    path_only: bool
    request_var_name: str


@dataclass(frozen=True)
class ViewFuncInfo:
    view_func_full_name: str
    parameters: Sequence[FuncParam[object]]


def is_annotated(type_annotation: type[object]) -> bool:
    origin = get_origin(type_annotation)
    return origin is Annotated


def is_optional(type_annotation: type[object]) -> bool:
    origin = get_origin(type_annotation)
    type_args = get_args(type_annotation)
    return origin in (Union, types.UnionType) and type(None) in type_args and len(type_args) == 2


API_PARAM_CONFIG_USAGE_HINT = f"""
    Detected incorrect usage of Annotated types for parameter {{param_name}}!
    Check the placement of the {ApiParamConfig.__name__} object in the type annotation:

    {{param_name}}: {{param_type}}

    The Annotated[T, ...] type annotation containing the
    {ApiParamConfig.__name__} object should not be nested inside another type.

    Correct examples:

    # Using Optional inside Annotated
    param: Annotated[Optional[int], ApiParamConfig(...)]
    param: Annotated[Optional[int], ApiParamConfig(...)]] = None

    # Not using Optional when the default is not None
    param: Annotated[int, ApiParamConfig(...)]

    Incorrect examples:

    # Nesting Annotated inside Optional
    param: Optional[Annotated[int, ApiParamConfig(...)]]
    param: Optional[Annotated[int, ApiParamConfig(...)]] = None

    # Nesting the Annotated type carrying ApiParamConfig inside other types like Union
    param: Union[str, Annotated[int, ApiParamConfig(...)]]
"""


def parse_single_parameter(
    param_name: str, param_type: type[T], parameter: inspect.Parameter
) -> FuncParam[T]:
    param_default = parameter.default
    # inspect._empty is the internal type used by inspect to indicate not
    # specified defaults.
    if param_default is inspect._empty:
        param_default = NotSpecified

    # Defaulting a value to None automatically wraps the type annotation with
    # Optional. We explicitly unwrap it for the case of Annotated, which
    # otherwise causes undesired behaviors that the annotated metadata gets
    # lost. This is fixed in Python 3.11:
    # https://github.com/python/cpython/issues/90353
    if param_default is None and is_optional(param_type):
        type_args = get_args(param_type)
        inner_type = type_args[0] if type_args[1] is type(None) else type_args[1]
        if is_annotated(inner_type):
            annotated_type, *annotations = get_args(inner_type)
            has_api_param_config = any(
                isinstance(annotation, ApiParamConfig) for annotation in annotations
            )
            # This prohibits the use of `Optional[Annotated[T, ApiParamConfig(...)]] = None`
            # and encourages `Annotated[Optional[T], ApiParamConfig(...)] = None`
            # to avoid confusion when the parameter metadata is unintentionally nested.
            assert not has_api_param_config or is_optional(annotated_type), (
                API_PARAM_CONFIG_USAGE_HINT.format(param_name=param_name, param_type=param_type)
            )
            param_type = inner_type

    param_config: ApiParamConfig | None = None
    if is_annotated(param_type):
        # The first type is the underlying type of the parameter, the rest are
        # metadata attached to Annotated. Note that we do not transform
        # param_type to its underlying type because the Annotated metadata might
        # still be needed by other parties like Pydantic.
        _type, *annotations = get_args(param_type)
        for annotation in annotations:
            if not isinstance(annotation, ApiParamConfig):
                continue
            assert param_config is None, "ApiParamConfig can only be defined once per parameter"
            param_config = annotation
    else:
        # When no parameter configuration is found, assert that there is none
        # nested somewhere in a Union type to avoid silently ignoring it. If it
        # does present in the stringified parameter type, it is very likely a
        # programming error.
        assert ApiParamConfig.__name__ not in str(param_type), API_PARAM_CONFIG_USAGE_HINT.format(
            param_name=param_name, param_type=param_type
        )
    # If param_config is still None at this point, we could not find an instance
    # of it in the type annotation of the function parameter. In this case, we
    # fallback to the defaults by constructing ApiParamConfig here.
    # This is common for simple parameters of type str, Json[int] and etc.
    if param_config is None:
        param_config = ApiParamConfig()

    # Metadata defines a validator making sure that argument_type_is_body is
    # incompatible with whence.
    if param_config.argument_type_is_body:
        request_var_name = "request"
    else:
        request_var_name = param_config.whence if param_config.whence is not None else param_name

    return FuncParam(
        default=param_default,
        param_name=param_name,
        param_type=param_type,
        type_adapter=TypeAdapter(param_type),
        aliases=param_config.aliases,
        argument_type_is_body=param_config.argument_type_is_body,
        documentation_status=param_config.documentation_status,
        path_only=param_config.path_only,
        request_var_name=request_var_name,
    )


def parse_view_func_signature(
    view_func: Callable[Concatenate[HttpRequest, ParamT], object],
) -> ViewFuncInfo:
    """This is responsible for inspecting the function signature and getting the
    metadata from the parameters. We want to keep this function as pure as
    possible not leaking side effects to the global state. Side effects should
    be executed separately after the ViewFuncInfo is returned.
    """
    type_hints = get_type_hints(view_func, include_extras=True)
    parameters = inspect.signature(view_func).parameters
    view_func_full_name = f"{view_func.__module__}.{view_func.__name__}"

    process_parameters: list[FuncParam[object]] = []

    for param_name, parameter in parameters.items():
        assert param_name in type_hints
        if parameter.kind != inspect.Parameter.KEYWORD_ONLY:
            continue
        param_info = parse_single_parameter(
            param_name=param_name, param_type=type_hints[param_name], parameter=parameter
        )
        process_parameters.append(param_info)

    return ViewFuncInfo(
        view_func_full_name=view_func_full_name,
        parameters=process_parameters,
    )


# TODO: To get coverage data, we should switch to match-case syntax when we
# upgrade to Python 3.10.
# This should be sorted alphabetically.
ERROR_TEMPLATES = {
    "bool_parsing": _("{var_name} is not a boolean"),
    "bool_type": _("{var_name} is not a boolean"),
    "dataclass_type": _("{var_name} does not have the expected format"),
    "datetime_parsing": _("{var_name} is not a date"),
    "datetime_type": _("{var_name} is not a date"),
    "dict_type": _("{var_name} is not a dict"),
    "enum": _("Invalid {var_name}"),
    "extra_forbidden": _('Argument "{argument}" at {var_name} is unexpected'),
    "float_parsing": _("{var_name} is not a float"),
    "float_type": _("{var_name} is not a float"),
    "greater_than": _("{var_name} is too small"),
    "greater_than_equal": _("{var_name} is too small"),
    "int_parsing": _("{var_name} is not an integer"),
    "int_type": _("{var_name} is not an integer"),
    "json_invalid": _("{var_name} is not valid JSON"),
    "json_type": _("{var_name} is not valid JSON"),
    "less_than": _("{var_name} is too large"),
    "list_type": _("{var_name} is not a list"),
    "literal_error": _("Invalid {var_name}"),
    "string_too_long": _("{var_name} is too long (limit: {max_length} characters)"),
    "string_too_short": _("{var_name} is too short."),
    "string_type": _("{var_name} is not a string"),
    "unexpected_keyword_argument": _('Argument "{argument}" at {var_name} is unexpected'),
    "string_pattern_mismatch": _("{var_name} has invalid format"),
    "string_fixed_length": _("{var_name} is not length {length}"),
    "too_long": _("{var_name} is too long (limit: {max_length} items)"),
    "too_short": _("{var_name} is too short (minimum {min_length} items)"),
}


def parse_value_for_parameter(parameter: FuncParam[T], value: object) -> T:
    try:
        return parameter.type_adapter.validate_python(value, strict=True)
    except ValidationError as exc:
        # If the validation fails, it is possible to get multiple errors from
        # Pydantic. We only send the first error back to the client.
        # See also on ValidationError:
        # https://docs.pydantic.dev/latest/errors/validation_errors/
        error = exc.errors()[0]
        # We require all Pydantic raised error types that we expect to be
        # explicitly handled here. The end result should either be a 400
        # error with a translated message or an internal server error.
        error_template = ERROR_TEMPLATES.get(error["type"])
        var_name = parameter.request_var_name + "".join(
            f"[{json.dumps(loc)}]" for loc in error["loc"]
        )
        context = {
            "var_name": var_name,
            **error.get("ctx", {}),
        }

        if error["type"] == "json_invalid" and parameter.argument_type_is_body:
            # argument_type_is_body is usually used by webhooks that do not
            # require a specific var_name for payload JSON decoding error.
            # We override it here.
            error_template = _("Malformed JSON")
        elif error["type"] in ("unexpected_keyword_argument", "extra_forbidden"):
            context["argument"] = error["loc"][-1]
        # This condition matches our StringRequiredConstraint
        elif error["type"] == "string_too_short" and error["ctx"].get("min_length") == 1:
            error_template = _("{var_name} cannot be blank")
        elif error["type"] == "value_error":
            context["msg"] = error["msg"]
            error_template = _("Invalid {var_name}: {msg}")
        elif error["type"] == "model_type":
            context["msg"] = error["msg"]
            error_template = _("Invalid {var_name}: {msg}")
        elif error["type"] == "missing":
            context["msg"] = error["msg"]
            error_template = _("{var_name} field is missing: {msg}")
        assert error_template is not None, MISSING_ERROR_TEMPLATE.format(
            error_type=error["type"],
            url=error.get("url", "(documentation unavailable)"),
            error=json.dumps(error, indent=4),
        )
        raise ApiParamValidationError(error_template.format(**context), error["type"])


MISSING_ERROR_TEMPLATE = f"""
    Pydantic validation error of type "{{error_type}}" does not have the
    corresponding error message template or is not handled explicitly. We expect
    that every validation error is formatted into a client-facing error message.
    Consider adding this type to {__package__}.ERROR_TEMPLATES with the appropriate
    internationalized error message or handle it in {__package__}.{parse_value_for_parameter.__name__}.

    Documentation for "{{error_type}}" can be found at {{url}}.

    Error information:
{{error}}
"""


UNEXPECTEDLY_MISSING_KEYWORD_ONLY_PARAMETERS = """
Parameters expected to be parsed from the request should be defined as
keyword-only parameters, but there is no keyword-only parameter found in
{view_func_name}.

Example usage:

```
@typed_endpoint
def view(
    request: HttpRequest,
    *,
    flag_value: Annotated[Json[bool], ApiParamConfig(
        whence="flag", documentation_status=INTENTIONALLY_UNDOCUMENTED,
    )]
) -> HttpResponse:
    ...
```

This is likely a programming error. See https://peps.python.org/pep-3102/ for details on how
to correctly declare your parameters as keyword-only parameters.
Endpoints that do not accept parameters should use @typed_endpoint_without_parameters.
"""

UNEXPECTED_KEYWORD_ONLY_PARAMETERS = """
Unexpected keyword-only parameters found in {view_func_name}.
keyword-only parameters are treated as parameters to be parsed from the request,
but @typed_endpoint_without_parameters does not expect any.

Use @typed_endpoint instead.
"""


def typed_endpoint_without_parameters(
    view_func: Callable[Concatenate[HttpRequest, ParamT], ReturnT],
) -> Callable[Concatenate[HttpRequest, ParamT], ReturnT]:
    return typed_endpoint(view_func, expect_no_parameters=True)


def typed_endpoint(
    view_func: Callable[Concatenate[HttpRequest, ParamT], ReturnT],
    *,
    expect_no_parameters: bool = False,
) -> Callable[Concatenate[HttpRequest, ParamT], ReturnT]:
    # Extract all the type information from the view function.
    endpoint_info = parse_view_func_signature(view_func)
    if expect_no_parameters:
        assert len(endpoint_info.parameters) == 0, UNEXPECTED_KEYWORD_ONLY_PARAMETERS.format(
            view_func_name=endpoint_info.view_func_full_name
        )
    else:
        assert len(endpoint_info.parameters) != 0, (
            UNEXPECTEDLY_MISSING_KEYWORD_ONLY_PARAMETERS.format(
                view_func_name=endpoint_info.view_func_full_name
            )
        )
    for func_param in endpoint_info.parameters:
        if func_param.path_only:
            assert func_param.default is NotSpecified, (
                f"Path-only parameter {func_param.param_name} should not have a default value"
            )
        # Record arguments that should be documented so that our
        # automated OpenAPI docs tests can compare these against the code.
        if (
            func_param.documentation_status is DocumentationStatus.DOCUMENTED
            and not func_param.path_only
        ):
            # TODO: Move arguments_map to here once zerver.lib.request does not
            # need it anymore.
            arguments_map[endpoint_info.view_func_full_name].append(func_param.request_var_name)

    @wraps(view_func)
    def _wrapped_view_func(
        request: HttpRequest, /, *args: ParamT.args, **kwargs: ParamT.kwargs
    ) -> ReturnT:
        request_notes = RequestNotes.get_notes(request)
        for parameter in endpoint_info.parameters:
            if parameter.path_only:
                # For path_only parameters, they should already have been passed via
                # the URL, so there's no need for us to do anything.
                #
                # TODO: Run validators for path_only parameters for NewType.
                assert parameter.param_name in kwargs, (
                    f"Path-only variable {parameter.param_name} should be passed already"
                )
            if parameter.param_name in kwargs:
                # Skip parameters that are already supplied by the caller.
                continue

            # Extract the value to parse from the request body if specified.
            if parameter.argument_type_is_body:
                try:
                    request_notes.processed_parameters.add(parameter.request_var_name)
                    kwargs[parameter.param_name] = parse_value_for_parameter(
                        parameter, request.body.decode(request.encoding or "utf-8")
                    )
                except UnicodeDecodeError:
                    raise JsonableError(_("Malformed payload"))
                # test_typed_endpoint.TestEndpoint.test_argument_type has
                # coverage of this, but coverage.py fails to recognize it for
                # some reason.
                continue  # nocoverage

            # Otherwise, try to find the matching request variable in one of the QueryDicts
            # This is a view bug, not a user error, and thus should throw a 500.
            possible_aliases = [parameter.request_var_name, *parameter.aliases]
            alias_used = None
            value_to_parse = None

            for current_alias in possible_aliases:
                if current_alias in request.POST:
                    value_to_parse = request.POST[current_alias]
                elif current_alias in request.GET:
                    value_to_parse = request.GET[current_alias]
                else:
                    # This is covered by
                    # test_typed_endpoint.TestEndpoint.test_aliases, but
                    # coverage.py fails to recognize this for some reason.
                    continue  # nocoverage
                if alias_used is not None:
                    raise RequestConfusingParamsError(alias_used, current_alias)
                alias_used = current_alias

            if alias_used is None:
                alias_used = parameter.request_var_name
                if parameter.default is NotSpecified:
                    raise RequestVariableMissingError(alias_used)
                # By skipping here, we leave it to Python to use the default value
                # of this parameter, because we cannot find the request variable in
                # the request.
                # This is tested test_typed_endpoint.TestEndpoint.test_json, but
                # coverage.py fails to recognize this for some reason.
                continue  # nocoverage

            # Note that value_to_parse comes from a QueryDict, so it has no chance
            # of having a user-provided None value.
            assert value_to_parse is not None
            request_notes.processed_parameters.add(alias_used)
            kwargs[parameter.param_name] = parse_value_for_parameter(parameter, value_to_parse)
        return_value = view_func(request, *args, **kwargs)

        if (
            isinstance(return_value, MutableJsonResponse)
            # TODO: Move is_webhook_view to the decorator
            and not request_notes.is_webhook_view
            # Implemented only for 200 responses.
            # TODO: Implement returning unsupported ignored parameters for 400
            # JSON error responses. This is complex because typed_endpoint can be
            # called multiple times, so when an error response is raised, there
            # may be supported parameters that have not yet been processed,
            # which could lead to inaccurate output.
            and 200 <= return_value.status_code < 300
        ):
            ignored_parameters = {*request.POST, *request.GET}.difference(
                request_notes.processed_parameters
            )

            # This will be called each time a function decorated with @typed_endpoint
            # returns a MutableJsonResponse with a success status_code. Because
            # a shared processed_parameters value is checked each time, the
            # value for the ignored_parameters_unsupported key is either
            # added/updated to the response data or it is removed in the case
            # that all of the request parameters have been processed.
            if ignored_parameters:
                return_value.get_data()["ignored_parameters_unsupported"] = sorted(
                    ignored_parameters
                )
            else:
                return_value.get_data().pop("ignored_parameters_unsupported", None)

        return return_value

    return _wrapped_view_func