Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

25.2 KB
Raw Blame
# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
# SPDX-License-Identifier: AGPL-3.0
"""HTTP routing, forwarding and response observation for one gateway request."""

import asyncio
import json
import logging
import time
import uuid
from dataclasses import replace
from typing import NamedTuple
from urllib.parse import quote, unquote_plus

import aiohttp
import orjson
from fastapi import HTTPException
from fastapi.responses import Response, StreamingResponse
from pydantic import ValidationError
from starlette.background import BackgroundTask

from .capture import CapturePipeline
from .models import KEY_PLACEHOLDER, Policy, validation_detail
from .protocols import HEARTBEAT_SECONDS, SSEDecoder, heartbeat, parse_body
from .storage import digest
from .tool_executor import ToolExecutor
from .tool_loop import HiddenToolLoop, ToolLoopError
from .tool_protocols import ResponseCapture, tool_protocol
from .tool_protocols.common import SummaryError
from .vendors import ARK_VENDORS, apply_vendor

logger = logging.getLogger(__name__)
HOP_HEADERS = {
    "connection",
    "keep-alive",
    "proxy-authenticate",
    "proxy-authorization",
    "te",
    "trailer",
    "transfer-encoding",
    "upgrade",
}
PATH_PROTOCOL = {
    "/v1/messages": "anthropic",
    "/v1/messages/count_tokens": "anthropic",
    "/v1/chat/completions": "chat",
    "/v1/responses": "responses",
}
PRIMARY_PATH = {
    "anthropic": "/v1/messages",
    "chat": "/v1/chat/completions",
    "responses": "/v1/responses",
}


def filtered_headers(headers, request=False):
    blocked = HOP_HEADERS | {
        part.strip().lower() for part in headers.get("connection", "").split(",")
    }
    if request:
        blocked |= {
            "host",
            "content-length",
            "authorization",
            "x-api-key",
            "cookie",
            "accept-encoding",
        }
    return {
        k: v
        for k, v in headers.items()
        if k.lower() not in blocked and not (request and k.lower().startswith("x-openviking-"))
    }


def query_names(pairs):
    return {unquote_plus(pair.partition("=")[0]) for pair in pairs}


def upstream_url(upstream, path, query=""):
    """The upstream address for a client path; ``{api_key}`` stays unfilled."""
    base, _, stored = upstream["base_url"].partition("?")
    base = base.rstrip("/")
    if not upstream.get("append_endpoint_path", True):
        primary = PRIMARY_PATH[upstream["protocol"]]
        if path != primary and not path.startswith(primary + "/"):
            raise HTTPException(404, f"This upstream is an exact endpoint for {primary} only")
        path = path[len(primary) :]
    elif upstream["protocol"] != "anthropic":
        path = path.removeprefix("/v1")
    pairs = [pair for pair in stored.split("&") if pair]
    names = query_names(pairs)
    pairs += [pair for pair in query.split("&") if pair and not query_names([pair]) & names]
    return base + path + ("?" + "&".join(pairs) if pairs else "")


def upstream_key(upstream, incoming):
    key = (
        upstream.get("api_key", "")
        if upstream.get("auth_mode") != "passthrough"
        else incoming.get("x-openviking-upstream-key", "")
    )
    if key.startswith("sk-ant-oat"):
        raise HTTPException(
            403, "Claude subscription OAuth credentials are not supported; use an API key"
        )
    if not key:
        raise HTTPException(401, "Upstream API key is missing")
    return key


def context_error(detail, status=400, reason="context_override_invalid"):
    """An ``X-OpenViking-Context`` error, in the management API's ``{reason, detail}`` shape."""
    return HTTPException(status, {"reason": reason, "detail": detail})


def context_override(policy, header):
    """The stored policy with the fields of an ``X-OpenViking-Context`` header applied."""
    if not policy.get("client_overrides", True):
        raise context_error(
            "This key's context profile does not accept X-OpenViking-Context.",
            403,
            "context_override_not_allowed",
        )
    try:
        override = json.loads(header)
    except (ValueError, RecursionError):
        raise context_error("X-OpenViking-Context is not valid JSON")
    if not isinstance(override, dict):
        raise context_error("X-OpenViking-Context must be a JSON object")
    # The profile's name and its switch are not per-conversation settings.
    if blocked := override.keys() & {"name", "client_overrides"}:
        raise context_error("X-OpenViking-Context cannot set " + ", ".join(sorted(blocked)))
    try:
        merged = Policy.model_validate({**policy, **override}).model_dump()
    except ValidationError as error:
        raise context_error(validation_detail(error))
    return {**policy, **merged}, sorted(override)


def fill_key(url, key):
    return url.replace(KEY_PLACEHOLDER, quote(key, safe=""))


def upstream_headers(upstream, incoming):
    headers = filtered_headers(incoming, request=True)
    headers.update(upstream.get("headers", {}))
    key = upstream_key(upstream, incoming)
    name = upstream.get("key_header", "authorization")
    # Drop configured copies of the auth headers set below.
    if KEY_PLACEHOLDER in upstream.get("base_url", ""):
        own, auth = {"authorization", "x-api-key", "api-key"}, {}
    elif name == "authorization":
        own, auth = {"authorization"}, {"Authorization": "Bearer " + key}
        # Anthropic-compatible services accept either header; some require Bearer.
        if upstream["protocol"] == "anthropic":
            own, auth = own | {"x-api-key"}, {**auth, "x-api-key": key}
    else:
        own, auth = {"authorization", name}, {name: key}
    headers = {k: v for k, v in headers.items() if k.lower() not in own}
    headers.update(auth)
    headers["accept-encoding"] = "identity"
    return headers


def check_coding_plan(upstream):
    if upstream.get("coding_plan") and not upstream.get("allow_coding_plan"):
        raise HTTPException(403, "Coding Plan upstreams are disabled; configure a model API key")


def matches(upstream, protocol, model):
    return (
        upstream.get("enabled", True)
        and (
            upstream["protocol"] == protocol
            if protocol
            else upstream.get("append_endpoint_path", True)
        )
        and (
            not model
            or not upstream.get("models")
            or model in upstream["models"]
            or model in upstream.get("aliases", {})
        )
    )


class Binding(NamedTuple):
    policy: dict | None
    # Disabled ones included; callers skip them.
    upstreams: list[dict]
    # Empty allows every model.
    models: list[str]


async def default_policy(management, config, account):
    """Use the account's default profile, or the deployment's when none is stored."""
    policy = await management.get(account, "policies", "default")
    return policy or config.default_profile.model_dump()


async def resolve_binding(management, config, credential):
    """The context profile, upstreams and models a credential's requests use."""
    account = credential["account"]
    upstreams = await management.list(account, "upstreams")
    if credential.get("source") == "openviking":
        # An OpenViking key stores no binding: it uses the account's defaults.
        return Binding(
            await default_policy(management, config, account),
            upstreams or [{**u.model_dump(), "id": i} for i, u in config.default_upstreams.items()],
            [],
        )
    return Binding(
        await management.get(account, "policies", credential["policy_id"]),
        [u for u in upstreams if u["id"] in credential["upstream_ids"]],
        credential["models"],
    )


class ProxyRequest:
    def __init__(self, app, request, path, credential):
        self.app, self.request, self.credential = app, request, credential
        self.config, self.store, self.management = (
            app.state.config,
            app.state.store,
            app.state.management,
        )
        self.path = "/" + path
        for prefix in ("/api/v3/", "/api/compatible/v1/"):
            if self.path.startswith(prefix):
                self.path = "/v1/" + self.path.removeprefix(prefix)
                break
        self.protocol = PATH_PROTOCOL.get(self.path)
        self.prepared = None
        self.metrics = {"request_id": uuid.uuid4().hex, "kind": "passthrough"}

    async def run(self):
        if self.request.headers.get("upgrade", "").lower() == "websocket":
            return Response(status_code=426, headers={"Upgrade": "HTTP/1.1"})
        raw = bytearray()
        async for chunk in self.request.stream():
            raw.extend(chunk)
            if len(raw) > self.config.max_body_bytes:
                raise HTTPException(413, "Request exceeds configured body limit")
        self.raw = bytes(raw)
        self.body = await asyncio.to_thread(parse_body, self.raw) if raw else None
        original = self.body
        await self.route()
        await self.prepare()
        if self.body != original:
            self.raw = await asyncio.to_thread(orjson.dumps, self.body)
        self.target = self.url(self.upstream)
        self.headers = upstream_headers(self.upstream, self.request.headers)
        if self.raw:
            self.headers.setdefault("content-type", "application/json")
        self.started = time.monotonic()
        try:
            self.response = await self.app.state.http.request(
                self.request.method,
                self.target,
                data=self.raw or None,
                headers=self.headers,
                timeout=aiohttp.ClientTimeout(total=self.config.upstream_timeout_seconds),
                allow_redirects=False,
            )
        except (aiohttp.ClientError, asyncio.TimeoutError):
            raise HTTPException(502, "Model upstream is unavailable")
        self.metrics.update(status=self.response.status, upstream_id=self.upstream["id"])
        self.capture = ResponseCapture(self.protocol or self.upstream["protocol"])
        # Only the tool loop rewrites a reply, so the recall notice goes through it too.
        if (
            self.prepared
            and (self.prepared.tools_active or self.prepared.reply_lead)
            and self.response.status < 300
            and not self.path.endswith("count_tokens")
        ):
            return await self.tool_response()
        return await self.observe_response()

    async def route(self):
        routing = self.body
        if routing is None and self.raw:
            # Unsafe numbers/duplicate keys can still be forwarded byte for byte;
            # the model ACL must also apply on this uncommon fallback path.
            try:
                routing = json.loads(self.raw)
            except (ValueError, RecursionError):
                pass
        self.model = routing.get("model", "") if isinstance(routing, dict) else ""
        binding = await resolve_binding(self.management, self.config, self.credential)
        # Calls on a stored response are pinned to its upstream and carry no model.
        if binding.models and self.raw and not self.path.startswith("/v1/responses/"):
            if self.request.headers.get("content-encoding", "identity").lower() != "identity":
                raise HTTPException(415, "Compressed request bodies are not supported")
            if not self.model or not isinstance(self.model, str):
                raise HTTPException(400, "Request model is missing")
        if binding.models and self.model and self.model not in binding.models:
            raise HTTPException(403, "Model is not allowed by this key")
        self.metrics["model"] = self.model
        self.policy = binding.policy
        # The header is checked on every request that carries it, enhanced or not.
        header = self.request.headers.get("x-openviking-context")
        if header is not None and self.policy:
            self.policy, self.metrics["context_override"] = context_override(self.policy, header)
        self.upstreams = binding.upstreams
        # A host in front of the gateway may pick the upstream, but only an allowed one.
        url = self.request.headers.get("x-openviking-upstream-url")
        if url is not None:
            self.upstreams = [u for u in self.upstreams if u["base_url"] == url.rstrip("/")]
            if not self.upstreams:
                raise HTTPException(403, "Upstream URL is not allowed")
        # A stored response is pinned below; an exact responses endpoint may hold it.
        stored_response = self.path.startswith("/v1/responses/")
        candidates = sorted(
            [
                u
                for u in self.upstreams
                if matches(u, self.protocol, self.model)
                or (stored_response and matches(u, "responses", self.model))
            ],
            key=lambda u: (-u["priority"], u["id"]),
        )
        if not candidates:
            raise HTTPException(404, "No allowed upstream matches this protocol and model")
        self.upstream = candidates[0]
        if self.path.startswith("/v1/responses/"):
            identifier = digest(self.credential["id"] + self.path.split("/")[3])
            mapping = await self.management.get(self.credential["account"], "responses", identifier)
            if not mapping:
                raise HTTPException(404, "Unknown response for this key")
            self.upstream = next(
                (u for u in self.upstreams if u["id"] == mapping["upstream_id"]), None
            )
            if not self.upstream or not self.upstream.get("enabled", True):
                raise HTTPException(403, "Response upstream is no longer allowed")
        # Pinned upstream selection in the kernel is restricted to allowed models.
        self.candidates = candidates
        # A missing client key fails before the request freezes a session or calls OpenViking.
        # A conversation may stay pinned to a candidate with its own key; url() checks that one.
        if all(u.get("auth_mode") == "passthrough" for u in candidates):
            upstream_key(self.upstream, self.request.headers)

    async def prepare(self):
        if (
            self.protocol
            and self.body
            and tool_protocol(self.protocol).enhanced_supported(self.body)
            and not self.request.headers.get("content-encoding")
        ):
            if not self.policy:
                raise HTTPException(403, "Context policy is unavailable")
            self.body, self.prepared, metrics = await self.app.state.kernel.enhance(
                self.body,
                self.protocol,
                dict(self.request.headers),
                self.credential,
                self.upstream,
                self.policy,
                self.path.endswith("count_tokens"),
                self.candidates,
                self.summarize,
            )
            self.metrics.update(metrics)
            if self.prepared:
                self.upstream = self.prepared.upstream
        elif self.raw and self.body is None:
            self.metrics["degradation"] = "unsafe_json"
        check_coding_plan(self.upstream)
        if self.body:
            self.body = await self.outgoing(self.body, self.upstream, self.prepared)
            if self.prepared:
                self.prepared.body = self.body
                self.metrics.update(self.prepared.metrics)

    def url(self, upstream):
        key = upstream_key(upstream, self.request.headers)
        filled = {**upstream, "base_url": fill_key(upstream["base_url"], key)}
        return upstream_url(filled, self.path, self.request.url.query)

    async def outgoing(self, body, upstream, prepared):
        """Vendor fields and the model alias, for the client request and its summary."""
        body = await apply_vendor(body, upstream, prepared, self.store)
        mapped = upstream.get("aliases", {}).get(self.model)
        return {**body, "model": mapped} if mapped else body

    async def summarize(self, prepared, body):
        """Send the kernel's summary request to the session's upstream and parse the reply."""
        upstream = prepared.upstream
        try:
            check_coding_plan(upstream)
            headers = upstream_headers(upstream, self.request.headers)
            target = self.url(upstream)
        except HTTPException as error:
            raise SummaryError(f"summary_http_{error.status_code}") from error
        headers.setdefault("content-type", "application/json")
        body = await self.outgoing(body, upstream, prepared)
        try:
            async with self.app.state.http.post(
                target,
                data=orjson.dumps(body),
                headers=headers,
                timeout=aiohttp.ClientTimeout(total=self.config.upstream_timeout_seconds),
                allow_redirects=False,
            ) as response:
                if response.status >= 300:
                    raise SummaryError(f"summary_http_{response.status}")
                return orjson.loads(await response.read())
        except (aiohttp.ClientError, asyncio.TimeoutError, ValueError) as error:
            raise SummaryError("summary_unavailable") from error

    async def relay(self):
        """Record a finished reply before the client receives its end."""
        if not (self.capture.finished and self.prepared and self.response.status < 300):
            return
        try:
            await self.app.state.kernel.relayed(self.prepared, self.capture)
        except Exception:
            logger.exception("OpenViking Gateway reply bookkeeping failed")

    async def finish(self):
        self.metrics.update(self.capture.usage or {})
        if self.upstream.get("vendor") in ARK_VENDORS:
            self.metrics["cache_min_tokens"] = self.upstream.get("cache_min_tokens", 1024)
            self.metrics["cache_eligible"] = (
                self.metrics.get("input_tokens", 0) >= self.metrics["cache_min_tokens"]
            )
        self.metrics["duration_ms"] = round((time.monotonic() - self.started) * 1000, 2)
        try:
            if (
                self.response.status < 300
                and self.prepared
                and await self.management.get(
                    self.credential["account"], "keys", self.credential["id"]
                )
            ):
                await self.app.state.kernel.completed(self.prepared, self.credential, self.capture)
            if (
                self.capture.response_id
                and self.path == "/v1/responses"
                and self.body
                and self.body.get("store") is not False
            ):
                await self.management.save(
                    self.credential["account"],
                    "responses",
                    digest(self.credential["id"] + self.capture.response_id),
                    {"upstream_id": self.upstream["id"], "user_id": self.credential["user_id"]},
                    ttl=self.config.response_ttl_seconds,
                )
            if self.prepared:
                # Delivery may have failed or recovered while this request was in flight.
                # A worker row inserted between this read and the log below can still be
                # older than this row; if the read fails, the prepare-time fields are kept.
                try:
                    capture = await self.store.capture.get(
                        self.prepared.scope, self.prepared.session
                    )
                    CapturePipeline.metrics(
                        replace(self.prepared, capture=capture, metrics=self.metrics),
                        Policy.model_validate(self.prepared.root["policy"]),
                    )
                except Exception:
                    logger.exception("OpenViking Gateway capture status refresh failed")
            await self.management.log(
                self.credential["account"], self.metrics, user_id=self.credential["user_id"]
            )
        except Exception:
            logger.exception("OpenViking Gateway response bookkeeping failed")

    async def tool_response(self):
        response, prepared, capture = self.response, self.prepared, self.capture
        executor = ToolExecutor(
            self.app.state.viking,
            self.store,
            prepared,
            self.credential,
            self.config.public_url,
            self.config.max_body_bytes,
            self.config.mcp_file_links,
        )
        loop = HiddenToolLoop(prepared, executor, self.store, capture)
        headers = {
            k.lower(): v
            for k, v in filtered_headers(response.headers).items()
            if k.lower() not in {"content-length", "etag", "content-encoding"}
        }

        async def send(payload, timeout):
            # Without a total tool deadline each continuation gets the usual upstream timeout.
            return await self.app.state.http.post(
                self.target,
                data=orjson.dumps(payload),
                headers=self.headers,
                timeout=aiohttp.ClientTimeout(
                    total=timeout or self.config.upstream_timeout_seconds
                ),
                allow_redirects=False,
            )

        async def stream():
            finished = False
            chunks = heartbeat(loop.run(response, send), HEARTBEAT_SECONDS)
            try:
                async for chunk in chunks:
                    if chunk is None:
                        yield b": keep-alive\n\n"
                        continue
                    await self.relay()
                    yield chunk
                finished = True
            except (ToolLoopError, aiohttp.ClientError) as error:
                self.metrics["degradation"] = "hidden_tool_loop_failed"
                yield loop.error(str(error))
            finally:
                await chunks.aclose()
                if not finished:
                    capture.complete = False
                self.metrics.update(prepared.metrics)

        if self.body.get("stream"):
            headers["content-type"] = "text/event-stream"
            return StreamingResponse(
                stream(), headers=headers, background=BackgroundTask(self.finish)
            )
        try:
            async for _ in loop.run(response, send):
                pass
        except (ToolLoopError, aiohttp.ClientError) as error:
            capture.complete = False
            self.metrics.update(
                degradation="hidden_tool_loop_failed", status=getattr(error, "status", 502)
            )
            await self.finish()
            if isinstance(error, ToolLoopError) and error.content is not None:
                return Response(
                    error.content, status_code=error.status, headers=filtered_headers(error.headers)
                )
            raise HTTPException(
                getattr(error, "status", 502), str(error), headers=getattr(error, "headers", None)
            )
        self.metrics.update(prepared.metrics)
        await self.relay()
        return Response(
            orjson.dumps(loop.final),
            headers=headers,
            media_type="application/json",
            background=BackgroundTask(self.finish),
        )

    async def observe_response(self):
        response, capture = self.response, self.capture
        headers = filtered_headers(response.headers)
        if "text/event-stream" in response.headers.get("content-type", ""):
            headers = {k: v for k, v in headers.items() if k.lower() != "content-length"}

            async def stream():
                decoder = SSEDecoder()
                observe = not response.headers.get("content-encoding")
                finished = False
                chunks = heartbeat(response.content.iter_any(), HEARTBEAT_SECONDS)
                try:
                    async for chunk in chunks:
                        if chunk is None:
                            if observe and not decoder.buffer:
                                yield b": keep-alive\n\n"
                            continue
                        if observe:
                            try:
                                # A chunk inside one event completes none: no reply to reread.
                                if frames := decoder.feed(chunk):
                                    capture.events(d for d in map(decoder.data, frames) if d)
                            except Exception:
                                observe = False
                                capture.complete = False
                                self.metrics["degradation"] = "capture_parse_failure"
                            await self.relay()
                        yield chunk
                    finished = True
                finally:
                    await chunks.aclose()
                    response.close()
                    if not finished:
                        capture.complete = False

            return StreamingResponse(
                stream(),
                status_code=response.status,
                headers=headers,
                background=BackgroundTask(self.finish),
            )
        try:
            content = await response.read()
            if response.status < 300 and not response.headers.get("content-encoding"):
                try:
                    capture.nonstream(orjson.loads(content))
                except (ValueError, TypeError, KeyError):
                    pass
        finally:
            response.close()
        await self.relay()
        return Response(
            content,
            status_code=response.status,
            headers=headers,
            background=BackgroundTask(self.finish),
        )