The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]
Python ai-gatewayanthropicazure-openaibedrockgatewaylangchainlitellmllmllm-gatewayllmopsmcp-gatewayopenaiopenai-proxyrustrust-aivertex-ai
import React, { useEffect, useRef, useState } from "react";
import ReactMarkdown from "react-markdown";
import { MessageSquare, Sparkles, X } from "lucide-react";
import { Button } from "@/components/ui/button";
import {
Combobox,
ComboboxContent,
ComboboxEmpty,
ComboboxInput,
ComboboxItem,
ComboboxList,
} from "@/components/ui/combobox";
import { Textarea } from "@/components/ui/textarea";
import { UiLoadingSpinner } from "@/components/ui/ui-loading-spinner";
import { modelHubCall, usageAiChatStream, UsageAiToolCallEvent } from "@/components/networking";
interface ToolCallStep {
tool_name: string;
tool_label: string;
arguments: Record<string, string>;
status: "running" | "complete" | "error";
error?: string;
}
interface ChatMessage {
role: "user" | "assistant";
content: string;
toolCalls?: ToolCallStep[];
}
interface UsageAIChatPanelProps {
open: boolean;
onClose: () => void;
accessToken: string | null;
}
const TOOL_ICONS: Record<string, string> = {
get_usage_data: "📊",
get_team_usage_data: "👥",
get_tag_usage_data: "🏷️",
};
const ToolCallDisplay: React.FC<{ step: ToolCallStep }> = ({ step }) => {
const icon = TOOL_ICONS[step.tool_name] || "🔧";
const args = step.arguments;
const dateRange = args.start_date && args.end_date ? `${args.start_date} → ${args.end_date}` : "";
const filter = args.team_ids || args.tags || args.user_id || "";
return (
<div className="flex items-start gap-2 px-3 py-2 rounded-lg bg-muted border border-border text-xs">
<span className="shrink-0 mt-0.5">
{step.status === "running" ? (
<UiLoadingSpinner className="size-3.5" />
) : step.status === "error" ? (
<span className="text-destructive">✗</span>
) : (
<span className="text-success">✓</span>
)}
</span>
<div className="min-w-0">
<div className="font-medium text-foreground">
{icon} {step.tool_label}
</div>
{dateRange && <div className="text-muted-foreground mt-0.5">{dateRange}</div>}
{filter && <div className="text-muted-foreground mt-0.5">Filter: {filter}</div>}
{step.status === "error" && step.error && <div className="text-destructive mt-0.5">{step.error}</div>}
</div>
</div>
);
};
const MarkdownContent: React.FC<{ content: string }> = ({ content }) => (
<ReactMarkdown
components={{
p: ({ children }) => <p className="mb-2 last:mb-0">{children}</p>,
strong: ({ children }) => <strong className="font-semibold">{children}</strong>,
ul: ({ children }) => <ul className="list-disc pl-4 mb-2 space-y-0.5">{children}</ul>,
ol: ({ children }) => <ol className="list-decimal pl-4 mb-2 space-y-0.5">{children}</ol>,
li: ({ children }) => <li>{children}</li>,
h1: ({ children }) => <h4 className="font-semibold text-sm mt-2 mb-1">{children}</h4>,
h2: ({ children }) => <h4 className="font-semibold text-sm mt-2 mb-1">{children}</h4>,
h3: ({ children }) => <h4 className="font-semibold text-sm mt-2 mb-1">{children}</h4>,
code: ({ children, className }) => {
const isBlock = className?.includes("language-");
return isBlock ? (
<pre className="bg-muted rounded-sm p-2 my-1 overflow-x-auto text-xs">
<code>{children}</code>
</pre>
) : (
<code className="px-1 py-0.5 rounded-sm bg-muted text-xs font-mono">{children}</code>
);
},
table: ({ children }) => (
<div className="overflow-x-auto my-2">
<table className="text-xs border-collapse w-full">{children}</table>
</div>
),
th: ({ children }) => (
<th className="border border-border px-2 py-1 bg-muted font-medium text-left">{children}</th>
),
td: ({ children }) => <td className="border border-border px-2 py-1">{children}</td>,
}}
>
{content}
</ReactMarkdown>
);
const UsageAIChatPanel: React.FC<UsageAIChatPanelProps> = ({ open, onClose, accessToken }) => {
const [messages, setMessages] = useState<ChatMessage[]>([]);
const [inputText, setInputText] = useState("");
const [isLoading, setIsLoading] = useState(false);
const [selectedModel, setSelectedModel] = useState<string | undefined>(undefined);
const [availableModels, setAvailableModels] = useState<string[]>([]);
const [isLoadingModels, setIsLoadingModels] = useState(false);
const [streamingContent, setStreamingContent] = useState("");
const [statusMessage, setStatusMessage] = useState<string | null>(null);
const [activeToolCalls, setActiveToolCalls] = useState<ToolCallStep[]>([]);
const messagesEndRef = useRef<HTMLDivElement>(null);
const abortControllerRef = useRef<AbortController | null>(null);
useEffect(() => {
if (open && availableModels.length === 0) {
loadModels();
}
}, [open]);
useEffect(() => {
if (typeof messagesEndRef.current?.scrollIntoView === "function") {
messagesEndRef.current.scrollIntoView({ behavior: "smooth" });
}
}, [messages, streamingContent, activeToolCalls, statusMessage]);
const loadModels = async () => {
if (!accessToken) return;
setIsLoadingModels(true);
try {
const fetchedModels = await modelHubCall(accessToken);
if (fetchedModels?.data?.length > 0) {
const models = fetchedModels.data.map((item: any) => item.model_group as string).sort();
setAvailableModels(models);
}
} catch (error) {
console.error("Failed to load models:", error);
} finally {
setIsLoadingModels(false);
}
};
const handleSend = async () => {
if (!accessToken || !inputText.trim() || isLoading) return;
const userMessage: ChatMessage = { role: "user", content: inputText.trim() };
const updatedMessages = [...messages, userMessage];
setMessages(updatedMessages);
setInputText("");
setIsLoading(true);
setStreamingContent("");
setStatusMessage(null);
setActiveToolCalls([]);
const abortController = new AbortController();
abortControllerRef.current = abortController;
let accumulated = "";
const toolCalls: ToolCallStep[] = [];
try {
await usageAiChatStream(
accessToken,
updatedMessages.slice(-20).map((m) => ({ role: m.role, content: m.content })),
selectedModel || "",
(content: string) => {
setStatusMessage(null);
accumulated += content;
setStreamingContent(accumulated);
},
() => {
setStatusMessage(null);
setActiveToolCalls([]);
setMessages((prev) => [
...prev,
{ role: "assistant", content: accumulated, toolCalls: toolCalls.length > 0 ? [...toolCalls] : undefined },
]);
setStreamingContent("");
},
(errorMsg: string) => {
setStatusMessage(null);
setActiveToolCalls([]);
setMessages((prev) => [...prev, { role: "assistant", content: `Error: ${errorMsg}` }]);
setStreamingContent("");
},
(status: string) => {
setStatusMessage(status);
},
(event: UsageAiToolCallEvent) => {
const idx = toolCalls.findIndex((tc) => tc.tool_name === event.tool_name);
if (idx >= 0) {
toolCalls[idx] = { ...event };
} else {
toolCalls.push({ ...event });
}
setActiveToolCalls([...toolCalls]);
},
abortController.signal,
);
} catch (error: any) {
if (error?.name === "AbortError" || abortController.signal.aborted) {
return;
}
const errorMsg = error?.message || "Failed to get response. Please try again.";
setMessages((prev) => [...prev, { role: "assistant", content: `Error: ${errorMsg}` }]);
setStreamingContent("");
} finally {
setIsLoading(false);
abortControllerRef.current = null;
}
};
const handleKeyDown = (e: React.KeyboardEvent<HTMLTextAreaElement>) => {
if (e.key === "Enter" && !e.shiftKey) {
e.preventDefault();
handleSend();
}
};
const handleClose = () => {
if (abortControllerRef.current) {
abortControllerRef.current.abort();
}
onClose();
};
const handleClear = () => {
setMessages([]);
setStreamingContent("");
setActiveToolCalls([]);
setStatusMessage(null);
};
return (
<div
data-testid="usage-ai-chat-panel"
className={`fixed top-0 right-0 h-full bg-card border-l border-border shadow-2xl z-overlay flex flex-col transition-transform duration-300 ease-in-out ${
open ? "translate-x-0" : "translate-x-full"
}`}
style={{ width: 420 }}
>
{/* Header */}
<div className="flex shrink-0 items-start justify-between gap-3 border-b border-border px-5 pt-4 pb-3">
<div className="min-w-0">
<h3 className="flex items-center gap-2 text-sm leading-5 font-medium text-foreground">
<Sparkles aria-hidden="true" className="size-4 shrink-0 text-muted-foreground" strokeWidth={1.75} />
Ask AI
</h3>
<p className="mt-0.5 text-xs text-muted-foreground">Ask about your spend, models, keys, and trends</p>
</div>
<Button variant="ghost" size="icon-sm" aria-label="Close" onClick={handleClose}>
<X />
</Button>
</div>
{/* Model selector */}
<div className="px-5 py-3 border-b border-border shrink-0">
<Combobox
items={availableModels}
value={selectedModel ?? null}
onValueChange={(value: string | null) => setSelectedModel(value ?? undefined)}
>
<ComboboxInput
className="w-full"
placeholder="Select a model (optional, defaults to gpt-4o-mini)"
aria-label="Select a model (optional, defaults to gpt-4o-mini)"
aria-busy={isLoadingModels}
showClear={selectedModel !== undefined}
/>
<ComboboxContent>
<ComboboxEmpty>{isLoadingModels ? "Loading models…" : "No models found"}</ComboboxEmpty>
<ComboboxList>
{(model: string) => (
<ComboboxItem key={model} value={model}>
{model}
</ComboboxItem>
)}
</ComboboxList>
</ComboboxContent>
</Combobox>
</div>
{/* Chat messages */}
<div className="flex-1 overflow-y-auto px-5 py-4 space-y-3 bg-muted/40">
{messages.length === 0 && !streamingContent && !isLoading && (
<div className="flex flex-col items-center justify-center h-full text-muted-foreground">
<MessageSquare aria-hidden="true" className="mb-2 size-6" strokeWidth={1.5} />
<p className="text-sm font-medium text-foreground">Ask a question about your usage</p>
<p className="text-xs mt-1">e.g. "Which model costs me the most?"</p>
</div>
)}
{messages.map((msg, idx) => (
<div key={idx}>
{msg.role === "user" ? (
<div className="flex justify-end">
<div className="max-w-[88%] rounded-xl px-3.5 py-2 text-sm leading-relaxed bg-info text-info-foreground">
{msg.content}
</div>
</div>
) : (
<div className="space-y-2">
{/* Tool calls for this message */}
{msg.toolCalls && msg.toolCalls.length > 0 && (
<div className="space-y-1.5">
{msg.toolCalls.map((tc, tcIdx) => (
<ToolCallDisplay key={tcIdx} step={tc} />
))}
</div>
)}
{/* Response */}
<div className="max-w-[95%] rounded-xl px-3.5 py-2.5 text-sm leading-relaxed bg-card border border-border text-foreground">
<MarkdownContent content={msg.content} />
</div>
</div>
)}
</div>
))}
{/* Active tool calls (in-progress) */}
{isLoading && activeToolCalls.length > 0 && (
<div className="space-y-1.5">
{activeToolCalls.map((tc, idx) => (
<ToolCallDisplay key={idx} step={tc} />
))}
</div>
)}
{/* Status / spinner */}
{isLoading && !streamingContent && (
<div className="flex items-center gap-2 px-3 py-2 text-xs text-muted-foreground">
<UiLoadingSpinner className="size-3.5" />
<span className="italic">{statusMessage || "Thinking..."}</span>
</div>
)}
{/* Streaming response */}
{streamingContent && (
<div className="max-w-[95%] rounded-xl px-3.5 py-2.5 text-sm leading-relaxed bg-card border border-border text-foreground">
<MarkdownContent content={streamingContent} />
</div>
)}
<div ref={messagesEndRef} />
</div>
{/* Input area */}
<div className="px-5 py-3 border-t border-border bg-card shrink-0">
<div className="flex gap-2">
<Textarea
value={inputText}
onChange={(e) => setInputText(e.target.value)}
onKeyDown={handleKeyDown}
placeholder="Ask about your usage..."
rows={1}
className="flex-1 min-h-9 max-h-24"
disabled={isLoading}
/>
<Button onClick={handleSend} disabled={!inputText.trim() || isLoading}>
{isLoading && <UiLoadingSpinner className="size-4" />}
Send
</Button>
</div>
<div className="flex justify-between items-center mt-2">
<button
onClick={handleClear}
className="text-xs text-muted-foreground hover:text-foreground transition-colors"
disabled={messages.length === 0}
>
Clear chat
</button>
<span className="text-xs text-muted-foreground">Enter to send</span>
</div>
</div>
</div>
);
};
export default UsageAIChatPanel;