OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
feat: openrag status page (#2219)
hunterxtang1f47f9753860
diff --git a/frontend/app/api/queries/useComponentActions.ts b/frontend/app/api/queries/useComponentActions.tsnew file mode 100644index 000000000..edd44446e--- /dev/null+++ b/frontend/app/api/queries/useComponentActions.ts@@ -0,0 +1,100 @@+import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query";+import type {+ ComponentState,+ ComponentStatus,+ ConsoleStatusResponse,+} from "./useConsoleStatusQuery";++// ─── shared types ──────────────────────────────────────────────────────────++export interface ComponentActionResponse {+ component: string;+ ok: boolean;+ message: string;+ status: ComponentStatus;+}++export interface DiagnosisResponse {+ component: string;+ state: ComponentState;+ summary: string;+ likely_cause?: string | null;+ remediation: string[];+ last_error?: string | null;+ target?: string | null;+}++const SEVERITY: Record<ComponentState, number> = {+ healthy: 0,+ degraded: 1,+ unknown: 2,+ unhealthy: 3,+};++const worstOf = (states: ComponentState[]): ComponentState =>+ states.reduce<ComponentState>(+ (worst, s) => (SEVERITY[s] > SEVERITY[worst] ? s : worst),+ "healthy",+ );++// ─── sync mutation ───────────────────────────────────────────────────────────++/** Re-checks one component and patches the ["console-status"] cache with the+ * returned fresh status. */+export function useComponentSyncMutation() {+ const queryClient = useQueryClient();+ return useMutation({+ mutationFn: async (component: string): Promise<ComponentActionResponse> => {+ const res = await fetch(+ `/api/status/${encodeURIComponent(component)}/sync`,+ { method: "POST" },+ );+ const body = await res.json().catch(() => ({}));+ if (!res.ok) {+ throw new Error(+ typeof body?.detail === "string" ? body.detail : `HTTP ${res.status}`,+ );+ }+ return body as ComponentActionResponse;+ },+ onSuccess: (result) => {+ queryClient.setQueryData<ConsoleStatusResponse>(+ ["console-status"],+ (prev) => {+ if (!prev) return prev;+ const components = prev.components.map((c) =>+ c.name === result.component ? result.status : c,+ );+ return {+ ...prev,+ components,+ overall_status: worstOf(components.map((c) => c.status)),+ };+ },+ );+ queryClient.invalidateQueries({+ queryKey: ["component-diagnose", result.component],+ });+ queryClient.invalidateQueries({+ queryKey: ["component-logs", result.component],+ });+ },+ });+}++// ─── diagnose query (lazy — fires when the modal opens) ──────────────────────++export const useComponentDiagnoseQuery = (component: string | null) =>+ useQuery({+ queryKey: ["component-diagnose", component],+ queryFn: async () => {+ const res = await fetch(+ `/api/status/${encodeURIComponent(component as string)}/diagnose`,+ );+ if (!res.ok) throw new Error(`HTTP ${res.status}`);+ return res.json() as Promise<DiagnosisResponse>;+ },+ enabled: !!component,+ staleTime: 5000,+ refetchOnWindowFocus: false,+ });diff --git a/frontend/app/api/queries/useComponentLogsQuery.ts b/frontend/app/api/queries/useComponentLogsQuery.tsnew file mode 100644index 000000000..78c9812d2--- /dev/null+++ b/frontend/app/api/queries/useComponentLogsQuery.ts@@ -0,0 +1,60 @@+import {+ type UseQueryOptions,+ useQuery,+ useQueryClient,+} from "@tanstack/react-query";++export interface LogEntry {+ timestamp: string; // ISO-8601 UTC+ level: string; // "debug" | "info" | "warning" | "error" | "critical"+ message: string;+ detail?: string | null;+}++export interface ComponentLogsResponse {+ component: string;+ entries: LogEntry[];+ count: number;+}++async function fetchComponentLogs(+ component: string,+ tail = 100,+): Promise<ComponentLogsResponse> {+ const response = await fetch(+ `/api/status/${encodeURIComponent(component)}/logs?tail=${tail}`,+ );+ if (!response.ok) {+ const body = await response.json().catch(() => ({}));+ const detail =+ typeof body?.detail === "string"+ ? body.detail+ : `HTTP ${response.status}`;+ throw new Error(detail);+ }+ return response.json() as Promise<ComponentLogsResponse>;+}++export const useComponentLogsQuery = (+ component: string | null,+ tail = 100,+ options?: Omit<+ UseQueryOptions<ComponentLogsResponse>,+ "queryKey" | "queryFn"+ >,+) => {+ const queryClient = useQueryClient();++ return useQuery(+ {+ queryKey: ["component-logs", component, tail],+ queryFn: () => fetchComponentLogs(component as string, tail),+ enabled: !!component,+ retry: 1,+ staleTime: 5000,+ refetchOnWindowFocus: false,+ ...options,+ },+ queryClient,+ );+};diff --git a/frontend/app/api/queries/useConsoleStatusQuery.ts b/frontend/app/api/queries/useConsoleStatusQuery.tsnew file mode 100644index 000000000..6bdc009ca--- /dev/null+++ b/frontend/app/api/queries/useConsoleStatusQuery.ts@@ -0,0 +1,77 @@+import {+ type UseQueryOptions,+ useQuery,+ useQueryClient,+} from "@tanstack/react-query";++export type ComponentState = "healthy" | "degraded" | "unhealthy" | "unknown";++export interface ComponentBuild {+ git_sha?: string | null;+ build_time?: string | null;+ image?: string | null;+ image_digest?: string | null;+}++export interface ComponentStatus {+ name: string;+ display_name: string;+ status: ComponentState;+ required: boolean;+ latency_ms?: number | null;+ message?: string | null;+ version?: string | null;+ build?: ComponentBuild;+ metadata?: Record<string, unknown>;+ /** Non-null when the last health-check failed; used to gate the Logs button. */+ last_error?: string | null;+ /** ISO-8601 UTC of when this component was last checked (drives "Last Sync"). */+ checked_at?: string | null;+}++export interface ConsoleStatusResponse {+ overall_status: ComponentState;+ checked_at: string;+ components: ComponentStatus[];+}++async function fetchConsoleStatus(): Promise<ConsoleStatusResponse> {+ const response = await fetch("/api/status");+ if (!response.ok) {+ // Surface the real error (401 auth, 500 server, etc.) so the query+ // enters an error state instead of resolving to an empty components array.+ const body = await response.json().catch(() => ({}));+ const detail =+ typeof body?.detail === "string"+ ? body.detail+ : typeof body?.error === "string"+ ? body.error+ : `HTTP ${response.status}`;+ throw new Error(detail);+ }+ return response.json() as Promise<ConsoleStatusResponse>;+}++export const useConsoleStatusQuery = (+ options?: Omit<+ UseQueryOptions<ConsoleStatusResponse>,+ "queryKey" | "queryFn"+ >,+) => {+ const queryClient = useQueryClient();++ return useQuery(+ {+ queryKey: ["console-status"],+ queryFn: fetchConsoleStatus,+ retry: 1,+ refetchInterval: 30000,+ // Re-check when the user returns to the tab so a status change that+ // happened while away surfaces promptly (drives the header notification).+ refetchOnWindowFocus: true,+ staleTime: 15000,+ ...options,+ },+ queryClient,+ );+};diff --git a/frontend/app/layout.tsx b/frontend/app/layout.tsxindex 6aba2a655..f8ee6d6cb 100644--- a/frontend/app/layout.tsx+++ b/frontend/app/layout.tsx@@ -9,6 +9,7 @@ import { TooltipProvider } from "@/components/ui/tooltip";import { AuthProvider } from "@/contexts/auth-context";import { BrandProvider } from "@/contexts/brand-context";import { ChatProvider } from "@/contexts/chat-context";+import { ConsoleStatusProvider } from "@/contexts/console-status-context";import { KnowledgeFilterProvider } from "@/contexts/knowledge-filter-context";import { TaskProvider } from "@/contexts/task-context";import Providers from "./providers";@@ -61,9 +62,11 @@ export default function RootLayout({<BrandProvider><TaskProvider><KnowledgeFilterProvider>- <ChatProvider>- <LayoutWrapper>{children}</LayoutWrapper>- </ChatProvider>+ <ConsoleStatusProvider>+ <ChatProvider>+ <LayoutWrapper>{children}</LayoutWrapper>+ </ChatProvider>+ </ConsoleStatusProvider></KnowledgeFilterProvider></TaskProvider></BrandProvider>diff --git a/frontend/components/console-status-panel.tsx b/frontend/components/console-status-panel.tsxnew file mode 100644index 000000000..db302792f--- /dev/null+++ b/frontend/components/console-status-panel.tsx@@ -0,0 +1,824 @@+"use client";++import {+ AlertTriangle,+ CheckCircle2,+ ChevronDown,+ ChevronUp,+ EvCharger,+ HelpCircle,+ RefreshCw,+ ScrollText,+ TrendingUp,+ Wrench,+ XCircle,+} from "lucide-react";+import { type ReactNode, useCallback, useState } from "react";+import { toast } from "sonner";+import {+ type DiagnosisResponse,+ useComponentDiagnoseQuery,+ useComponentSyncMutation,+} from "@/app/api/queries/useComponentActions";+import {+ type LogEntry,+ useComponentLogsQuery,+} from "@/app/api/queries/useComponentLogsQuery";+import {+ type ComponentState,+ type ComponentStatus,+ useConsoleStatusQuery,+} from "@/app/api/queries/useConsoleStatusQuery";+import {+ type ProviderHealthResponse,+ useProviderHealthQuery,+} from "@/app/api/queries/useProviderHealthQuery";+import {+ Dialog,+ DialogContent,+ DialogHeader,+ DialogTitle,+} from "@/components/ui/dialog";+import { cn } from "@/lib/utils";++// ─── status helpers ──────────────────────────────────────────────────────────++function statusColor(status: ComponentState) {+ switch (status) {+ case "healthy":+ return "text-emerald-400";+ case "degraded":+ return "text-amber-400";+ case "unhealthy":+ return "text-red-400";+ default:+ return "text-zinc-400";+ }+}++function statusBadgeCls(status: ComponentState) {+ switch (status) {+ case "healthy":+ return "bg-emerald-500/15 text-emerald-400 border-emerald-500/30";+ case "degraded":+ return "bg-amber-500/15 text-amber-400 border-amber-500/30";+ case "unhealthy":+ return "bg-red-500/15 text-red-400 border-red-500/30";+ default:+ return "bg-zinc-500/15 text-zinc-400 border-zinc-500/30";+ }+}++function StatusIcon({+ status,+ size = 16,+}: {+ status: ComponentState;+ size?: number;+}) {+ switch (status) {+ case "healthy":+ return <CheckCircle2 size={size} className="text-emerald-400 shrink-0" />;+ case "degraded":+ return <AlertTriangle size={size} className="text-amber-400 shrink-0" />;+ case "unhealthy":+ return <XCircle size={size} className="text-red-400 shrink-0" />;+ default:+ return <HelpCircle size={size} className="text-zinc-400 shrink-0" />;+ }+}++// ─── summary placard ─────────────────────────────────────────────────────────++interface SummaryCardProps {+ label: string;+ count: number;+ state: ComponentState;+}++function SummaryCard({ label, count, state }: SummaryCardProps) {+ const colorClass = statusColor(state);+ return (+ <div className="flex-1 rounded-lg bg-zinc-800/60 border border-zinc-700/50 px-3 py-2.5">+ <p className="text-[11px] font-medium text-zinc-400 uppercase tracking-wide">+ {label}+ </p>+ <p className={cn("text-2xl font-semibold mt-0.5", colorClass)}>{count}</p>+ </div>+ );+}++// ─── metadata row ────────────────────────────────────────────────────────────++function MetaRow({ label, value }: { label: string; value: string }) {+ return (+ <div className="flex items-center justify-between gap-2 py-1">+ <span className="text-xs text-zinc-500 shrink-0">{label}</span>+ <span className="text-xs text-zinc-300 text-right truncate max-w-[60%]">+ {value}+ </span>+ </div>+ );+}++// ─── action button ────────────────────────────────────────────────────────────++/** Compact card action. Icon-only when `label` is omitted (uses `ariaLabel`). */+function ActionButton({+ icon,+ label,+ ariaLabel,+ disabled,+ onClick,+}: {+ icon: ReactNode;+ label?: string;+ ariaLabel?: string;+ disabled?: boolean;+ onClick: () => void;+}) {+ return (+ <button+ type="button"+ onClick={onClick}+ disabled={disabled}+ aria-label={ariaLabel ?? label}+ title={ariaLabel ?? label}+ className={cn(+ "flex items-center gap-1.5 rounded text-xs font-medium",+ label ? "px-2.5 py-1" : "p-1.5",+ "bg-zinc-700/60 border border-zinc-600/60 text-zinc-200",+ "hover:bg-zinc-600/60 hover:border-zinc-500/60 transition-colors",+ "disabled:opacity-50 disabled:cursor-not-allowed",+ )}+ >+ <span className="shrink-0">{icon}</span>+ {label}+ </button>+ );+}++/** "just now" / "5 minutes ago" from an ISO-8601 timestamp. */+function formatRelative(iso?: string | null): string {+ if (!iso) return "—";+ const secs = Math.round((Date.now() - new Date(iso).getTime()) / 1000);+ if (!Number.isFinite(secs) || secs < 0) return "just now";+ if (secs < 45) return "just now";+ const mins = Math.round(secs / 60);+ if (mins < 60) return `${mins} minute${mins === 1 ? "" : "s"} ago`;+ const hrs = Math.round(mins / 60);+ if (hrs < 24) return `${hrs} hour${hrs === 1 ? "" : "s"} ago`;+ const days = Math.round(hrs / 24);+ return `${days} day${days === 1 ? "" : "s"} ago`;+}++// ─── log level helpers ────────────────────────────────────────────────────────++function logLevelColor(level: string) {+ switch (level.toLowerCase()) {+ case "error":+ case "critical":+ return "text-red-400";+ case "warning":+ return "text-amber-400";+ case "info":+ return "text-sky-400";+ default:+ return "text-zinc-400";+ }+}++// ─── component logs modal ─────────────────────────────────────────────────────++interface ComponentLogsModalProps {+ component: string;+ displayName: string;+ open: boolean;+ onOpenChange: (open: boolean) => void;+}++function ComponentLogsModal({+ component,+ displayName,+ open,+ onOpenChange,+}: ComponentLogsModalProps) {+ const { data, isLoading, isError, error, refetch, isFetching } =+ useComponentLogsQuery(open ? component : null, 100);++ const entries: LogEntry[] = data?.entries ?? [];++ return (+ <Dialog open={open} onOpenChange={onOpenChange}>+ <DialogContent+ className={cn(+ "bg-zinc-900 border-zinc-700 text-zinc-100",+ "w-[560px] max-w-[95vw] max-h-[70vh] flex flex-col gap-0 p-0",+ )}+ >+ {/* Modal header */}+ <DialogHeader className="shrink-0 flex flex-row items-center justify-between pl-4 pr-10 py-3 border-b border-zinc-700/60">+ <div className="flex items-center gap-2">+ <ScrollText size={14} className="text-zinc-400" />+ <DialogTitle className="text-sm font-semibold text-zinc-100">+ {displayName} — Logs+ </DialogTitle>+ {isFetching && (+ <RefreshCw size={11} className="text-zinc-500 animate-spin" />+ )}+ </div>+ <button+ type="button"+ onClick={() => void refetch()}+ disabled={isFetching}+ className="text-xs text-zinc-500 hover:text-zinc-300 transition-colors disabled:opacity-40"+ >+ Refresh+ </button>+ </DialogHeader>++ {/* Log list */}+ <div className="flex-1 overflow-y-auto px-4 py-3 space-y-1.5 min-h-0">+ {isLoading ? (+ <div className="space-y-1.5">+ {[...Array(5)].map((_, i) => (+ <div+ // biome-ignore lint/suspicious/noArrayIndexKey: skeleton placeholders+ key={i}+ className="h-8 rounded bg-zinc-800/60 animate-pulse"+ />+ ))}+ </div>+ ) : isError ? (+ <p className="text-sm text-red-400">+ {error instanceof Error ? error.message : "Failed to load logs."}+ </p>+ ) : entries.length === 0 ? (+ <p className="text-xs text-zinc-500 italic text-center py-6">+ No log entries recorded yet.+ </p>+ ) : (+ entries.map((entry, idx) => (+ <div+ // biome-ignore lint/suspicious/noArrayIndexKey: log entries have no stable id+ key={`${entry.timestamp}-${entry.message}`}+ className="rounded bg-zinc-800/50 border border-zinc-700/40 px-2.5 py-1.5"+ >+ <div className="flex items-center gap-2 flex-wrap">+ <span+ className={cn(+ "text-[10px] font-semibold uppercase tabular-nums shrink-0",+ logLevelColor(entry.level),+ )}+ >+ {entry.level}+ </span>+ <span className="text-[10px] text-zinc-500 tabular-nums shrink-0">+ {new Date(entry.timestamp).toLocaleTimeString()}+ </span>+ <span className="text-xs text-zinc-200 break-all">+ {entry.message}+ </span>+ </div>+ {entry.detail && (+ <p className="text-[11px] text-zinc-400 mt-0.5 ml-0 break-all font-mono">+ {entry.detail}+ </p>+ )}+ </div>+ ))+ )}+ </div>++ {/* Footer */}+ <div className="shrink-0 px-4 py-2 border-t border-zinc-700/60">+ <span className="text-[11px] text-zinc-500">+ {entries.length} entr{entries.length === 1 ? "y" : "ies"} (most+ recent 100)+ </span>+ </div>+ </DialogContent>+ </Dialog>+ );+}++// ─── diagnostics modal ────────────────────────────────────────────────────────++interface ComponentDiagnoseModalProps {+ component: string;+ displayName: string;+ open: boolean;+ onOpenChange: (open: boolean) => void;+}++function ComponentDiagnoseModal({+ component,+ displayName,+ open,+ onOpenChange,+}: ComponentDiagnoseModalProps) {+ const { data, isLoading, isError, error } = useComponentDiagnoseQuery(+ open ? component : null,+ );+ const d: DiagnosisResponse | undefined = data;++ return (+ <Dialog open={open} onOpenChange={onOpenChange}>+ <DialogContent className="bg-zinc-900 border-zinc-700 text-zinc-100 w-[520px] max-w-[95vw]">+ <DialogHeader>+ <DialogTitle className="flex items-center gap-2 text-sm font-semibold">+ <Wrench size={14} className="text-zinc-400" />+ {displayName} — Debug+ </DialogTitle>+ </DialogHeader>++ {isLoading ? (+ <div className="h-24 rounded bg-zinc-800/60 animate-pulse" />+ ) : isError ? (+ <p className="text-sm text-red-400">+ {error instanceof Error ? error.message : "Failed to diagnose."}+ </p>+ ) : d ? (+ <div className="space-y-3 text-sm">+ <p className="text-zinc-100">{d.summary}</p>+ {d.likely_cause && (+ <div>+ <p className="text-[11px] uppercase tracking-wide text-zinc-500">+ Likely cause+ </p>+ <p className="text-zinc-300 mt-0.5">{d.likely_cause}</p>+ </div>+ )}+ {d.remediation.length > 0 && (+ <div>+ <p className="text-[11px] uppercase tracking-wide text-zinc-500">+ How to fix+ </p>+ <ul className="mt-1 space-y-1 list-disc list-inside text-zinc-300">+ {d.remediation.map((step) => (+ <li key={step}>{step}</li>+ ))}+ </ul>+ </div>+ )}+ {d.target && (+ <p className="text-[11px] text-zinc-500">+ Endpoint: <span className="font-mono">{d.target}</span>+ </p>+ )}+ {d.last_error && (+ <pre className="text-[11px] text-zinc-400 bg-zinc-800/60 rounded p-2 whitespace-pre-wrap break-all font-mono">+ {d.last_error}+ </pre>+ )}+ </div>+ ) : null}+ </DialogContent>+ </Dialog>+ );+}++// ─── component placard ───────────────────────────────────────────────────────++interface ComponentCardProps {+ component: ComponentStatus;+}++function ComponentCard({ component }: ComponentCardProps) {+ const [expanded, setExpanded] = useState(false);+ const [logsOpen, setLogsOpen] = useState(false);+ const [debugOpen, setDebugOpen] = useState(false);++ // The synthetic "providers" card is not a real component — no actions.+ const isRealComponent = component.name !== "providers";++ const syncMutation = useComponentSyncMutation();+ const syncing =+ syncMutation.isPending && syncMutation.variables === component.name;++ const {+ display_name,+ status,+ version,+ latency_ms,+ message,+ build,+ metadata,+ checked_at,+ } = component;++ const hasBuildDetails =+ build &&+ (build.git_sha || build.build_time || build.image || build.image_digest);+ const hasMetadata = metadata && Object.keys(metadata).length > 0;++ return (+ <div+ className={cn(+ "rounded-xl border bg-zinc-800/60 transition-colors",+ expanded ? "border-zinc-600/70" : "border-zinc-700/50",+ )}+ >+ {/* Header row — always clickable; chevron always visible */}+ <button+ type="button"+ className="w-full flex items-center gap-3 px-3.5 py-3 text-left"+ onClick={() => setExpanded((v) => !v)}+ aria-expanded={expanded}+ >+ <StatusIcon status={status} size={16} />++ <div className="flex-1 min-w-0">+ <div className="flex items-center gap-2 flex-wrap">+ <span className="text-sm font-medium text-zinc-100">+ {display_name}+ </span>+ <span+ className={cn(+ "inline-flex items-center px-1.5 py-0.5 rounded text-[10px] font-semibold border capitalize",+ statusBadgeCls(status),+ )}+ >+ {status}+ </span>+ {version && (+ <span className="text-[11px] text-zinc-500">• v{version}</span>+ )}+ </div>+ {message && (+ <p className="text-xs text-zinc-400 mt-0.5 truncate">{message}</p>+ )}+ </div>++ <div className="flex items-center gap-2 shrink-0">+ {latency_ms != null && (+ <span className="text-[11px] text-zinc-500 tabular-nums">+ {latency_ms}ms+ </span>+ )}+ <span className="text-zinc-500">+ {expanded ? <ChevronUp size={14} /> : <ChevronDown size={14} />}+ </span>+ </div>+ </button>++ {/* Expanded details */}+ {expanded && (+ <div className="border-t border-zinc-700/50 px-3.5 py-2.5 space-y-1">+ {isRealComponent && (+ <>+ <MetaRow label="Last Sync" value={formatRelative(checked_at)} />+ <div className="flex items-center justify-between gap-2 py-1">+ <span className="text-xs text-zinc-500 shrink-0">+ Response Time+ </span>+ <span className="flex items-center gap-1 text-xs text-zinc-300 tabular-nums">+ <TrendingUp size={12} className="text-emerald-400" />+ {latency_ms != null ? `${latency_ms}ms` : "—"}+ </span>+ </div>+ </>+ )}+ {build?.git_sha && (+ <MetaRow label="Git SHA" value={build.git_sha.slice(0, 12)} />+ )}+ {build?.build_time && (+ <MetaRow label="Build Time" value={build.build_time} />+ )}+ {build?.image && <MetaRow label="Image" value={build.image} />}+ {build?.image_digest && (+ <MetaRow+ label="Digest"+ value={`${build.image_digest.slice(0, 20)}…`}+ />+ )}+ {hasMetadata+ ? Object.entries(metadata).map(([k, v]) => (+ <MetaRow key={k} label={k} value={String(v)} />+ ))+ : null}+ {!isRealComponent && !hasBuildDetails && !hasMetadata && (+ <p className="text-xs text-zinc-500 italic">+ No additional details available.+ </p>+ )}++ {/* Action row — not shown for the synthetic Model Providers card */}+ {isRealComponent && (+ <div className="flex items-center justify-between gap-2 pt-2">+ <ActionButton+ icon={<Wrench size={14} />}+ ariaLabel="Debug this component"+ onClick={() => setDebugOpen(true)}+ />+ <div className="flex items-center gap-1.5">+ <ActionButton+ icon={<ScrollText size={13} />}+ label="Logs"+ onClick={() => setLogsOpen(true)}+ />+ <ActionButton+ icon={+ <RefreshCw+ size={13}+ className={cn(syncing && "animate-spin")}+ />+ }+ label="Sync"+ disabled={syncing}+ onClick={() =>+ syncMutation.mutate(component.name, {+ onError: (err) => {+ toast.error(+ `Sync failed: ${err instanceof Error ? err.message : "unknown error"}`,+ );+ },+ })+ }+ />+ </div>+ </div>+ )}+ </div>+ )}++ {/* Modals */}+ {isRealComponent && (+ <>+ <ComponentLogsModal+ component={component.name}+ displayName={display_name}+ open={logsOpen}+ onOpenChange={setLogsOpen}+ />+ <ComponentDiagnoseModal+ component={component.name}+ displayName={display_name}+ open={debugOpen}+ onOpenChange={setDebugOpen}+ />+ </>+ )}+ </div>+ );+}++// ─── provider / api-key health ───────────────────────────────────────────────++/** Map the /provider/health status onto the console-status component states. */+function providerHealthState(+ status: ProviderHealthResponse["status"],+): ComponentState {+ switch (status) {+ case "healthy":+ return "healthy";+ case "unhealthy":+ case "error":+ return "unhealthy";+ default:+ // "backend-unavailable" (or anything unexpected) — can't determine.+ return "unknown";+ }+}++/** Prefer the specific llm/embedding key errors; fall back to the summary. */+function providerHealthMessage(health: ProviderHealthResponse): string {+ if (health.status === "healthy") {+ return health.message || "Providers configured and validated";+ }+ const { llm_error: llmError, embedding_error: embeddingError } = health;+ if (llmError && embeddingError) {+ return llmError === embeddingError+ ? llmError+ : `${llmError}; ${embeddingError}`;+ }+ return (+ llmError || embeddingError || health.message || "Provider validation failed"+ );+}++/** Adapt a provider-health response into a synthetic status component so the+ * panel renders API-key health with the same card UI as backend components. */+function providerHealthToComponent(+ health: ProviderHealthResponse,+): ComponentStatus {+ const metadata: Record<string, unknown> = {};+ const llmProvider = health.llm_provider ?? health.provider;+ if (llmProvider) metadata["LLM provider"] = llmProvider;+ if (health.embedding_provider) {+ metadata["Embedding provider"] = health.embedding_provider;+ }+ if (health.details?.llm_model) {+ metadata["LLM model"] = health.details.llm_model;+ }+ if (health.details?.embedding_model) {+ metadata["Embedding model"] = health.details.embedding_model;+ }++ return {+ name: "providers",+ display_name: "Model Providers",+ status: providerHealthState(health.status),+ required: true,+ message: providerHealthMessage(health),+ metadata,+ };+}++// ─── main panel ──────────────────────────────────────────────────────────────++interface ConsoleStatusPanelProps {+ onClose: () => void;+}++export function ConsoleStatusPanel({ onClose }: ConsoleStatusPanelProps) {+ const {+ data,+ isLoading,+ isFetching,+ isError,+ error,+ refetch,+ dataUpdatedAt,+ } = useConsoleStatusQuery();++ const handleRefresh = useCallback(() => {+ void refetch();+ }, [refetch]);++ // Reuses the shared /provider/health query (same cache as the banner), so an+ // API-key failure surfaces here without an extra network round-trip.+ const { data: providerHealth } = useProviderHealthQuery();++ // Defensive: always read from data.components array+ const backendComponents = Array.isArray(data?.components)+ ? data.components+ : [];++ // Append provider / API-key health as a card so the panel is the single+ // place to spot a key failure. Skipped while the query has no result yet+ // (e.g. non-admins under RBAC, or during active ingestion).+ const components = providerHealth+ ? [...backendComponents, providerHealthToComponent(providerHealth)]+ : backendComponents;++ const counts = {+ healthy: components.filter((c) => c.status === "healthy").length,+ degraded: components.filter((c) => c.status === "degraded").length,+ unhealthy: components.filter((c) => c.status === "unhealthy").length,+ unknown: components.filter((c) => c.status === "unknown").length,+ };++ const lastUpdated = dataUpdatedAt+ ? new Date(dataUpdatedAt).toLocaleTimeString()+ : null;++ return (+ /*+ * Floating card — portalled into document.body so this fixed position+ * is always relative to the true viewport (no transformed ancestors).+ * bottom-[5.5rem] clears the button (40px tall) + 24px gap.+ */+ <div+ className={cn(+ "fixed right-6 top-[64px] z-40",+ "w-[440px] max-h-[calc(100vh-8rem)] flex flex-col",+ "bg-zinc-900 border border-zinc-700/70 rounded-2xl shadow-2xl overflow-hidden",+ )}+ >+ {/* Panel header */}+ <div className="flex items-center justify-between px-4 py-3 border-b border-zinc-700/60 shrink-0">+ <div className="flex items-center gap-2">+ <EvCharger size={15} className="text-zinc-400" />+ <h2 className="text-sm font-semibold text-zinc-100">+ Console Status+ </h2>+ {isFetching && (+ <RefreshCw size={12} className="text-zinc-500 animate-spin" />+ )}+ </div>+ <button+ type="button"+ onClick={onClose}+ className="text-zinc-500 hover:text-zinc-300 transition-colors p-1 rounded hover:bg-zinc-700/50"+ aria-label="Close console status"+ >+ <XCircle size={16} />+ </button>+ </div>++ {/* Scrollable body */}+ <div className="flex-1 overflow-y-auto scrollbar-hide px-4 py-3 space-y-3">+ {/* Summary placards row */}+ <div className="grid grid-cols-4 gap-2">+ <SummaryCard label="Healthy" count={counts.healthy} state="healthy" />+ <SummaryCard+ label="Degraded"+ count={counts.degraded}+ state="degraded"+ />+ <SummaryCard+ label="Unhealthy"+ count={counts.unhealthy}+ state="unhealthy"+ />+ <SummaryCard label="Unknown" count={counts.unknown} state="unknown" />+ </div>++ {/* Error state */}+ {isError && (+ <div className="rounded-xl border border-red-500/30 bg-red-500/10 px-3.5 py-3 text-sm text-red-400">+ {error instanceof Error ? error.message : "Failed to load status."}+ </div>+ )}++ {/* Component placards */}+ {isLoading ? (+ <div className="space-y-2">+ {[...Array(4)].map((_, i) => (+ <div+ // biome-ignore lint/suspicious/noArrayIndexKey: skeleton placeholders have no identity+ key={i}+ className="h-14 rounded-xl bg-zinc-800/50 border border-zinc-700/50 animate-pulse"+ />+ ))}+ </div>+ ) : components.length === 0 ? (+ <p className="text-xs text-zinc-500 text-center py-6">+ No components returned by status API.+ </p>+ ) : (+ <div className="space-y-2">+ {components.map((comp) => (+ <ComponentCard key={comp.name} component={comp} />+ ))}+ </div>+ )}+ </div>++ {/* Footer */}+ <div className="shrink-0 flex items-center justify-between px-4 py-2.5 border-t border-zinc-700/60">+ <span className="text-xs text-zinc-500">+ {lastUpdated ? `Last updated: ${lastUpdated}` : "Loading…"}+ </span>+ <button+ type="button"+ onClick={handleRefresh}+ disabled={isFetching}+ className="flex items-center gap-1.5 text-xs text-zinc-400 hover:text-zinc-200 transition-colors disabled:opacity-50"+ >+ <RefreshCw size={11} className={cn(isFetching && "animate-spin")} />+ Refresh All+ </button>+ </div>+ </div>+ );+}++// ─── floating trigger button ──────────────────────────────────────────────────++interface ConsoleStatusButtonProps {+ onClick: () => void;+ isOpen: boolean;+ overallStatus?: ComponentState;+}++export function ConsoleStatusButton({+ onClick,+ isOpen,+ overallStatus,+}: ConsoleStatusButtonProps) {+ return (+ <button+ type="button"+ onClick={onClick}+ aria-pressed={isOpen}+ className={cn(+ "flex items-center gap-2 px-3 py-2 rounded-lg",+ "bg-zinc-800 border border-zinc-700 text-zinc-200 text-sm font-medium",+ "hover:bg-zinc-700 hover:border-zinc-600 transition-colors shadow-lg",+ isOpen && "bg-zinc-700 border-zinc-500",+ )}+ >+ <EvCharger size={14} className="shrink-0 text-zinc-400" />+ <span>Console Status</span>+ {overallStatus && (+ <span className="relative flex h-2 w-2 shrink-0">+ {overallStatus === "unhealthy" && (+ // Pulsing ring draws the eye to an outage; hidden under reduced motion.+ <span className="absolute inline-flex h-full w-full rounded-full bg-red-400 opacity-75 animate-ping motion-reduce:hidden" />+ )}+ <span+ className={cn(+ "relative inline-flex h-2 w-2 rounded-full",+ overallStatus === "healthy" && "bg-emerald-400",+ overallStatus === "degraded" && "bg-amber-400",+ overallStatus === "unhealthy" && "bg-red-400",+ overallStatus === "unknown" && "bg-zinc-400",+ )}+ />+ </span>+ )}+ </button>+ );+}diff --git a/frontend/components/header.tsx b/frontend/components/header.tsxindex bdab23c5f..c55d8ee46 100644--- a/frontend/components/header.tsx+++ b/frontend/components/header.tsx@@ -2,16 +2,32 @@import { Bell } from "lucide-react";import { BrandSwitcher } from "@/components/brand-switcher";+import { ConsoleStatusButton } from "@/components/console-status-panel";import { DevRoleToggle } from "@/components/dev-role-toggle";import Logo from "@/components/icons/openrag-logo";import { UserNav } from "@/components/user-nav";+import { useAuth } from "@/contexts/auth-context";import { useIsCloudBrand } from "@/contexts/brand-context";+import { useConsoleStatus } from "@/contexts/console-status-context";import { useTask } from "@/contexts/task-context";import { cn } from "@/lib/utils";+import { useProviderHealth } from "./provider-health-banner";export function Header() {const isCloudBrand = useIsCloudBrand();const { tasks, toggleMenu } = useTask();+ const { runMode } = useAuth();++ const {+ hasProblem,+ toggle,+ isOpen,+ overallStatus: consoleOverallStatus,+ } = useConsoleStatus();+ const { isUnhealthy: isProviderUnhealthy } = useProviderHealth();+ const overallStatus = isProviderUnhealthy+ ? "unhealthy"+ : consoleOverallStatus;// Calculate active tasks for the bell iconconst activeTasks = tasks.filter(@@ -21,6 +37,9 @@ export function Header() {task.status === "processing",);+ // The bell dot lights for in-flight tasks OR a degraded/down component.+ const showNotificationDot = activeTasks.length > 0 || hasProblem;+return (<header className={cn(`flex w-full h-full items-center justify-between`)}><div className="header-start-display px-[16px]">@@ -58,7 +77,19 @@ export function Header() {</>)}- {/* Task Notification Bell */}+ {/* Console Status button — OSS only */}+ {runMode === "oss" && (+ <>+ <ConsoleStatusButton+ onClick={toggle}+ isOpen={isOpen}+ overallStatus={overallStatus}+ />+ <div className="w-px h-6 bg-border mx-3" />+ </>+ )}++ {/* Task + System Notification Bell */}<buttontype="button"onClick={toggleMenu}@@ -71,7 +102,7 @@ export function Header() {isCloudBrand ? "text-foreground" : "text-muted-foreground"}/>- {activeTasks.length > 0 && <div className="header-notifications" />}+ {showNotificationDot && <div className="header-notifications" />}</button>{/* Separator */}diff --git a/frontend/components/layout-wrapper.tsx b/frontend/components/layout-wrapper.tsxindex f31097c18..941a9b8a6 100644--- a/frontend/components/layout-wrapper.tsx+++ b/frontend/components/layout-wrapper.tsx@@ -3,6 +3,7 @@import { AnimatePresence, motion } from "framer-motion";import { usePathname, useRouter } from "next/navigation";import { useEffect } from "react";+import { createPortal } from "react-dom";import { useGetSettingsQuery } from "@/app/api/queries/useGetSettingsQuery";import {DoclingHealthBanner,@@ -18,33 +19,63 @@ import { TaskNotificationMenu } from "@/components/task-notification-menu";import { useAuth } from "@/contexts/auth-context";import { useIsCloudBrand } from "@/contexts/brand-context";import { useChat } from "@/contexts/chat-context";+import { useConsoleStatus } from "@/contexts/console-status-context";import { useKnowledgeFilter } from "@/contexts/knowledge-filter-context";import { useTask } from "@/contexts/task-context";+import { usePortal } from "@/hooks/use-portal";import { ANIMATION_DURATION, HEADER_HEIGHT } from "@/lib/constants";import { isFailureLikeTask } from "@/lib/task-utils";import { cn } from "@/lib/utils";import { AnimatedConditional } from "./animated-conditional";import { ChatRenderer } from "./chat-renderer";+import { ConsoleStatusPanel } from "./console-status-panel";import { FlowsUpdateDialog } from "./flows-update-dialog";import { Header } from "./header";import FailedTasksInfo from "./tasks_details";+/** Renders the Console Status button + panel directly into document.body via a+ * portal so that CSS transforms on ancestor elements (framer-motion sidebar+ * slide animation) cannot break `position: fixed` viewport anchoring. */+function ConsoleStatusPortal({+ isOpen,+ onClose,+}: {+ isOpen: boolean;+ onClose: () => void;+}) {+ const host = usePortal("console-status-portal");+ if (!host) return null;+ return createPortal(+ isOpen ? <ConsoleStatusPanel onClose={onClose} /> : null,+ host,+ );+}+export function LayoutWrapper({ children }: { children: React.ReactNode }) {const pathname = usePathname();const router = useRouter();- const { tasks, isMenuOpen } = useTask();+ const { tasks, isMenuOpen, closeMenu } = useTask();const isCloudBrand = useIsCloudBrand();const { isPanelOpen, panelMode, closePanelOnly } = useKnowledgeFilter();const failedTasks = tasks.filter(isFailureLikeTask);+ const { isOpen: isStatusOpen, close: closeStatus } = useConsoleStatus();+const isOnKnowledgePage = pathname.startsWith("/knowledge");- // Only one panel can be open at a time+ // Close status panel when task menu opens, and vice-versauseEffect(() => {if (isMenuOpen) {closePanelOnly();+ closeStatus();}- }, [isMenuOpen, closePanelOnly]);+ }, [isMenuOpen, closePanelOnly, closeStatus]);+ // Close the knowledge filter panel and the status panel when the task menu opens+ useEffect(() => {+ if (isStatusOpen) {+ closeMenu();+ }+ }, [isStatusOpen, closeMenu]);const { isLoading, isAuthenticated, isNoAuthMode, isIbmAuthMode, runMode } =useAuth();@@ -187,6 +218,14 @@ export function LayoutWrapper({ children }: { children: React.ReactNode }) {</div></div>+ {/* Console Status — portalled into document.body so fixed positioning+ is always relative to the viewport, not to any transformed ancestor.+ OSS-only: not rendered in saas or on_prem deployments. */}+ {isOnboardingComplete &&+ (isAuthenticated || isNoAuthMode) &&+ runMode === "oss" && (+ <ConsoleStatusPortal isOpen={isStatusOpen} onClose={closeStatus} />+ )}{(isAuthenticated || isNoAuthMode) && runMode === "oss" && (<FlowsUpdateDialog />)}diff --git a/frontend/components/task-notification-menu.tsx b/frontend/components/task-notification-menu.tsxindex 83c946e14..bacb46293 100644--- a/frontend/components/task-notification-menu.tsx+++ b/frontend/components/task-notification-menu.tsx@@ -27,6 +27,7 @@ import {CardTitle,} from "@/components/ui/card";import { useIsCloudBrand } from "@/contexts/brand-context";+import { useConsoleStatus } from "@/contexts/console-status-context";import { Task, useTask } from "@/contexts/task-context";import {hasIssueFileEntries,@@ -149,6 +150,7 @@ export function TaskNotificationMenu() {closeMenu,openTaskDialog,} = useTask();+ const { problems, hasProblem, open: openConsoleStatus } = useConsoleStatus();const [isPastOpen, setIsPastOpen] = useState(true);const lastHandledSelectionTriggerRef = useRef(0);@@ -342,6 +344,47 @@ export function TaskNotificationMenu() {{/* Content */}<div className="flex-1 overflow-y-auto">+ {/* System Status events — a container / service is degraded or down.+ Clicking an event opens the Console Status panel. */}+ {problems.length > 0 && (+ <div className="flex flex-col gap-2 p-4">+ <h4 className="text-sm font-medium text-muted-foreground">+ System Status+ </h4>+ {problems.map((component) => {+ const isDown = component.status === "unhealthy";+ return (+ <button+ key={component.name}+ type="button"+ data-testid="system-status-event"+ onClick={() => {+ openConsoleStatus();+ closeMenu();+ }}+ className="w-full rounded-lg border border-muted p-3 text-left transition-colors hover:bg-muted/60"+ >+ <div className="flex items-center gap-2">+ {isDown ? (+ <XCircle className="size-4 shrink-0 text-destructive" />+ ) : (+ <AlertCircle className="size-4 shrink-0 text-brand-amber" />+ )}+ <span className="text-sm font-medium">+ {component.display_name}{" "}+ {isDown ? "is down" : `is ${component.status}`}+ </span>+ </div>+ <p className="mt-1 text-xs text-muted-foreground">+ {component.message ||+ "Open Console Status for more details."}+ </p>+ </button>+ );+ })}+ </div>+ )}+{/* Active Tasks */}{activeTasks.length > 0 && (<div className="flex flex-col gap-2 p-4">@@ -524,18 +567,20 @@ export function TaskNotificationMenu() {</div>{/* Empty State */}- {activeTasks.length === 0 && terminalTasks.length === 0 && (- <div className="p-8 text-center">- <Bell className="h-12 w-12 text-muted-foreground/50 mx-auto mb-4" />- <h4 className="text-sm font-medium text-muted-foreground mb-2">- No tasks yet- </h4>- <p className="text-xs text-muted-foreground">- Task notifications will appear here when you upload files or- sync connectors.- </p>- </div>- )}+ {activeTasks.length === 0 &&+ terminalTasks.length === 0 &&+ problems.length === 0 && (+ <div className="p-8 text-center">+ <Bell className="h-12 w-12 text-muted-foreground/50 mx-auto mb-4" />+ <h4 className="text-sm font-medium text-muted-foreground mb-2">+ No tasks yet+ </h4>+ <p className="text-xs text-muted-foreground">+ Task notifications will appear here when you upload files or+ sync connectors.+ </p>+ </div>+ )}</div></div></div>diff --git a/frontend/contexts/console-status-context.tsx b/frontend/contexts/console-status-context.tsxnew file mode 100644index 000000000..e0d408f9e--- /dev/null+++ b/frontend/contexts/console-status-context.tsx@@ -0,0 +1,176 @@+"use client";++import type React from "react";+import {+ createContext,+ use,+ useCallback,+ useEffect,+ useMemo,+ useRef,+ useState,+} from "react";+import { toast } from "sonner";+import {+ type ComponentState,+ type ComponentStatus,+ type ConsoleStatusResponse,+ useConsoleStatusQuery,+} from "@/app/api/queries/useConsoleStatusQuery";+import { useAuth } from "@/contexts/auth-context";++/** Collapses the four component states into the two things the UI reacts to:+ * a warning (amber) or an outage (red). `ok` means nothing to show. */+export type StatusSeverity = "ok" | "warn" | "down";++interface ConsoleStatusContextType {+ overallStatus: ComponentState | undefined;+ /** Components that are not healthy, sorted by name (stable order). Each of+ * these becomes a "system event" in the notification bell menu. */+ problems: ComponentStatus[];+ severity: StatusSeverity;+ /** True when at least one component is degraded / unknown / down. Drives the+ * notification-bell dot. */+ hasProblem: boolean;+ /** Query couldn't complete (e.g. auth) — treated as neutral, never a problem. */+ isError: boolean;+ isOpen: boolean;+ open: () => void;+ close: () => void;+ toggle: () => void;+}++const ConsoleStatusContext = createContext<+ ConsoleStatusContextType | undefined+>(undefined);++// ─── helpers ───────────────────────────────────────────────────────────────++function severityOf(status: ComponentState | undefined): StatusSeverity {+ switch (status) {+ case "unhealthy":+ return "down";+ case "degraded":+ case "unknown":+ return "warn";+ default:+ // "healthy" or no data yet+ return "ok";+ }+}++const SEVERITY_RANK: Record<StatusSeverity, number> = {+ ok: 0,+ warn: 1,+ down: 2,+};++function problemSummary(problems: ComponentStatus[]): string {+ if (problems.length === 0) return "A component needs attention.";+ if (problems.length === 1) {+ const p = problems[0];+ return `${p.display_name} is ${p.status}.`;+ }+ const names = problems.map((p) => p.display_name).join(", ");+ return `${problems.length} components need attention: ${names}.`;+}++// ─── provider ────────────────────────────────────────────────────────────────++export function ConsoleStatusProvider({+ children,+}: {+ children: React.ReactNode;+}) {+ const { runMode } = useAuth();+ const isOss = runMode === "oss";++ // Shares the ["console-status"] cache key with the panel/button — React Query+ // dedupes, so mounting this provider does not add a second poll.+ // Disabled outside OSS so polling never fires in saas or on_prem.+ const { data, isError } = useConsoleStatusQuery({ enabled: isOss });++ const [isOpen, setIsOpen] = useState(false);++ const overallStatus = (data as ConsoleStatusResponse | undefined)+ ?.overall_status;++ const problems = useMemo(() => {+ const components = Array.isArray(data?.components) ? data.components : [];+ return components+ .filter((c) => c.status !== "healthy")+ .sort((a, b) => a.name.localeCompare(b.name));+ }, [data]);++ const severity = severityOf(overallStatus);+ const hasProblem = severity !== "ok";++ const open = useCallback(() => setIsOpen(true), []);+ const close = useCallback(() => setIsOpen(false), []);+ const toggle = useCallback(() => setIsOpen((v) => !v), []);++ // Active push: one toast per healthy→bad / bad→worse transition. `prevRef`+ // starts undefined and is seeded on the first successful fetch so we never+ // toast on initial load or on a page refresh.+ const prevStatusRef = useRef<ComponentState | undefined>(undefined);+ useEffect(() => {+ if (!overallStatus) return;++ const prev = prevStatusRef.current;+ prevStatusRef.current = overallStatus;++ if (prev === undefined || prev === overallStatus) return;++ const prevSev = severityOf(prev);+ const nextSev = severityOf(overallStatus);++ if (SEVERITY_RANK[nextSev] > SEVERITY_RANK[prevSev]) {+ const message =+ nextSev === "down" ? "OpenRAG component down" : "OpenRAG degraded";+ const fire = nextSev === "down" ? toast.error : toast.warning;+ fire(message, {+ description: problemSummary(problems),+ action: { label: "View", onClick: () => open() },+ });+ } else if (nextSev === "ok") {+ toast.success("All systems healthy");+ }+ }, [overallStatus, problems, open]);++ const value: ConsoleStatusContextType = {+ overallStatus,+ problems,+ severity,+ hasProblem,+ isError,+ isOpen,+ open,+ close,+ toggle,+ };++ return (+ <ConsoleStatusContext.Provider value={value}>+ {children}+ </ConsoleStatusContext.Provider>+ );+}++// Safe default returned when the feature is disabled (non-OSS run modes).+// Consumers such as header.tsx and task-notification-menu.tsx receive+// hasProblem=false and no-op callbacks, keeping the UI inert.+const NOOP_STATUS: Readonly<ConsoleStatusContextType> = Object.freeze({+ overallStatus: undefined,+ problems: [],+ severity: "ok" as StatusSeverity,+ hasProblem: false,+ isError: false,+ isOpen: false,+ open: () => {},+ close: () => {},+ toggle: () => {},+});++export function useConsoleStatus(): ConsoleStatusContextType {+ return use(ConsoleStatusContext) ?? NOOP_STATUS;+}diff --git a/frontend/hooks/use-portal.ts b/frontend/hooks/use-portal.tsnew file mode 100644index 000000000..fd6430716--- /dev/null+++ b/frontend/hooks/use-portal.ts@@ -0,0 +1,31 @@+"use client";++import { useEffect, useState } from "react";++/**+ * Returns a stable <div> element appended directly to document.body.+ * Mounting is deferred to the client so SSR never touches the DOM.+ * The element is removed on unmount.+ */+export function usePortal(id: string): HTMLElement | null {+ const [host, setHost] = useState<HTMLElement | null>(null);++ useEffect(() => {+ let el = document.getElementById(id);+ let created = false;+ if (!el) {+ el = document.createElement("div");+ el.id = id;+ document.body.appendChild(el);+ created = true;+ }+ setHost(el);+ return () => {+ if (created && el?.parentNode) {+ el.parentNode.removeChild(el);+ }+ };+ }, [id]);++ return host;+}diff --git a/src/api/health.py b/src/api/health.pyindex 31d697556..aa0e586d9 100644--- a/src/api/health.py+++ b/src/api/health.py@@ -3,15 +3,91 @@import asyncioimport httpx-from fastapi import Request+from fastapi import Depends, HTTPException, Query, Requestfrom fastapi.responses import JSONResponse+from api.schemas.status import (+ ComponentActionResponse,+ DiagnosisResponse,+ LogEntry,+ LogsResponse,+ StatusResponse,+)from config.settings import clients+from dependencies import require_permission+from services.component_logs import KNOWN_COMPONENTS, get_entries+from services.status_diagnostics import diagnose_component+from services.status_service import aggregate_status, check_one+from session_manager import Userfrom utils.logging_config import get_loggerlogger = get_logger(__name__)+async def get_console_status(+ user: User = Depends(require_permission("providers:read")),+) -> StatusResponse:+ """Aggregate component readiness for the browser UI. GET /status"""+ try:+ return await aggregate_status()+ except Exception as e:+ logger.error("Failed to get console status", error=str(e))+ raise HTTPException(status_code=500, detail="Failed to get status") from e+++async def get_console_component_logs(+ component: str,+ tail: int = Query(default=100, ge=1, le=500),+ user: User = Depends(require_permission("providers:read")),+) -> LogsResponse:+ """Return recent log entries for one component. GET /status/{component}/logs"""+ if component not in KNOWN_COMPONENTS:+ valid = ", ".join(sorted(KNOWN_COMPONENTS))+ raise HTTPException(+ status_code=404,+ detail=f"Unknown component '{component}'. Valid names: {valid}",+ )++ raw = get_entries(component, tail=tail)+ entries = [LogEntry(**e) for e in raw]+ return LogsResponse(component=component, entries=entries, count=len(entries))+++def _require_known_component(component: str) -> None:+ if component not in KNOWN_COMPONENTS:+ valid = ", ".join(sorted(KNOWN_COMPONENTS))+ raise HTTPException(404, f"Unknown component '{component}'. Valid names: {valid}")+++async def sync_console_component(+ component: str,+ user: User = Depends(require_permission("providers:read")),+) -> ComponentActionResponse:+ """Re-run one component's health check. POST /status/{component}/sync"""+ _require_known_component(component)+ status = await check_one(component)+ if status is None:+ raise HTTPException(status_code=503, detail=f"No health check registered for '{component}'")++ return ComponentActionResponse(+ component=component, ok=True, message="Status refreshed.", status=status+ )+++async def diagnose_console_component(+ component: str,+ user: User = Depends(require_permission("providers:read")),+) -> DiagnosisResponse:+ """Explain what's wrong with a component and how to fix it.+ GET /status/{component}/diagnose"""+ _require_known_component(component)+ status = await check_one(component)+ if status is None:+ raise HTTPException(status_code=503, detail=f"No health check registered for '{component}'")++ return diagnose_component(status, get_entries(component, tail=50))++async def health_check(request: Request):"""Simple liveness probe: Indicates that the OpenRAG Backend service is online and running."""return JSONResponse({"status": "ok"}, status_code=200)diff --git a/src/api/schemas/status.py b/src/api/schemas/status.pynew file mode 100644index 000000000..539a5719a--- /dev/null+++ b/src/api/schemas/status.py@@ -0,0 +1,75 @@+from enum import StrEnum+from typing import Any++from pydantic import BaseModel, Field+++class ComponentState(StrEnum):+ HEALTHY = "healthy"+ DEGRADED = "degraded"+ UNHEALTHY = "unhealthy"+ UNKNOWN = "unknown"+++class ComponentBuild(BaseModel):+ git_sha: str | None = None+ build_time: str | None = None+ image: str | None = None+ image_digest: str | None = None+++class ComponentStatus(BaseModel):+ name: str+ display_name: str+ status: ComponentState+ required: bool+ latency_ms: int | None = None+ message: str | None = None+ version: str | None = None+ build: ComponentBuild = Field(default_factory=ComponentBuild)+ metadata: dict[str, Any] = Field(default_factory=dict)+ # Added by #2178 — None when healthy, last exception detail when not.+ # The frontend uses this as the signal to show the Logs button.+ last_error: str | None = None+ # Added by #2183 — when this component was last checked (ISO-8601 UTC).+ checked_at: str | None = None+++class LogEntry(BaseModel):+ timestamp: str # ISO-8601 UTC+ level: str+ message: str+ detail: str | None = None+++class LogsResponse(BaseModel):+ component: str+ entries: list[LogEntry] = Field(default_factory=list)+ count: int+++class StatusResponse(BaseModel):+ overall_status: ComponentState+ checked_at: str # ISO 8601 UTC+ components: list[ComponentStatus] = Field(default_factory=list)+++class ComponentActionResponse(BaseModel):+ """Result of a per-component action (sync/restart), carrying fresh status."""++ component: str+ ok: bool+ message: str+ status: ComponentStatus+++class DiagnosisResponse(BaseModel):+ """Guidance for the "Debug" button: what's wrong and how to fix it."""++ component: str+ state: ComponentState+ summary: str+ likely_cause: str | None = None+ remediation: list[str] = Field(default_factory=list)+ last_error: str | None = None+ target: str | None = Nonediff --git a/src/api/v1/status.py b/src/api/v1/status.pynew file mode 100644index 000000000..b5c30c777--- /dev/null+++ b/src/api/v1/status.py@@ -0,0 +1,45 @@+from fastapi import Depends, HTTPException, Query++from api.schemas.status import LogEntry, LogsResponse, StatusResponse+from dependencies import require_api_key_permission+from services.component_logs import KNOWN_COMPONENTS, get_entries+from services.status_service import aggregate_status+from session_manager import User+from utils.logging_config import get_logger++logger = get_logger(__name__)+++async def get_status_endpoint(+ user: User = Depends(require_api_key_permission("providers:read")),+) -> StatusResponse:+ """Aggregate component readiness. GET /v1/status"""+ try:+ return await aggregate_status()+ except Exception as e:+ logger.error("Failed to get status", error=str(e))+ raise HTTPException(status_code=500, detail="Failed to get status") from e+++async def get_component_logs_endpoint(+ component: str,+ tail: int = Query(default=100, ge=1, le=500),+ user: User = Depends(require_api_key_permission("providers:read")),+) -> LogsResponse:+ """Return recent log entries for one component. GET /v1/status/{component}/logs++ The in-process ring buffer is the uniform source of truth for all four+ components. A Langflow native-logs proxy was evaluated and dropped:+ the endpoint path, auth requirements, and response shape are unstable+ across Langflow versions (0.9.6 vs 0.11.2rc0+).+ """+ if component not in KNOWN_COMPONENTS:+ valid = ", ".join(sorted(KNOWN_COMPONENTS))+ raise HTTPException(+ status_code=404,+ detail=f"Unknown component '{component}'. Valid names: {valid}",+ )++ raw = get_entries(component, tail=tail)+ entries = [LogEntry(**e) for e in raw]+ return LogsResponse(component=component, entries=entries, count=len(entries))diff --git a/src/app/routes/internal.py b/src/app/routes/internal.pyindex 0fe449026..b427edfe7 100644--- a/src/app/routes/internal.py+++ b/src/app/routes/internal.py@@ -31,9 +31,17 @@ from api import (upload,)from api import keys as api_keys-from api.health import health_check, opensearch_health_ready+from api.health import (+ diagnose_console_component,+ get_console_component_logs,+ get_console_status,+ health_check,+ opensearch_health_ready,+ sync_console_component,+)from api.schemas.tasks import ErrorResponse, TaskRetryResponsefrom connectors.registry import get_connector_classes+from utils.run_mode_utils import is_run_mode_ossdef register_internal_routes(app: FastAPI):@@ -387,6 +395,33 @@ def register_internal_routes(app: FastAPI):app.add_api_route("/health", health_check, methods=["GET"], tags=["internal"])app.add_api_route("/search/health", opensearch_health_ready, methods=["GET"], tags=["internal"])+ # Console status endpoint (browser session auth — mirrors /v1/status for the UI).+ # OSS-only: not registered in saas or on_prem deployments.+ if is_run_mode_oss():+ app.add_api_route("/status", get_console_status, methods=["GET"], tags=["internal"])++ # Console status endpoints (browser session auth — mirrors /v1/status* for the UI)+ # The specific /logs sub-route must be registered before the bare /status route.+ app.add_api_route(+ "/status/{component}/logs",+ get_console_component_logs,+ methods=["GET"],+ tags=["internal"],+ )+ # Per-component actions (#2183): sync re-checks, diagnose explains failures.+ app.add_api_route(+ "/status/{component}/sync",+ sync_console_component,+ methods=["POST"],+ tags=["internal"],+ )+ app.add_api_route(+ "/status/{component}/diagnose",+ diagnose_console_component,+ methods=["GET"],+ tags=["internal"],+ )+# Models endpointsapp.add_api_route("/models/openai", models.get_openai_models, methods=["POST"], tags=["internal"]diff --git a/src/app/routes/public_v1.py b/src/app/routes/public_v1.pyindex c87d5fcfd..534689e40 100644--- a/src/app/routes/public_v1.py+++ b/src/app/routes/public_v1.py@@ -23,6 +23,8 @@ from api.v1 import (from api.v1 import (settings as v1_settings,)+from api.v1 import status as v1_status+from utils.run_mode_utils import is_run_mode_ossdef register_public_v1_routes(app: FastAPI):@@ -134,6 +136,24 @@ def register_public_v1_routes(app: FastAPI):tags=["public"],)+ # Status endpoints (OSS-only) — component logs route must be registered before+ # the bare /v1/status route so Starlette matches the literal "/logs" suffix+ # rather than treating it as a {component} path parameter.+ if is_run_mode_oss():+ app.add_api_route(+ "/v1/status/{component}/logs",+ v1_status.get_component_logs_endpoint,+ methods=["GET"],+ tags=["public"],+ )++ app.add_api_route(+ "/v1/status",+ v1_status.get_status_endpoint,+ methods=["GET"],+ tags=["public"],+ )+# Files get_all endpointapp.add_api_route("/v1/files/get_all",diff --git a/src/services/component_logs.py b/src/services/component_logs.pynew file mode 100644index 000000000..bf46ce0aa--- /dev/null+++ b/src/services/component_logs.py@@ -0,0 +1,155 @@+"""In-memory ring buffer for per-component log entries.++Intentionally free of FastAPI imports so services can use it at any layer.+Thread-safe via a per-component threading.Lock.+"""++import re+import threading+from collections import deque+from datetime import UTC, datetime+from typing import TypedDict++# ---------------------------------------------------------------------------+# Constants+# ---------------------------------------------------------------------------++BUFFER_MAXLEN = 200++# Valid component names — used by the API layer for 404 gating.+KNOWN_COMPONENTS: frozenset[str] = frozenset({"openrag", "langflow", "docling", "opensearch"})++# Reuse the same pattern as logging_config._SENSITIVE_HEADER_RE so that keys+# / tokens never surface in HTTP responses.+_SENSITIVE_RE = re.compile(r"(key|token|secret|password|apikey|credential|jwt|auth)", re.IGNORECASE)+# Strip bare Bearer tokens from free-form detail strings.+_BEARER_RE = re.compile(r"Bearer\s+\S+", re.IGNORECASE)+++# ---------------------------------------------------------------------------+# Types+# ---------------------------------------------------------------------------+++class LogEntry(TypedDict):+ timestamp: str # ISO-8601 UTC+ level: str # "debug" | "info" | "warning" | "error" | "critical"+ message: str+ detail: str | None+++# ---------------------------------------------------------------------------+# Internal state+# ---------------------------------------------------------------------------++# deque per component name+_buffers: dict[str, deque[LogEntry]] = {}+_locks: dict[str, threading.Lock] = {}+# last outcome per component: True = last check was healthy, False = unhealthy/unknown+_last_ok: dict[str, bool] = {}+_state_lock = threading.Lock()+++def _get_or_create(component: str) -> tuple[deque[LogEntry], threading.Lock]:+ """Return (buffer, lock) for *component*, creating them on first access."""+ with _state_lock:+ if component not in _buffers:+ _buffers[component] = deque(maxlen=BUFFER_MAXLEN)+ _locks[component] = threading.Lock()+ return _buffers[component], _locks[component]+++# ---------------------------------------------------------------------------+# Redaction+# ---------------------------------------------------------------------------+++def _redact(text: str | None) -> str | None:+ """Strip Bearer tokens from *text*. Key/secret values are not present in+ free-form detail strings (they come from exception messages), but Bearer+ tokens can appear in httpx error repr if auth headers are dumped."""+ if text is None:+ return None+ return _BEARER_RE.sub("Bearer ***", text)+++def _redact_value(v: object) -> object:+ """Recursively redact sensitive values in nested dicts/lists."""+ if isinstance(v, dict):+ return _redact_dict(v)+ if isinstance(v, (list, tuple)):+ return type(v)(_redact_value(item) for item in v)+ if isinstance(v, str):+ return _BEARER_RE.sub("Bearer ***", v)+ return v+++def _redact_dict(event_dict: dict) -> dict:+ """Return a copy of *event_dict* with sensitive values masked, recursively."""+ out: dict[str, object] = {}+ for k, v in event_dict.items():+ if _SENSITIVE_RE.search(str(k)):+ out[k] = "***"+ else:+ out[k] = _redact_value(v)+ return out+++# ---------------------------------------------------------------------------+# Public API+# ---------------------------------------------------------------------------+++def record(component: str, level: str, message: str, detail: str | None = None) -> None:+ """Append one entry to *component*'s ring buffer.++ Unknown component names are silently accepted so callers never need to+ guard — the buffer just grows dynamically (still capped at BUFFER_MAXLEN).+ """+ buf, lock = _get_or_create(component)+ entry: LogEntry = {+ "timestamp": datetime.now(UTC).isoformat(),+ "level": level,+ "message": message,+ "detail": _redact(detail),+ }+ with lock:+ buf.append(entry)+++def record_check_result(+ component: str,+ ok: bool,+ message: str,+ detail: str | None = None,+) -> None:+ """Write to the buffer with flood-prevention logic.++ - Failure (ok=False): always records an error entry.+ - Recovery (False→True transition): records one info "recovered" entry.+ - Steady healthy (True→True): records nothing, keeping error history intact.+ """+ with _state_lock:+ was_ok = _last_ok.get(component, True) # assume healthy on first call+ _last_ok[component] = ok++ if not ok:+ record(component, "error", message, detail=detail)+ elif not was_ok:+ # Transition: unhealthy → healthy+ record(component, "info", f"recovered: {message}")+++def get_entries(component: str, tail: int = 100) -> list[LogEntry]:+ """Return up to *tail* most-recent entries for *component* (oldest→newest).++ Returns an empty list for unknown / never-recorded components.+ """+ if component not in _buffers:+ return []+ buf, lock = _get_or_create(component)+ with lock:+ entries = list(buf)+ if tail >= len(entries):+ return entries+ return entries[-tail:]diff --git a/src/services/status_checks.py b/src/services/status_checks.pynew file mode 100644index 000000000..370aa53f1--- /dev/null+++ b/src/services/status_checks.py@@ -0,0 +1,275 @@+from time import perf_counter++from api.schemas.status import ComponentBuild, ComponentState, ComponentStatus+from config.settings import DOCLING_SERVE_URL, LANGFLOW_URL, clients, get_openrag_config+from services.component_logs import record_check_result+from utils.logging_config import get_logger+from utils.version_utils import OPENRAG_VERSION++logger = get_logger(__name__)++_CHECK_TIMEOUT_S = 2.0+++async def check_openrag_backend() -> ComponentStatus:+ start = perf_counter()+ try:+ get_openrag_config()+ except Exception as e:+ logger.warning("OpenRAG config not loaded", error=str(e))+ message = "OpenRAG configuration is not loaded"+ record_check_result(+ "openrag",+ False,+ message,+ detail=f"{type(e).__name__}: {e}",+ )+ return ComponentStatus(+ name="openrag",+ display_name="OpenRAG Backend",+ status=ComponentState.UNHEALTHY,+ required=True,+ latency_ms=int((perf_counter() - start) * 1000),+ message=message,+ version=None,+ build=ComponentBuild(), # NOTE: deferring this to later Version and build traceability step+ metadata={},+ last_error=f"{type(e).__name__}: {e}",+ )++ missing = []+ if clients.opensearch is None:+ missing.append("opensearch client")+ if clients.langflow_http_client is None:+ missing.append("langflow client")+ if clients.docling_http_client is None:+ missing.append("docling client")++ if missing:+ status = ComponentState.DEGRADED+ message = "Backend serving but not fully initialized: " + ", ".join(missing)+ record_check_result("openrag", False, message)+ last_error: str | None = message+ else:+ status, message = ComponentState.HEALTHY, "OpenRAG backend is ready"+ record_check_result("openrag", True, message)+ last_error = None++ return ComponentStatus(+ name="openrag",+ display_name="OpenRAG Backend",+ status=status,+ required=True,+ latency_ms=int((perf_counter() - start) * 1000),+ message=message,+ version=OPENRAG_VERSION,+ build=ComponentBuild(), # NOTE: deferring this to later Version and build traceability step+ metadata={},+ last_error=last_error,+ )+++async def check_docling() -> ComponentStatus:+ start = perf_counter()+ version = None+ target_url = f"{DOCLING_SERVE_URL}/version"++ try:+ docling_client = clients.docling_http_client+ if not docling_client:+ message = "Docling client is not initialized"+ record_check_result("docling", False, message)+ return ComponentStatus(+ name="docling",+ display_name="Docling",+ status=ComponentState.UNKNOWN,+ required=True,+ latency_ms=int((perf_counter() - start) * 1000),+ message=message,+ version=version,+ build=ComponentBuild(), # NOTE: deferring this to later Version and build traceability step+ metadata={},+ last_error=message,+ )++ resp = await docling_client.get(target_url, timeout=_CHECK_TIMEOUT_S)+ if resp.status_code == 200:+ status, message = ComponentState.HEALTHY, "Docling Serve reachable"+ version = resp.json().get("docling-serve")+ record_check_result("docling", True, message)+ last_error = None+ else:+ message = f"Docling returned HTTP {resp.status_code}"+ status = ComponentState.UNHEALTHY+ record_check_result(+ "docling",+ False,+ message,+ detail=f"HTTP {resp.status_code} — target: {target_url}",+ )+ last_error = message+ except Exception as e:+ logger.warning("Docling status check failed", error=str(e))+ message = "Docling Serve unreachable"+ status = ComponentState.UNHEALTHY+ record_check_result(+ "docling",+ False,+ message,+ detail=f"{type(e).__name__}: {e} — target: {target_url}",+ )+ last_error = f"{type(e).__name__}: {e} — target: {target_url}"++ return ComponentStatus(+ name="docling",+ display_name="Docling",+ status=status,+ required=True,+ latency_ms=int((perf_counter() - start) * 1000),+ message=message,+ version=version,+ build=ComponentBuild(), # NOTE: deferring this to later Version and build traceability step+ metadata={},+ last_error=last_error,+ )+++async def check_langflow() -> ComponentStatus:+ start = perf_counter()+ version = None+ target_url = f"{LANGFLOW_URL}/api/v1/version"++ try:+ langflow_client = clients.langflow_http_client+ if not langflow_client:+ message = "Langflow client is not initialized"+ record_check_result("langflow", False, message)+ return ComponentStatus(+ name="langflow",+ display_name="Langflow",+ status=ComponentState.UNKNOWN,+ required=True,+ latency_ms=int((perf_counter() - start) * 1000),+ message=message,+ version=version,+ build=ComponentBuild(), # NOTE: deferring this to later Version and build traceability step+ metadata={},+ last_error=message,+ )++ resp = await langflow_client.get("/api/v1/version", timeout=_CHECK_TIMEOUT_S)+ if resp.status_code == 200:+ status, message = ComponentState.HEALTHY, "Langflow API reachable"+ version = resp.json().get("version")+ record_check_result("langflow", True, message)+ last_error = None+ else:+ message = f"Langflow returned HTTP {resp.status_code}"+ status = ComponentState.UNHEALTHY+ record_check_result(+ "langflow",+ False,+ message,+ detail=f"HTTP {resp.status_code} — target: {target_url}",+ )+ last_error = message+ except Exception as e:+ logger.warning("Langflow status check failed", error=str(e))+ message = "Langflow is unreachable"+ status = ComponentState.UNHEALTHY+ record_check_result(+ "langflow",+ False,+ message,+ detail=f"{type(e).__name__}: {e} — target: {target_url}",+ )+ last_error = f"{type(e).__name__}: {e} — target: {target_url}"++ return ComponentStatus(+ name="langflow",+ display_name="Langflow",+ status=status,+ required=True,+ latency_ms=int((perf_counter() - start) * 1000),+ message=message,+ version=version,+ build=ComponentBuild(), # NOTE: deferring this to later Version and build traceability step+ metadata={},+ last_error=last_error,+ )+++async def check_opensearch() -> ComponentStatus:+ start = perf_counter()+ version = None+ os_version = None++ try:+ opensearch = clients.opensearch+ if opensearch is None:+ message = "OpenSearch client is not initialized"+ record_check_result("opensearch", False, message)+ return ComponentStatus(+ name="opensearch",+ display_name="OpenSearch",+ status=ComponentState.UNKNOWN,+ required=True,+ latency_ms=int((perf_counter() - start) * 1000),+ message=message,+ version=version,+ build=ComponentBuild(), # NOTE: deferring this to later Version and build traceability step+ metadata={},+ last_error=message,+ )++ health = await opensearch.cluster.health()+ info = await opensearch.info()+ cluster_status = health.get("status")+ os_version = (info.get("version") or {}).get("number")+ distribution = (info.get("version") or {}).get("distribution")++ status = {+ "green": ComponentState.HEALTHY,+ "yellow": ComponentState.DEGRADED,+ "red": ComponentState.UNHEALTHY,+ }.get(cluster_status, ComponentState.UNKNOWN)+ message = f"Cluster Health is {cluster_status}"+ metadata = {+ "cluster_name": health.get("cluster_name"),+ "cluster_health": cluster_status,+ "distribution": distribution,+ }+ ok = status in (ComponentState.HEALTHY, ComponentState.DEGRADED)+ record_check_result(+ "opensearch",+ ok,+ message,+ detail=None if ok else f"cluster_health={cluster_status}",+ )+ last_error = None if ok else f"cluster_health={cluster_status}"++ except Exception as e:+ logger.warning("OpenSearch status check failed", error=str(e))+ message = "OpenSearch is unreachable"+ status = ComponentState.UNHEALTHY+ metadata = {}+ record_check_result(+ "opensearch",+ False,+ message,+ detail=f"{type(e).__name__}: {e}",+ )+ last_error = f"{type(e).__name__}: {e}"++ return ComponentStatus(+ name="opensearch",+ display_name="OpenSearch",+ status=status,+ required=True,+ latency_ms=int((perf_counter() - start) * 1000),+ message=message,+ version=os_version,+ build=ComponentBuild(), # NOTE: deferring this to later Version and build traceability step+ metadata=metadata,+ last_error=last_error,+ )diff --git a/src/services/status_diagnostics.py b/src/services/status_diagnostics.pynew file mode 100644index 000000000..98b07dbf7--- /dev/null+++ b/src/services/status_diagnostics.py@@ -0,0 +1,104 @@+"""Rule-based diagnosis for the Console Status "Debug" button (#2183).++Turns a component's status + recent log detail into plain-English guidance:+what's likely wrong and concrete steps to fix it.+"""++from api.schemas.status import ComponentState, ComponentStatus, DiagnosisResponse+from services.component_logs import LogEntry++_DISPLAY = {+ "openrag": "OpenRAG Backend",+ "langflow": "Langflow",+ "docling": "Docling",+ "opensearch": "OpenSearch",+}++_CONNECTION_HINTS = ("connect", "refused", "timed out", "timeout", "unreachable")+++def _target(component: str) -> str | None:+ from config.settings import DOCLING_SERVE_URL, LANGFLOW_URL, OPENSEARCH_URL++ return {+ "langflow": f"{LANGFLOW_URL}/api/v1/version",+ "docling": f"{DOCLING_SERVE_URL}/version",+ "opensearch": OPENSEARCH_URL,+ }.get(component)+++def diagnose_component(+ status: ComponentStatus, entries: list[LogEntry] | None = None+) -> DiagnosisResponse:+ component = status.name+ display = _DISPLAY.get(component, status.display_name or component)+ target = _target(component)+ last_err = next(+ (+ e.get("detail")+ for e in reversed(entries or [])+ if e.get("level") in ("error", "critical")+ ),+ None,+ )+ detail = status.last_error or last_err++ def resp(summary, cause=None, steps=None):+ return DiagnosisResponse(+ component=component,+ state=status.status,+ summary=summary,+ likely_cause=cause,+ remediation=steps or [],+ last_error=detail,+ target=target,+ )++ if status.status == ComponentState.HEALTHY:+ return resp(f"{display} is healthy — no action needed.")++ if component == "opensearch" and status.status == ComponentState.DEGRADED:+ return resp(+ f"{display} cluster health is yellow (degraded).",+ "Replica shards are unassigned — normal for a single-node dev cluster.",+ [+ "On a single-node setup this is expected and safe to ignore.",+ "For production, add data nodes so replicas can be assigned.",+ ],+ )++ if detail and any(h in detail.lower() for h in _CONNECTION_HINTS):+ if target:+ return resp(+ f"{display} is unreachable — the backend couldn't connect to it.",+ f"The service isn't accepting connections at {target}. It may be stopped or still starting.",+ [+ f"Confirm the service is running at {target}.",+ 'Click "Sync" once it\'s back to re-check.',+ 'Check "Logs" for the underlying error.',+ ],+ )+ return resp(+ f"{display} is unreachable — the backend couldn't connect to it.",+ "The service may be stopped or still starting.",+ [+ 'Click "Sync" once it\'s back to re-check.',+ 'Check "Logs" for the underlying error.',+ ],+ )++ if status.status == ComponentState.UNKNOWN:+ return resp(+ f"{display}'s status could not be determined.",+ "The health check didn't complete — the client may not be initialized, or it timed out.",+ [+ 'Click "Sync" to re-run the check.',+ "Wait a moment if the backend is still starting.",+ ],+ )++ return resp(+ f"{display} reported: {status.message or status.status}.",+ detail or "The service responded but not in a healthy state.",+ ['Click "Sync" to confirm the current status.', 'Review "Logs" for details.'],+ )diff --git a/src/services/status_service.py b/src/services/status_service.pynew file mode 100644index 000000000..5033a8d39--- /dev/null+++ b/src/services/status_service.py@@ -0,0 +1,99 @@+import asyncio+from collections.abc import Awaitable, Callable+from datetime import UTC, datetime++from api.schemas.status import ComponentState, ComponentStatus, StatusResponse+from services.component_logs import record+from services.status_checks import (+ check_docling,+ check_langflow,+ check_openrag_backend,+ check_opensearch,+)+from utils.logging_config import get_logger++logger = get_logger(__name__)++CHECK_TIMEOUT_S = 5.0++CHECK_SPECS = [check_openrag_backend, check_docling, check_langflow, check_opensearch]+++def _check_name(fn: Callable) -> str:+ return fn.__name__.replace("check_", "").replace("openrag_backend", "openrag")+++# component name → its check function, for refreshing a single component (#2183).+CHECK_BY_NAME: dict[str, Callable[[], Awaitable[ComponentStatus]]] = {+ _check_name(fn): fn for fn in CHECK_SPECS+}+++async def _run_check(fn: Callable[[], Awaitable[ComponentStatus]]) -> ComponentStatus:+ """Run one check function, wrapping timeouts and unexpected exceptions.++ Both failure modes are recorded to the component log buffer so that the+ /v1/status/{component}/logs endpoint can surface the detail later.+ """+ name = _check_name(fn)+ try:+ result = await asyncio.wait_for(fn(), timeout=CHECK_TIMEOUT_S)+ except TimeoutError:+ msg = f"Status check timed out after {CHECK_TIMEOUT_S}s"+ logger.warning("Status check timed out", component=name, timeout_s=CHECK_TIMEOUT_S)+ record(name, "error", msg, detail=f"asyncio.TimeoutError — timeout={CHECK_TIMEOUT_S}s")+ result = ComponentStatus(+ name=name,+ display_name=name.title(),+ status=ComponentState.UNKNOWN,+ required=True,+ message="Status check did not complete",+ last_error=msg,+ )+ except Exception as e:+ msg = "Status check did not complete"+ logger.warning("Status check did not complete", component=name, error=str(e))+ record(name, "error", msg, detail=f"{type(e).__name__}: {e}")+ result = ComponentStatus(+ name=name,+ display_name=name.title(),+ status=ComponentState.UNKNOWN,+ required=True,+ message=msg,+ last_error=f"{type(e).__name__}: {e}",+ )++ # Stamp the check time here so every code path (aggregate or single) is consistent.+ result.checked_at = datetime.now(UTC).isoformat()+ return result+++SEVERITY_ORDER = [+ ComponentState.HEALTHY,+ ComponentState.DEGRADED,+ ComponentState.UNKNOWN,+ ComponentState.UNHEALTHY,+]+++def _worst_status(results: list[ComponentStatus]) -> ComponentState:+ """This calculates the final status (the worst one)"""+ severity = {state: i for i, state in enumerate(SEVERITY_ORDER)}+ return SEVERITY_ORDER[max((severity[r.status] for r in results), default=0)]+++async def aggregate_status() -> StatusResponse:+ """Runs all component health checks concurrently and return the aggregate status"""+ results = await asyncio.gather(*(_run_check(fn) for fn in CHECK_SPECS))++ return StatusResponse(+ overall_status=_worst_status(list(results)),+ checked_at=datetime.now(UTC).isoformat(),+ components=list(results),+ )+++async def check_one(name: str) -> ComponentStatus | None:+ """Re-run one component's health check (#2183 "Sync"). None if unknown."""+ fn = CHECK_BY_NAME.get(name)+ return await _run_check(fn) if fn is not None else Nonediff --git a/src/utils/logging_config.py b/src/utils/logging_config.pyindex d703feb22..5efddb7db 100644--- a/src/utils/logging_config.py+++ b/src/utils/logging_config.py@@ -32,6 +32,10 @@ _SENSITIVE_HEADER_RE = re.compile(# reference the same chain that configure_logging() assembled._shared_processors: list = []+# Minimum stdlib level integer that gets mirrored into the component buffer.+# WARNING=30, ERROR=40, CRITICAL=50.+_MIRROR_MIN_LEVEL = logging.WARNING+# ---------------------------------------------------------------------------# Standalone processors (module-level so they can be reused in stdlib bridge)@@ -92,6 +96,46 @@ def add_global_fields_factory(service: str, env: str, version: str):return processor+def mirror_to_component_buffer(_, __, event_dict: dict[str, Any]) -> dict[str, Any]:+ """Structlog processor: mirror WARNING+ backend events into the 'openrag' buffer.++ Runs after add_log_level so event_dict["level"] is always present.+ Uses lazy import to avoid circular imports at module load time.+ Reuses _SENSITIVE_HEADER_RE for key redaction; also strips Bearer tokens.+ This processor is a pass-through — it never drops or modifies the event.+ """+ try:+ level_str = event_dict.get("level", "info")+ level_int = getattr(logging, level_str.upper(), logging.INFO)+ if level_int < _MIRROR_MIN_LEVEL:+ return event_dict++ from services.component_logs import _redact_dict, record # noqa: PLC0415++ safe = _redact_dict(dict(event_dict))+ message = str(safe.pop("event", ""))+ # Build a compact detail string from remaining context keys (skip+ # structlog internals and the fields already in message/level).+ skip = {+ "level",+ "service",+ "env",+ "version",+ "timestamp",+ "func_name",+ "filename",+ "lineno",+ "pathname",+ }+ detail_parts = [f"{k}={v}" for k, v in safe.items() if k not in skip and v is not None]+ detail = "; ".join(detail_parts) if detail_parts else None++ record("openrag", level_str.lower(), message, detail=detail)+ except Exception: # pragma: no cover — never let logging crash the app+ pass+ return event_dict++# ---------------------------------------------------------------------------# Security helper# ---------------------------------------------------------------------------@@ -130,6 +174,9 @@ def configure_logging(clean_log_location,add_global_fields_factory(service_name, app_env, app_version),structlog.processors.add_log_level,+ # Mirror WARNING+ to the per-component ring buffer (must come after+ # add_log_level so event_dict["level"] is populated).+ mirror_to_component_buffer,structlog.processors.StackInfoRenderer(),drop_color_message_key,]diff --git a/tests/unit/api/test_status_logs_endpoint.py b/tests/unit/api/test_status_logs_endpoint.pynew file mode 100644index 000000000..dc61d7493--- /dev/null+++ b/tests/unit/api/test_status_logs_endpoint.py@@ -0,0 +1,167 @@+"""Unit tests for GET /v1/status/{component}/logs endpoint handler.++Tests call the endpoint function directly (not via a full TestClient stack)+to stay consistent with the other unit/api tests in this directory.+"""++import sys+from pathlib import Path+from unittest.mock import MagicMock++import pytest+from fastapi import HTTPException++ROOT = Path(__file__).resolve().parent.parent.parent.parent+SRC = ROOT / "src"+if str(SRC) not in sys.path:+ sys.path.insert(0, str(SRC))++import services.component_logs as cl # noqa: E402+from api.schemas.status import LogsResponse # noqa: E402+from api.v1.status import get_component_logs_endpoint # noqa: E402+from session_manager import User # noqa: E402++# ---------------------------------------------------------------------------+# Helpers+# ---------------------------------------------------------------------------+++def _user():+ return MagicMock(spec=User)++[email protected](autouse=True)+def _reset_buffer():+ cl._buffers.clear()+ cl._locks.clear()+ cl._last_ok.clear()+ yield+ cl._buffers.clear()+ cl._locks.clear()+ cl._last_ok.clear()+++# ---------------------------------------------------------------------------+# 404 — unknown component+# ---------------------------------------------------------------------------+++async def test_unknown_component_raises_404():+ with pytest.raises(HTTPException) as exc_info:+ await get_component_logs_endpoint(component="foobar", tail=50, user=_user())++ assert exc_info.value.status_code == 404+ assert "foobar" in exc_info.value.detail+ assert "Valid names" in exc_info.value.detail+++async def test_404_detail_lists_all_known_names():+ with pytest.raises(HTTPException) as exc_info:+ await get_component_logs_endpoint(component="unknown", tail=50, user=_user())++ detail = exc_info.value.detail+ for name in ("docling", "langflow", "openrag", "opensearch"):+ assert name in detail+++# ---------------------------------------------------------------------------+# 200 — known components from buffer+# ---------------------------------------------------------------------------+++async def test_known_component_returns_200_shape():+ cl.record("opensearch", "error", "cluster red", detail="status=red")++ result = await get_component_logs_endpoint(component="opensearch", tail=50, user=_user())++ assert isinstance(result, LogsResponse)+ assert result.component == "opensearch"+ assert result.count == 1+ assert len(result.entries) == 1+ e = result.entries[0]+ assert e.level == "error"+ assert e.message == "cluster red"+ assert e.detail == "status=red"+++async def test_empty_buffer_returns_zero_count():+ result = await get_component_logs_endpoint(component="docling", tail=50, user=_user())++ assert result.component == "docling"+ assert result.count == 0+ assert result.entries == []+++async def test_tail_parameter_limits_entries():+ for i in range(20):+ cl.record("openrag", "warning", f"msg-{i}")++ result = await get_component_logs_endpoint(component="openrag", tail=5, user=_user())++ assert result.count == 5+ assert len(result.entries) == 5+ # Should be most recent 5+ assert result.entries[-1].message == "msg-19"+ assert result.entries[0].message == "msg-15"+++async def test_entries_are_oldest_to_newest():+ for i in range(3):+ cl.record("docling", "error", f"err-{i}")++ result = await get_component_logs_endpoint(component="docling", tail=10, user=_user())++ messages = [e.message for e in result.entries]+ assert messages == ["err-0", "err-1", "err-2"]+++async def test_count_matches_entries_length():+ cl.record("opensearch", "error", "a")+ cl.record("opensearch", "error", "b")+ cl.record("opensearch", "error", "c")++ result = await get_component_logs_endpoint(component="opensearch", tail=100, user=_user())++ assert result.count == len(result.entries) == 3+++# ---------------------------------------------------------------------------+# Response schema validation+# ---------------------------------------------------------------------------+++async def test_log_entry_detail_can_be_none():+ cl.record("openrag", "warning", "no detail")++ result = await get_component_logs_endpoint(component="openrag", tail=10, user=_user())++ assert result.entries[0].detail is None+++async def test_langflow_uses_buffer_like_all_other_components():+ """Langflow now goes through the same buffer path as every other component."""+ cl.record("langflow", "error", "langflow-down", detail="ConnectError")++ result = await get_component_logs_endpoint(component="langflow", tail=50, user=_user())++ assert result.component == "langflow"+ assert result.count == 1+ assert result.entries[0].message == "langflow-down"+++async def test_all_known_components_return_200():+ """All four known components must not raise, even with empty buffers."""+ for name in ("openrag", "langflow", "docling", "opensearch"):+ result = await get_component_logs_endpoint(component=name, tail=10, user=_user())+ assert result.component == name+ assert isinstance(result.entries, list)diff --git a/tests/unit/api/test_status_route_gating.py b/tests/unit/api/test_status_route_gating.pynew file mode 100644index 000000000..fe35e5fbc--- /dev/null+++ b/tests/unit/api/test_status_route_gating.py@@ -0,0 +1,171 @@+"""Unit tests asserting that status routes are registered only in OSS run mode.++Route registration reads OPENRAG_RUN_MODE (via is_run_mode_oss()) at the+moment register_*_routes() is called. Because each route module imports+is_run_mode_oss by name at module load time, we must patch the name inside+each route module — not the originating utils module.+"""++import sys+from pathlib import Path++from fastapi import FastAPI++ROOT = Path(__file__).resolve().parent.parent.parent.parent+SRC = ROOT / "src"+if str(SRC) not in sys.path:+ sys.path.insert(0, str(SRC))++import app.routes.internal as internal_routes # noqa: E402+import app.routes.public_v1 as public_v1_routes # noqa: E402+from app.routes.internal import register_internal_routes # noqa: E402+from app.routes.public_v1 import register_public_v1_routes # noqa: E402++# ---------------------------------------------------------------------------+# Helpers+# ---------------------------------------------------------------------------+++def _route_paths(app: FastAPI) -> set[str]:+ """Return the set of URL path strings registered on the app."""+ return {route.path for route in app.routes if hasattr(route, "path")}+++def _matching_routes(app: FastAPI, path: str, method: str) -> list:+ """Return every registered route matching *path* and *method* (no dedup).++ Unlike _route_paths (a set), this preserves duplicate registrations so+ tests can assert exactly one route exists for a given path/method.+ """+ return [+ route+ for route in app.routes+ if getattr(route, "path", None) == path+ and method.upper() in getattr(route, "methods", set())+ ]+++def _make_app_public(monkeypatch, *, oss: bool) -> FastAPI:+ monkeypatch.setattr(public_v1_routes, "is_run_mode_oss", lambda: oss)+ app = FastAPI()+ register_public_v1_routes(app)+ return app+++def _make_app_internal(monkeypatch, *, oss: bool) -> FastAPI:+ monkeypatch.setattr(internal_routes, "is_run_mode_oss", lambda: oss)+ app = FastAPI()+ register_internal_routes(app)+ return app+++# ---------------------------------------------------------------------------+# Public /v1/status routes+# ---------------------------------------------------------------------------+++class TestPublicStatusRoutesOss:+ def test_v1_status_registered(self, monkeypatch):+ app = _make_app_public(monkeypatch, oss=True)+ assert "/v1/status" in _route_paths(app)++ def test_v1_status_logs_registered(self, monkeypatch):+ app = _make_app_public(monkeypatch, oss=True)+ assert "/v1/status/{component}/logs" in _route_paths(app)+++class TestPublicStatusRoutesSaas:+ def test_v1_status_absent(self, monkeypatch):+ app = _make_app_public(monkeypatch, oss=False)+ assert "/v1/status" not in _route_paths(app)++ def test_v1_status_logs_absent(self, monkeypatch):+ app = _make_app_public(monkeypatch, oss=False)+ assert "/v1/status/{component}/logs" not in _route_paths(app)+++class TestPublicStatusRoutesOnPrem:+ """on_prem is not OSS — routes must be absent."""++ def test_v1_status_absent(self, monkeypatch):+ app = _make_app_public(monkeypatch, oss=False)+ assert "/v1/status" not in _route_paths(app)++ def test_v1_status_logs_absent(self, monkeypatch):+ app = _make_app_public(monkeypatch, oss=False)+ assert "/v1/status/{component}/logs" not in _route_paths(app)+++# ---------------------------------------------------------------------------+# Internal /api/status route+# ---------------------------------------------------------------------------+++class TestInternalStatusRouteOss:+ def test_status_registered(self, monkeypatch):+ app = _make_app_internal(monkeypatch, oss=True)+ assert "/status" in _route_paths(app)++ def test_status_registered_exactly_once(self, monkeypatch):+ """Guards against a duplicate GET /status registration silently+ passing the set-based `_route_paths` check above."""+ app = _make_app_internal(monkeypatch, oss=True)+ assert len(_matching_routes(app, "/status", "GET")) == 1+++class TestInternalStatusRouteSaas:+ def test_status_absent(self, monkeypatch):+ app = _make_app_internal(monkeypatch, oss=False)+ assert "/status" not in _route_paths(app)+++class TestInternalStatusRouteOnPrem:+ def test_status_absent(self, monkeypatch):+ app = _make_app_internal(monkeypatch, oss=False)+ assert "/status" not in _route_paths(app)+++# ---------------------------------------------------------------------------+# OSS unset (env default) — is_run_mode_oss() treats unset as "oss"+# ---------------------------------------------------------------------------+++class TestPublicStatusDefaultEnv:+ def test_v1_status_present_when_env_unset(self, monkeypatch):+ """OPENRAG_RUN_MODE unset defaults to oss — routes must be registered."""+ from utils import run_mode_utils++ monkeypatch.delenv("OPENRAG_RUN_MODE", raising=False)+ # Restore the real function so we exercise get_run_mode()'s default logic.+ monkeypatch.setattr(+ public_v1_routes,+ "is_run_mode_oss",+ run_mode_utils.is_run_mode_oss,+ )+ app = FastAPI()+ register_public_v1_routes(app)+ assert "/v1/status" in _route_paths(app)+ assert "/v1/status/{component}/logs" in _route_paths(app)+++# ---------------------------------------------------------------------------+# Non-status routes are always registered regardless of run mode+# ---------------------------------------------------------------------------+++class TestNonStatusRoutesUnaffected:+ def test_chat_route_present_in_saas(self, monkeypatch):+ app = _make_app_public(monkeypatch, oss=False)+ assert "/v1/chat" in _route_paths(app)++ def test_search_route_present_in_saas(self, monkeypatch):+ app = _make_app_public(monkeypatch, oss=False)+ assert "/v1/search" in _route_paths(app)++ def test_health_route_present_in_saas(self, monkeypatch):+ app = _make_app_internal(monkeypatch, oss=False)+ assert "/health" in _route_paths(app)++ def test_chat_route_present_in_oss(self, monkeypatch):+ app = _make_app_public(monkeypatch, oss=True)+ assert "/v1/chat" in _route_paths(app)diff --git a/tests/unit/services/test_component_logs.py b/tests/unit/services/test_component_logs.pynew file mode 100644index 000000000..35660dd6f--- /dev/null+++ b/tests/unit/services/test_component_logs.py@@ -0,0 +1,183 @@+"""Unit tests for services.component_logs.++Covers: ring buffer behaviour, record_check_result() transition logic,+get_entries() tail slicing, redaction of sensitive values and Bearer tokens.+"""++import sys+from pathlib import Path++import pytest++ROOT = Path(__file__).resolve().parent.parent.parent.parent+SRC = ROOT / "src"+if str(SRC) not in sys.path:+ sys.path.insert(0, str(SRC))++import services.component_logs as cl # noqa: E402++[email protected](autouse=True)+def _reset_buffer():+ """Clear all buffer state before every test so tests are independent."""+ cl._buffers.clear()+ cl._locks.clear()+ cl._last_ok.clear()+ yield+ cl._buffers.clear()+ cl._locks.clear()+ cl._last_ok.clear()+++# ---------------------------------------------------------------------------+# record() and get_entries() basics+# ---------------------------------------------------------------------------+++def test_record_stores_entry():+ cl.record("langflow", "error", "down", detail="ConnectError")+ entries = cl.get_entries("langflow", tail=10)+ assert len(entries) == 1+ e = entries[0]+ assert e["level"] == "error"+ assert e["message"] == "down"+ assert e["detail"] == "ConnectError"+ assert "timestamp" in e+++def test_get_entries_unknown_component_returns_empty():+ result = cl.get_entries("nope", tail=50)+ assert result == []+++def test_get_entries_tail_slices_most_recent():+ for i in range(10):+ cl.record("opensearch", "error", f"msg-{i}")+ entries = cl.get_entries("opensearch", tail=3)+ assert len(entries) == 3+ # Last three messages+ assert entries[-1]["message"] == "msg-9"+ assert entries[0]["message"] == "msg-7"+++def test_get_entries_tail_larger_than_buffer_returns_all():+ cl.record("docling", "error", "one")+ cl.record("docling", "error", "two")+ assert len(cl.get_entries("docling", tail=500)) == 2+++def test_buffer_respects_maxlen():+ """Oldest entries are evicted once maxlen is hit."""+ for i in range(cl.BUFFER_MAXLEN + 5):+ cl.record("langflow", "error", f"msg-{i}")+ entries = cl.get_entries("langflow", tail=cl.BUFFER_MAXLEN + 5)+ assert len(entries) == cl.BUFFER_MAXLEN+ # First entry should be msg-5 (the first five were evicted)+ assert entries[0]["message"] == "msg-5"+++def test_entries_oldest_to_newest_order():+ for i in range(5):+ cl.record("openrag", "warning", f"w{i}")+ msgs = [e["message"] for e in cl.get_entries("openrag", tail=10)]+ assert msgs == ["w0", "w1", "w2", "w3", "w4"]+++def test_separate_components_have_independent_buffers():+ cl.record("langflow", "error", "langflow-err")+ cl.record("docling", "error", "docling-err")+ assert cl.get_entries("langflow", 10)[0]["message"] == "langflow-err"+ assert cl.get_entries("docling", 10)[0]["message"] == "docling-err"+ assert len(cl.get_entries("opensearch", 10)) == 0+++# ---------------------------------------------------------------------------+# record_check_result() transition logic+# ---------------------------------------------------------------------------+++def test_failure_always_records():+ cl.record_check_result("langflow", False, "down", detail="err")+ entries = cl.get_entries("langflow", 10)+ assert len(entries) == 1+ assert entries[0]["level"] == "error"+++def test_repeated_failures_all_recorded():+ for _ in range(3):+ cl.record_check_result("langflow", False, "still down")+ assert len(cl.get_entries("langflow", 10)) == 3+++def test_healthy_after_healthy_records_nothing():+ """Steady-state healthy should not write anything."""+ cl.record_check_result("langflow", True, "ok")+ cl.record_check_result("langflow", True, "ok")+ assert cl.get_entries("langflow", 10) == []+++def test_recovery_transition_writes_one_info_entry():+ """False→True transition: exactly one info 'recovered' entry."""+ cl.record_check_result("opensearch", False, "cluster red")+ cl.record_check_result("opensearch", True, "cluster green")+ entries = cl.get_entries("opensearch", 10)+ assert len(entries) == 2 # 1 error + 1 info+ assert entries[0]["level"] == "error"+ assert entries[1]["level"] == "info"+ assert "recovered" in entries[1]["message"]+++def test_no_recovery_entry_when_first_call_is_healthy():+ """The very first call with ok=True (no prior failure) records nothing."""+ cl.record_check_result("docling", True, "reachable")+ assert cl.get_entries("docling", 10) == []+++def test_healthy_after_recovery_records_nothing_more():+ """After a recovery entry, subsequent healthy calls are still silent."""+ cl.record_check_result("langflow", False, "down")+ cl.record_check_result("langflow", True, "up") # recovery → 1 info+ cl.record_check_result("langflow", True, "up") # steady healthy → nothing+ cl.record_check_result("langflow", True, "up")+ assert len(cl.get_entries("langflow", 10)) == 2 # still just error + recovery+++def test_second_failure_after_recovery_is_recorded():+ """Failure → recovery → failure again: second failure should be recorded."""+ cl.record_check_result("opensearch", False, "down1") # error+ cl.record_check_result("opensearch", True, "up") # info recovery+ cl.record_check_result("opensearch", False, "down2") # error again+ entries = cl.get_entries("opensearch", 10)+ assert len(entries) == 3+ levels = [e["level"] for e in entries]+ assert levels == ["error", "info", "error"]+++# ---------------------------------------------------------------------------+# Redaction+# ---------------------------------------------------------------------------+++def test_bearer_token_in_detail_is_redacted():+ cl.record("openrag", "error", "auth failed", detail="Authorization: Bearer abc123token")+ detail = cl.get_entries("openrag", 1)[0]["detail"]+ assert "abc123token" not in detail+ assert "Bearer ***" in detail+++def test_redact_dict_masks_sensitive_keys():+ safe = cl._redact_dict({"api_key": "sk-secret", "message": "hello", "token": "xyz"})+ assert safe["api_key"] == "***"+ assert safe["token"] == "***"+ assert safe["message"] == "hello"+++def test_redact_dict_strips_bearer_from_string_values():+ safe = cl._redact_dict({"info": "header: Bearer supersecret rest"})+ assert "supersecret" not in safe["info"]+ assert "Bearer ***" in safe["info"]+++def test_detail_none_stored_as_none():+ cl.record("docling", "error", "fail", detail=None)+ assert cl.get_entries("docling", 1)[0]["detail"] is Nonediff --git a/tests/unit/services/test_status_checks.py b/tests/unit/services/test_status_checks.pynew file mode 100644index 000000000..2e0f2581f--- /dev/null+++ b/tests/unit/services/test_status_checks.py@@ -0,0 +1,384 @@+from unittest.mock import AsyncMock, MagicMock++import httpx+import pytest++import services.component_logs as _cl+from api.schemas.status import ComponentState+from config.settings import clients+from services import status_checks+from services.status_checks import (+ check_docling,+ check_langflow,+ check_openrag_backend,+ check_opensearch,+)+from utils.version_utils import OPENRAG_VERSION++# OpenRAG backend check tests+++def config_ok(monkeypatch):+ monkeypatch.setattr(status_checks, "get_openrag_config", lambda: object(), raising=True)+++async def test_openrag_all_initialized_is_healthy(monkeypatch, config_ok):+ monkeypatch.setattr(clients, "opensearch", MagicMock(), raising=True)+ monkeypatch.setattr(clients, "langflow_http_client", MagicMock(), raising=True)+ monkeypatch.setattr(clients, "docling_http_client", MagicMock(), raising=True)++ r = await check_openrag_backend()++ assert r.name == "openrag"+ assert r.status == ComponentState.HEALTHY+ assert r.version == OPENRAG_VERSION+++async def test_openrag_missing_client_is_degraded(monkeypatch, config_ok):+ monkeypatch.setattr(clients, "opensearch", None, raising=False)+ monkeypatch.setattr(clients, "langflow_http_client", MagicMock(), raising=False)+ monkeypatch.setattr(clients, "docling_http_client", MagicMock(), raising=False)++ r = await check_openrag_backend()++ assert r.status == ComponentState.DEGRADED+ assert "opensearch" in (r.message or "").lower()+++async def test_openrag_config_not_loaded_is_unhealthy(monkeypatch):+ def _raise():+ raise RuntimeError("config not loaded")++ monkeypatch.setattr(status_checks, "get_openrag_config", _raise, raising=True)++ r = await check_openrag_backend()++ assert r.status == ComponentState.UNHEALTHY+ assert "configuration" in (r.message or "").lower()+++async def test_openrag_latency_is_measured(monkeypatch, config_ok):+ ticks = iter([1000.0, 1000.25])+ monkeypatch.setattr(status_checks, "perf_counter", lambda: next(ticks))+ monkeypatch.setattr(clients, "opensearch", MagicMock(), raising=False)+ monkeypatch.setattr(clients, "langflow_http_client", MagicMock(), raising=False)+ monkeypatch.setattr(clients, "docling_http_client", MagicMock(), raising=False)++ r = await check_openrag_backend()++ assert r.latency_ms == 250+++# Docling check tests+++def _mock_http(status_code=None, raises=None, json_data=None):+ c = MagicMock()+ if raises is not None:+ c.get = AsyncMock(side_effect=raises)+ else:+ resp = MagicMock(spec=httpx.Response)+ resp.status_code = status_code+ resp.json.return_value = (+ json_data+ if json_data is not None+ else {"docling-serve": "1.26.0", "version": "0.11.2rc0"}+ )+ c.get = AsyncMock(return_value=resp)+ return c+++ argnames="status_code,expected_status",+ argvalues=[+ (200, ComponentState.HEALTHY),+ (503, ComponentState.UNHEALTHY),+ ],+)+async def test_docling_correct_status(monkeypatch, status_code, expected_status):+ monkeypatch.setattr(clients, "docling_http_client", _mock_http(status_code), raising=False)++ r = await check_docling()++ assert r.name == "docling"+ assert r.status == expected_status+ assert r.required is True+ if expected_status == ComponentState.HEALTHY:+ assert r.version == "1.26.0"+++async def test_docling_unreachable_is_unhealthy(monkeypatch):+ monkeypatch.setattr(+ clients,+ "docling_http_client",+ _mock_http(raises=httpx.ConnectError("refused")),+ raising=False,+ )+ r = await check_docling()+ assert r.status == ComponentState.UNHEALTHY+ assert "unreachable" in (r.message or "").lower()+ assert r.version is None # guards the unbound-variable regression on the failure path+++# Langflow check tests+++ argnames="status_code,expected_status",+ argvalues=[(200, ComponentState.HEALTHY), (500, ComponentState.UNHEALTHY)],+)+async def test_langflow_correct_status(monkeypatch, status_code, expected_status):+ monkeypatch.setattr(clients, "langflow_http_client", _mock_http(status_code), raising=False)+ r = await check_langflow()+ assert r.name == "langflow"+ assert r.status == expected_status+ if expected_status == ComponentState.HEALTHY:+ assert r.version == "0.11.2rc0"+++async def test_langflow_unreachable_is_unhealthy(monkeypatch):+ monkeypatch.setattr(+ clients,+ "langflow_http_client",+ _mock_http(raises=httpx.ConnectError("refused")),+ raising=False,+ )+ r = await check_langflow()+ assert r.status == ComponentState.UNHEALTHY+ assert "unreachable" in (r.message or "").lower()+ assert r.version is None+++# OpenSearch Check tests+++def _mock_os(health=None, raises=None):+ os = MagicMock()+ os.info = AsyncMock(return_value={"version": {"number": "3.2.0", "distribution": "opensearch"}})+ if raises is not None:+ os.cluster.health = AsyncMock(side_effect=raises)+ else:+ os.cluster.health = AsyncMock(return_value=health)+ return os+++ argnames="os_status,expected_status",+ argvalues=[+ ("green", ComponentState.HEALTHY),+ ("yellow", ComponentState.DEGRADED),+ ("red", ComponentState.UNHEALTHY),+ ],+)+async def test_opensearch_status_is_correct(monkeypatch, os_status, expected_status):+ monkeypatch.setattr(+ clients, "opensearch", _mock_os({"status": os_status, "cluster_name": "c"}), raising=False+ )++ r = await check_opensearch()++ assert r.name == "opensearch"+ assert r.status == expected_status+ assert r.version == "3.2.0"+ assert r.metadata.get("distribution") == "opensearch"+++async def test_opensearch_unreachable_is_unhealthy(monkeypatch):+ monkeypatch.setattr(+ clients, "opensearch", _mock_os(raises=ConnectionError("down")), raising=False+ )+ r = await check_opensearch()+ assert r.status == ComponentState.UNHEALTHY+ assert "unreachable" in (r.message or "").lower()+ assert r.version is None+++# ---------------------------------------------------------------------------+# Buffer recording assertions (added by #2178)+# ---------------------------------------------------------------------------+# Each block verifies that record_check_result() is called correctly by the+# check functions: errors land in the buffer, healthy checks do not.++[email protected](autouse=True)+def _reset_log_buffer():+ _cl._buffers.clear()+ _cl._locks.clear()+ _cl._last_ok.clear()+ yield+ _cl._buffers.clear()+ _cl._locks.clear()+ _cl._last_ok.clear()+++async def test_openrag_unhealthy_records_to_buffer(monkeypatch):+ def _raise():+ raise RuntimeError("config missing")++ monkeypatch.setattr(status_checks, "get_openrag_config", _raise, raising=True)++ r = await check_openrag_backend()++ assert r.status == ComponentState.UNHEALTHY+ entries = _cl.get_entries("openrag", 10)+ assert len(entries) >= 1+ assert entries[-1]["level"] == "error"+ assert entries[-1]["detail"] is not None+ assert "RuntimeError" in entries[-1]["detail"]+++async def test_openrag_degraded_records_to_buffer(monkeypatch):+ monkeypatch.setattr(status_checks, "get_openrag_config", lambda: object(), raising=True)+ monkeypatch.setattr(clients, "opensearch", None, raising=False)+ monkeypatch.setattr(clients, "langflow_http_client", MagicMock(), raising=False)+ monkeypatch.setattr(clients, "docling_http_client", MagicMock(), raising=False)++ r = await check_openrag_backend()++ assert r.status == ComponentState.DEGRADED+ entries = _cl.get_entries("openrag", 10)+ assert len(entries) >= 1+ assert entries[-1]["level"] == "error"+++async def test_openrag_healthy_does_not_flood_buffer(monkeypatch):+ """Steady-state healthy should not write to the buffer on repeated calls."""+ monkeypatch.setattr(status_checks, "get_openrag_config", lambda: object(), raising=True)+ monkeypatch.setattr(clients, "opensearch", MagicMock(), raising=False)+ monkeypatch.setattr(clients, "langflow_http_client", MagicMock(), raising=False)+ monkeypatch.setattr(clients, "docling_http_client", MagicMock(), raising=False)++ await check_openrag_backend()+ await check_openrag_backend()+ await check_openrag_backend()++ # No error entries; buffer is empty (first-healthy → nothing written)+ assert _cl.get_entries("openrag", 10) == []+++async def test_docling_unreachable_records_detail_with_target_url(monkeypatch):+ monkeypatch.setattr(+ clients,+ "docling_http_client",+ _mock_http(raises=httpx.ConnectError("refused")),+ raising=False,+ )++ r = await check_docling()++ assert r.status == ComponentState.UNHEALTHY+ entries = _cl.get_entries("docling", 10)+ assert len(entries) >= 1+ e = entries[-1]+ assert e["level"] == "error"+ assert "ConnectError" in (e["detail"] or "")+ assert "target:" in (e["detail"] or "")+++async def test_docling_healthy_does_not_flood_buffer(monkeypatch):+ monkeypatch.setattr(clients, "docling_http_client", _mock_http(200), raising=False)++ await check_docling()+ await check_docling()++ assert _cl.get_entries("docling", 10) == []+++async def test_langflow_unreachable_records_detail_with_target_url(monkeypatch):+ monkeypatch.setattr(+ clients,+ "langflow_http_client",+ _mock_http(raises=httpx.ConnectError("refused")),+ raising=False,+ )++ r = await check_langflow()++ assert r.status == ComponentState.UNHEALTHY+ entries = _cl.get_entries("langflow", 10)+ assert len(entries) >= 1+ e = entries[-1]+ assert e["level"] == "error"+ assert "ConnectError" in (e["detail"] or "")+ assert "target:" in (e["detail"] or "")+ # last_error on the returned status should match detail prefix+ assert r.last_error is not None+ assert "ConnectError" in r.last_error+++async def test_opensearch_unreachable_records_to_buffer(monkeypatch):+ monkeypatch.setattr(+ clients, "opensearch", _mock_os(raises=ConnectionError("down")), raising=False+ )++ r = await check_opensearch()++ assert r.status == ComponentState.UNHEALTHY+ entries = _cl.get_entries("opensearch", 10)+ assert len(entries) >= 1+ assert entries[-1]["level"] == "error"+ assert "ConnectionError" in (entries[-1]["detail"] or "")+ assert r.last_error is not None+++async def test_opensearch_recovery_writes_info_entry(monkeypatch):+ """After a failure, a subsequent healthy check writes one 'recovered' info entry."""+ # First call: unhealthy+ monkeypatch.setattr(+ clients, "opensearch", _mock_os(raises=ConnectionError("down")), raising=False+ )+ await check_opensearch()++ # Second call: healthy+ monkeypatch.setattr(+ clients, "opensearch", _mock_os({"status": "green", "cluster_name": "c"}), raising=False+ )+ await check_opensearch()++ entries = _cl.get_entries("opensearch", 10)+ levels = [e["level"] for e in entries]+ assert "error" in levels+ assert "info" in levels+ info_entries = [e for e in entries if e["level"] == "info"]+ assert any("recovered" in e["message"] for e in info_entries)+++async def test_last_error_is_none_when_healthy(monkeypatch):+ """last_error on ComponentStatus must be None for a healthy check."""+ monkeypatch.setattr(clients, "docling_http_client", _mock_http(200), raising=False)+ r = await check_docling()+ assert r.status == ComponentState.HEALTHY+ assert r.last_error is None+++async def test_last_error_populated_on_failure(monkeypatch):+ monkeypatch.setattr(+ clients,+ "langflow_http_client",+ _mock_http(raises=httpx.ConnectError("refused")),+ raising=False,+ )+ r = await check_langflow()+ assert r.last_error is not None+ assert len(r.last_error) > 0diff --git a/tests/unit/services/test_status_service.py b/tests/unit/services/test_status_service.pynew file mode 100644index 000000000..ff5338ca1--- /dev/null+++ b/tests/unit/services/test_status_service.py@@ -0,0 +1,144 @@+import asyncio++import pytest++import services.component_logs as _cl+from api.schemas.status import ComponentState, ComponentStatus+from services import status_service+++def _result(name, status, required=True):+ return ComponentStatus(name=name, display_name=name.title(), status=status, required=required)+++async def test_all_healthy_overall_healthy(monkeypatch):+ async def ok(name):+ return _result(name, ComponentState.HEALTHY)++ monkeypatch.setattr(+ status_service,+ "CHECK_SPECS",+ [+ lambda: ok("openrag"),+ lambda: ok("opensearch"),+ ],+ raising=True,+ )++ result = await status_service.aggregate_status()+ assert result.overall_status == ComponentState.HEALTHY+ assert len(result.components) == 2+ assert isinstance(result.components[0], ComponentStatus)+ assert result.checked_at+++async def test_worst_wins(monkeypatch):+ async def healthy():+ return _result("openrag", ComponentState.HEALTHY)++ async def unhealthy():+ return _result("opensearch", ComponentState.UNHEALTHY)++ monkeypatch.setattr(status_service, "CHECK_SPECS", [healthy, unhealthy], raising=True)++ result = await status_service.aggregate_status()+ assert result.overall_status == ComponentState.UNHEALTHY+++async def test_timeout_becomes_unknown_and_does_not_block(monkeypatch):+ async def slow():+ await asyncio.sleep(5)+ return _result("docling", ComponentState.HEALTHY)++ async def fast():+ return _result("openrag", ComponentState.HEALTHY)++ monkeypatch.setattr(status_service, "CHECK_TIMEOUT_S", 0.05, raising=True)+ monkeypatch.setattr(status_service, "CHECK_SPECS", [fast, slow], raising=True)++ result = await status_service.aggregate_status()++ assert len(result.components) == 2+ unknown = [c for c in result.components if c.status == ComponentState.UNKNOWN]+ assert len(unknown) == 1, [(c.name, c.status) for c in result.components]++ assert result.overall_status == ComponentState.UNKNOWN+++async def test_check_exception_becomes_unknown(monkeypatch):+ async def healthy():+ return _result("openrag", ComponentState.HEALTHY)++ async def boom():+ raise RuntimeError("x")++ monkeypatch.setattr(status_service, "CHECK_SPECS", [healthy, boom], raising=True)++ result = await status_service.aggregate_status()++ assert result.overall_status == ComponentState.UNKNOWN+++# ---------------------------------------------------------------------------+# Buffer recording assertions for _run_check (added by #2178)+# ---------------------------------------------------------------------------++[email protected](autouse=True)+def _reset_log_buffer():+ _cl._buffers.clear()+ _cl._locks.clear()+ _cl._last_ok.clear()+ yield+ _cl._buffers.clear()+ _cl._locks.clear()+ _cl._last_ok.clear()+++async def test_timeout_records_error_to_buffer(monkeypatch):+ """A timed-out check must write an error entry to the component buffer."""++ async def slow():+ await asyncio.sleep(5)+ return _result("docling", ComponentState.HEALTHY)++ monkeypatch.setattr(status_service, "CHECK_TIMEOUT_S", 0.05, raising=True)+ monkeypatch.setattr(status_service, "CHECK_SPECS", [slow], raising=True)++ result = await status_service.aggregate_status()++ assert result.components[0].status == ComponentState.UNKNOWN+ # _run_check names the component from the function name: "slow" → but we+ # patch the name via fn.__name__. Use the actual function name here.+ entries = _cl.get_entries("slow", tail=10)+ assert len(entries) >= 1+ assert entries[-1]["level"] == "error"+ assert (+ "timeout" in entries[-1]["message"].lower() or "timed out" in entries[-1]["message"].lower()+ )+ assert result.components[0].last_error is not None+++async def test_check_exception_records_error_to_buffer(monkeypatch):+ """An unexpected exception must write an error entry to the component buffer."""++ async def boom():+ raise ValueError("unexpected")++ monkeypatch.setattr(status_service, "CHECK_SPECS", [boom], raising=True)++ result = await status_service.aggregate_status()++ assert result.components[0].status == ComponentState.UNKNOWN+ entries = _cl.get_entries("boom", tail=10)+ assert len(entries) >= 1+ assert entries[-1]["level"] == "error"+ assert "ValueError" in (entries[-1]["detail"] or "")+ assert result.components[0].last_error is not None+ assert "ValueError" in result.components[0].last_error