LLM inference in C/C++
C++ ggml
server: remove loading.html (#25500)
Xuan-Son Nguyena935fbffe1a3
diff --git a/.github/workflows/ui-publish.yml b/.github/workflows/ui-publish.ymlindex c3b7343c65..99a6d8420f 100644--- a/.github/workflows/ui-publish.yml+++ b/.github/workflows/ui-publish.yml@@ -73,4 +73,3 @@ jobs:hf buckets rm ggml-org/${{ env.HF_BUCKET_NAME }}/index.html --yes 2>/dev/null || truehf buckets rm ggml-org/${{ env.HF_BUCKET_NAME }}/bundle.js --yes 2>/dev/null || truehf buckets rm ggml-org/${{ env.HF_BUCKET_NAME }}/bundle.css --yes 2>/dev/null || true- hf buckets rm ggml-org/${{ env.HF_BUCKET_NAME }}/loading.html --yes 2>/dev/null || truediff --git a/tools/server/server-http.cpp b/tools/server/server-http.cppindex bb88dda219..87eee5fd4d 100644--- a/tools/server/server-http.cpp+++ b/tools/server/server-http.cpp@@ -175,6 +175,15 @@ bool server_http_context::init(const common_params & params) {// Middlewares//+ // Frontend paths - all embedded UI assets+ static const std::unordered_set<std::string> frontend_paths = []() {+ std::unordered_set<std::string> paths { "/" };+ for (const llama_ui_asset & a : llama_ui_get_assets()) {+ paths.insert("/" + a.name);+ }+ return paths;+ }();+// Public endpoints - API routes plus all embedded UI assetsstatic const std::unordered_set<std::string> get_public_endpoints = []() {std::unordered_set<std::string> endpoints {@@ -182,11 +191,8 @@ bool server_http_context::init(const common_params & params) {"/v1/health","/models","/v1/models",- "/",};- for (const llama_ui_asset & a : llama_ui_get_assets()) {- endpoints.insert("/" + a.name);- }+ endpoints.insert(frontend_paths.begin(), frontend_paths.end());return endpoints;}();@@ -239,18 +245,9 @@ bool server_http_context::init(const common_params & params) {auto middleware_server_state = [this](const httplib::Request & req, httplib::Response & res) {if (!is_ready.load()) {-#if defined(LLAMA_UI_HAS_ASSETS)- if (const auto tmp = string_split<std::string>(req.path, '.');- req.path == "/" || (!tmp.empty() && tmp.back() == "html")) {- if (const llama_ui_asset * a = llama_ui_find_asset("loading.html")) {- res.status = 503;- res.set_content(reinterpret_cast<const char*>(a->data), a->size, "text/html; charset=utf-8");- return false;- }+ if (frontend_paths.count(req.path)) {+ return true; // frontend asset, allow it to load and show "loading"}-#else- (void)req;-#endif// no endpoints are allowed to be accessed when the server is not ready// this is to prevent any data races or inconsistent statesres.status = 503;diff --git a/tools/ui/embed.cpp b/tools/ui/embed.cppindex cdbb642323..914d51fa1d 100644--- a/tools/ui/embed.cpp+++ b/tools/ui/embed.cpp@@ -187,7 +187,6 @@ int main(int argc, char ** argv) {struct required_check { const char * label; match_fn match; bool found; };required_check checks[] = {{ "index.html", exact("index.html"), false },- { "loading.html", exact("loading.html"), false },{ "manifest.webmanifest", exact("manifest.webmanifest"), false },{ "sw.js", exact("sw.js"), false },{ "build.json", exact("build.json"), false },diff --git a/tools/ui/src/lib/components/app/chat/ChatScreen/ChatScreenServerError.svelte b/tools/ui/src/lib/components/app/chat/ChatScreen/ChatScreenServerError.svelteindex 2a998dbebf..5345f19a57 100644--- a/tools/ui/src/lib/components/app/chat/ChatScreen/ChatScreenServerError.svelte+++ b/tools/ui/src/lib/components/app/chat/ChatScreen/ChatScreenServerError.svelte@@ -1,10 +1,11 @@<script lang="ts">- import { AlertTriangle, RefreshCw } from '@lucide/svelte';+ import { AlertTriangle, Loader2, RefreshCw } from '@lucide/svelte';import { fadeInView } from '$lib/actions/fade-in-view.svelte';import * as Alert from '$lib/components/ui/alert';- import { serverError, serverLoading, serverStore } from '$lib/stores/server.svelte';+ import { serverError, serverLoading, serverStatus, serverStore } from '$lib/stores/server.svelte';let hasError = $derived(!!serverError());+ let isLoadingModel = $derived(serverStatus() === 503);</script>{#if hasError}@@ -12,23 +13,31 @@class="pointer-events-auto mx-auto mb-4 max-w-[48rem] px-1"use:fadeInView={{ y: 10, duration: 250 }}>- <Alert.Root variant="destructive">- <AlertTriangle class="h-4 w-4" />+ <Alert.Root variant={isLoadingModel ? 'default' : 'destructive'}>+ {#if isLoadingModel}+ <Loader2 class="h-4 w-4 animate-spin" />+ {:else}+ <AlertTriangle class="h-4 w-4" />+ {/if}<Alert.Title class="flex items-center justify-between">- <span>Server unavailable</span>+ <span>{isLoadingModel ? 'Loading model' : 'Server unavailable'}</span>- <button- onclick={() => serverStore.fetch()}- disabled={serverLoading()}- class="flex items-center gap-1.5 rounded-lg bg-destructive/20 px-2 py-1 text-xs font-medium hover:bg-destructive/30 disabled:opacity-50"- >- <RefreshCw class="h-3 w-3 {serverLoading() ? 'animate-spin' : ''}" />- {serverLoading() ? 'Retrying...' : 'Retry'}- </button>+ {#if !isLoadingModel}+ <button+ onclick={() => serverStore.fetch()}+ disabled={serverLoading()}+ class="flex items-center gap-1.5 rounded-lg bg-destructive/20 px-2 py-1 text-xs font-medium hover:bg-destructive/30 disabled:opacity-50"+ >+ <RefreshCw class="h-3 w-3 {serverLoading() ? 'animate-spin' : ''}" />+ {serverLoading() ? 'Retrying...' : 'Retry'}+ </button>+ {/if}</Alert.Title>- <Alert.Description>{serverError()}</Alert.Description>+ {#if !isLoadingModel}+ <Alert.Description>{serverError()}</Alert.Description>+ {/if}</Alert.Root></div>{/if}diff --git a/tools/ui/src/lib/constants/pwa.ts b/tools/ui/src/lib/constants/pwa.tsindex b9171d4bbf..343bcaf3cd 100644--- a/tools/ui/src/lib/constants/pwa.ts+++ b/tools/ui/src/lib/constants/pwa.ts@@ -258,12 +258,6 @@ export const GLOB_PATTERNS: string[] = ['**/*.{js,css,html,ico,svg,png,webp,woff,woff2,json,webmanifest}'];-// loading.html is the model loading page served by llama-server itself.-// The SvelteKit PWA manifest transform strips the html extension from every-// precache entry to match clean URLs, but loading.html is a plain static asset-// with no clean URL, so static servers answer 404 and the SW install fails.-export const GLOB_IGNORES: string[] = ['**/loading.html'];-export const SW_CONFIG = {CHECK_INTERVAL_MS: 60000,UPDATE_FETCH_OPTIONS: {@@ -317,7 +311,6 @@ export const SVELTEKIT_PWA_OPTIONS: SvelteKitPWAOptions = {// Uses '**/' because SvelteKit outputs files under _app/immutable/// subdirectories.globPatterns: GLOB_PATTERNS,- globIgnores: GLOB_IGNORES,maximumFileSizeToCacheInBytes: CACHE_SETTINGS.MAX_FILE_SIZE_BYTES,// Prevent @vite-pwa/sveltekit from auto-adding a NavigationRoute bydiff --git a/tools/ui/src/lib/stores/server.svelte.ts b/tools/ui/src/lib/stores/server.svelte.tsindex d9a9f855a9..66ab411194 100644--- a/tools/ui/src/lib/stores/server.svelte.ts+++ b/tools/ui/src/lib/stores/server.svelte.ts@@ -1,5 +1,8 @@import { PropsService } from '$lib/services/props.service';import { ServerRole } from '$lib/enums';+import { ApiError } from '$lib/utils/api-fetch';++const LOADING_RETRY_INTERVAL_MS = 1000;/*** serverStore - Server connection state, configuration, and role detection@@ -29,8 +32,10 @@ class ServerStore {props = $state<ApiLlamaCppServerProps | null>(null);loading = $state(false);error = $state<string | null>(null);+ status = $state<number | null>(null);role = $state<ServerRole | null>(null);private fetchPromise: Promise<void> | null = null;+ private retryTimer: ReturnType<typeof setTimeout> | null = null;/***@@ -70,23 +75,43 @@ class ServerStore {**/- async fetch(): Promise<void> {+ /**+ * @param background - Set by the automatic "still loading" poll. Skips the+ * `loading` flag flip so the UI doesn't bounce between the full loading+ * splash and the chat screen every retry tick.+ */+ async fetch({ background = false }: { background?: boolean } = {}): Promise<void> {if (this.fetchPromise) return this.fetchPromise;- this.loading = true;- this.error = null;+ this.clearRetryTimer();+ if (!background) {+ this.loading = true;+ }+ // Don't clear an existing "still loading" error before a retry -+ // doing so would unmount/remount the error banner every second.+ if (this.status !== 503) {+ this.error = null;+ }const fetchPromise = (async () => {try {const props = await PropsService.fetch();this.props = props;this.error = null;+ this.status = null;this.detectRole(props);} catch (error: unknown) {this.error = error instanceof Error ? error.message : String(error);+ this.status = error instanceof ApiError ? error.status : null;console.error('Error fetching server properties:', error);++ if (this.status === 503) {+ this.scheduleRetry();+ }} finally {- this.loading = false;+ if (!background) {+ this.loading = false;+ }this.fetchPromise = null;}})();@@ -96,13 +121,30 @@ class ServerStore {}clear(): void {+ this.clearRetryTimer();this.props = null;this.error = null;+ this.status = null;this.loading = false;this.role = null;this.fetchPromise = null;}+ private scheduleRetry(): void {+ if (this.retryTimer) return;+ this.retryTimer = setTimeout(() => {+ this.retryTimer = null;+ this.fetch({ background: true });+ }, LOADING_RETRY_INTERVAL_MS);+ }++ private clearRetryTimer(): void {+ if (this.retryTimer) {+ clearTimeout(this.retryTimer);+ this.retryTimer = null;+ }+ }+/****@@ -125,6 +167,7 @@ export const serverStore = new ServerStore();export const serverProps = () => serverStore.props;export const serverLoading = () => serverStore.loading;export const serverError = () => serverStore.error;+export const serverStatus = () => serverStore.status;export const serverRole = () => serverStore.role;export const defaultParams = () => serverStore.defaultParams;export const contextSize = () => serverStore.contextSize;diff --git a/tools/ui/src/lib/utils/api-fetch.ts b/tools/ui/src/lib/utils/api-fetch.tsindex 82a9383ddf..e9d9062583 100644--- a/tools/ui/src/lib/utils/api-fetch.ts+++ b/tools/ui/src/lib/utils/api-fetch.ts@@ -12,6 +12,21 @@ import { ERROR_MESSAGES, HTTP_CODE_TO_STRING } from '$lib/constants/error';* - Base path resolution*/+/**+ * Error thrown when an API request fails, carrying the HTTP status code+ * so callers can distinguish e.g. a 503 "still loading" response from a+ * genuine failure.+ */+export class ApiError extends Error {+ status: number;++ constructor(message: string, status: number) {+ super(message);+ this.name = 'ApiError';+ this.status = status;+ }+}+export interface ApiFetchOptions extends Omit<RequestInit, 'headers'> {/*** Use auth-only headers (no Content-Type).@@ -67,7 +82,7 @@ export async function apiFetch<T>(path: string, options: ApiFetchOptions = {}):if (!response.ok) {const errorMessage = await parseErrorMessage(response);- throw new Error(errorMessage);+ throw new ApiError(errorMessage, response.status);}return response.json() as Promise<T>;@@ -119,7 +134,7 @@ export async function apiFetchWithParams<T>(if (!response.ok) {const errorMessage = await parseErrorMessage(response);- throw new Error(errorMessage);+ throw new ApiError(errorMessage, response.status);}return response.json() as Promise<T>;diff --git a/tools/ui/static/loading.html b/tools/ui/static/loading.htmldeleted file mode 100644index c3fd19a0f5..0000000000--- a/tools/ui/static/loading.html+++ /dev/null@@ -1,12 +0,0 @@-<!DOCTYPE html>-<html>- <head>- <meta http-equiv="refresh" content="5">- </head>- <body>- <div id="loading">- The model is loading. Please wait.<br/>- The user interface will appear soon.- </div>- </body>-</html>diff --git a/tools/ui/tests/unit/pwa.spec.ts b/tools/ui/tests/unit/pwa.spec.tsindex 17da279866..64b9630d22 100644--- a/tools/ui/tests/unit/pwa.spec.ts+++ b/tools/ui/tests/unit/pwa.spec.ts@@ -189,9 +189,5 @@ describe('PWA Build Output', () => {expect(existsSync(resolve(DIST_DIR, 'pwa-192x192.png'))).toBeTruthy();expect(existsSync(resolve(DIST_DIR, 'pwa-512x512.png'))).toBeTruthy();});-- it('has loading.html fallback page', () => {- expect(existsSync(resolve(DIST_DIR, 'loading.html'))).toBeTruthy();- });});});