Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

15.2 KB
Raw Blame
use std::path::PathBuf;

use iocraft::prelude::*;
use shoji::types::{
    basic::{ReasoningEffort, SamplingParameters},
    model::{Model, ModelSpecialization},
};
use uzu::{
    engine::Engine,
    storage::{DownloadPhase, DownloadState},
};

use crate::{
    common::thinking::{ThinkingPreference, ThinkingSupport},
    interactive::{
        APP_IDENTIFIER,
        components::{CommandInput, HistoryCell, HistoryCellType, Logo, Preferences, SelectedModel, Theme},
        flows::{ExitFlow, Flow, FlowEvent, FlowRegistry, ModelRegistriesFlow, SettingsFlow, ThemeFlow},
        helpers::SYMBOL_COMMAND,
        keyboard_enhancement::KeyboardEnhancement,
        sessions::{self, SessionState},
    },
};

const HISTORY_LIMIT: usize = 20;

#[derive(Default, Props)]
pub struct ApplicationProps {
    pub engine: Option<Engine>,
    pub model: Option<String>,
    pub reasoning_effort: Option<ReasoningEffort>,
    pub seed: Option<i64>,
    pub no_tools: bool,
}

pub struct ModelState {
    pub model: Model,
    pub download_state: DownloadState,
    pub session_state: Option<Box<dyn SessionState>>,
    pub thinking: ThinkingSupport,
    pub sampling_defaults: SamplingParameters,
    /// The thinking effort is baked into the prompt on the first turn and
    /// cannot be changed afterwards.
    pub thinking_locked: bool,
}

pub struct ApplicationState {
    preferences: Preferences,
    /// Session-scoped thinking override from the command line; never persisted.
    thinking_override: Option<ThinkingPreference>,
    pub seed: Option<i64>,
    pub no_tools: bool,
    pub engine: Engine,
    pub flow: Option<Box<dyn Flow>>,
    pub history: Vec<HistoryCellType>,
    pub registry: FlowRegistry,
    pub model_state: Option<ModelState>,
}

impl ApplicationState {
    pub fn load_preferences() -> Result<Preferences, Box<dyn std::error::Error>> {
        Preferences::load(&Self::settings_file_path("config")?)
    }

    pub fn session_state(&self) -> Option<&dyn SessionState> {
        self.model_state.as_ref().and_then(|model_state| model_state.session_state.as_deref())
    }

    pub fn preferences(&self) -> &Preferences {
        &self.preferences
    }

    pub fn thinking(&self) -> ThinkingPreference {
        self.thinking_override.unwrap_or(self.preferences.thinking)
    }

    pub fn reasoning_effort_override(&self) -> Option<ReasoningEffort> {
        self.thinking_override.map(|preference| preference.level)
    }

    pub fn theme(&self) -> &Theme {
        &self.preferences.theme
    }

    pub fn set_preferences(
        &mut self,
        prefs: &Preferences,
    ) -> Result<(), Box<dyn std::error::Error>> {
        if prefs.thinking != self.preferences.thinking {
            self.thinking_override = None;
        }
        self.preferences = prefs.clone();
        self.preferences.store(&Self::settings_file_path("config")?)
    }

    pub fn set_theme(
        &mut self,
        theme: Theme,
    ) -> Result<(), Box<dyn std::error::Error>> {
        let mut preferences = self.preferences.clone();
        preferences.theme = theme;
        self.set_preferences(&preferences)
    }

    fn settings_file_path(name: &str) -> Result<PathBuf, Box<dyn std::error::Error>> {
        confy::get_configuration_file_path(APP_IDENTIFIER, name).map_err(Into::into)
    }
}

#[component]
pub fn Application(
    props: &ApplicationProps,
    mut hooks: Hooks,
) -> impl Into<AnyElement<'static>> {
    hooks.use_hook(KeyboardEnhancement::default);
    let engine = props.engine.clone().expect("Application requires an engine");

    let state = hooks.use_state(|| ApplicationState {
        engine,
        preferences: ApplicationState::load_preferences().unwrap_or_else(|error| {
            tracing::warn!("Unable to load or migrate CLI preferences: {error}");
            Preferences::default()
        }),
        thinking_override: props.reasoning_effort.map(|effort| ThinkingPreference {
            level: effort,
            enabled: effort != ReasoningEffort::Disabled,
        }),
        seed: props.seed,
        no_tools: props.no_tools,
        flow: None,
        history: Vec::new(),
        registry: FlowRegistry::default()
            .register("model", "Choose the model", false, || Box::new(ModelRegistriesFlow))
            .register("settings", "Configure thinking and sampling", true, || Box::new(SettingsFlow))
            .register("theme", "Choose the theme", false, || Box::new(ThemeFlow))
            .register("exit", "Exit the CLI", false, || Box::new(ExitFlow))
            .register("quit", "Exit the CLI", false, || Box::new(ExitFlow)),
        model_state: None,
    });
    let (width, _) = hooks.use_terminal_size();
    let requested_model = props.model.clone().or_else(|| state.read().preferences().selected_model_id.clone());

    hooks.use_future({
        let engine = state.read().engine.clone();
        let mut state = state;
        async move {
            let Some(identifier) = requested_model else {
                state.write().flow = Some(Box::new(ModelRegistriesFlow));
                return;
            };
            match engine.model(identifier.clone()).await {
                Ok(Some(model)) => {
                    let model_exists = model.is_downloadable()
                        || !model.is_on_device()
                        || matches!(
                            engine.model_path(&model).await,
                            Some(path) if std::path::Path::new(&path).exists()
                        );
                    if !model_exists {
                        state.write().flow = Some(Box::new(ModelRegistriesFlow));
                        return;
                    }
                    state.write().model_state = Some(ModelState {
                        model,
                        download_state: DownloadState::default(),
                        session_state: None,
                        thinking: ThinkingSupport::default(),
                        sampling_defaults: SamplingParameters::default(),
                        thinking_locked: false,
                    });
                },
                Ok(None) => {
                    state.write().history.push(HistoryCellType::CommandResult {
                        result: format!("Unknown model: {}", identifier),
                    });
                    state.write().flow = Some(Box::new(ModelRegistriesFlow));
                },
                Err(error) => {
                    state.write().history.push(HistoryCellType::CommandResult {
                        result: format!("Failed to load model {}: {}", identifier, error),
                    });
                    state.write().flow = Some(Box::new(ModelRegistriesFlow));
                },
            }
        }
    });

    let on_command = hooks.use_async_handler(move |text: String| async move {
        let mut state = state;

        if let Some(name) = text.strip_prefix(SYMBOL_COMMAND) {
            state.write().history.push(HistoryCellType::Command {
                name: name.to_string(),
            });
            let registry = state.read().registry.clone();
            let model_loaded = state.read().model_state.is_some();
            match registry.command(name) {
                Some(command) if command.requires_model && !model_loaded => {
                    state.write().history.push(HistoryCellType::CommandResult {
                        result: "Load a model first to open settings".to_string(),
                    })
                },
                Some(command) => state.write().flow = Some((command.factory)()),
                None => state.write().history.push(HistoryCellType::CommandResult {
                    result: format!("Unknown command: /{}", name),
                }),
            }
            return;
        }

        state.write().history.push(HistoryCellType::Request {
            text: text.clone(),
        });

        let model_with_download_state = state
            .read()
            .model_state
            .as_ref()
            .map(|model_state| (model_state.model.clone(), model_state.download_state.clone()));
        let (model, download_state) = match model_with_download_state {
            Some(pair) => pair,
            None => {
                state.write().history.push(HistoryCellType::CommandResult {
                    result: "No model is selected".to_string(),
                });
                return;
            },
        };

        if model.is_downloadable() && !matches!(download_state.phase, DownloadPhase::Downloaded {}) {
            state.write().history.push(HistoryCellType::CommandResult {
                result: "Model is not downloaded".to_string(),
            });
            let engine = state.read().engine.clone();
            let downloader = engine.downloader(&model);
            let _ = downloader.resume().await;
            return;
        }

        if model.is_chat_capable() {
            let has_running_session = state.read().session_state().is_some_and(SessionState::is_busy);
            if has_running_session {
                return;
            }
            let Some(session) = sessions::chat::ensure_session(state, &model).await else {
                return;
            };
            sessions::chat::run_session(state, session, text).await;
        } else if model.is_classification_capable() {
            let has_running_session = state.read().session_state().is_some_and(SessionState::is_busy);
            if has_running_session {
                return;
            }
            let Some(session) = sessions::classification::ensure_session(state, &model).await else {
                return;
            };
            sessions::classification::run_session(state, session, text).await;
        } else if model.specializations.contains(&ModelSpecialization::Decision {}) {
            let has_running_session = state.read().session_state().is_some_and(SessionState::is_busy);
            if has_running_session {
                return;
            }
            let Some(session) = sessions::decision::ensure_session(state, &model).await else {
                return;
            };
            sessions::decision::run_session(state, session, text).await;
        } else if model.is_text_to_speech_capable() {
            let has_running_session = state.read().session_state().is_some_and(SessionState::is_busy);
            if has_running_session {
                return;
            }
            let Some(runtime) = sessions::text_to_speech::ensure_session(state, &model).await else {
                return;
            };
            sessions::text_to_speech::run_session(state, runtime, text).await;
        } else {
            state.write().history.push(HistoryCellType::CommandResult {
                result: "Model is not supported yet".to_string(),
            });
        }
    });

    hooks.use_terminal_events(move |event| {
        let TerminalEvent::Key(KeyEvent {
            code,
            kind,
            modifiers,
            ..
        }) = event
        else {
            return;
        };
        if kind == KeyEventKind::Release {
            return;
        }

        let is_escape = matches!(code, KeyCode::Esc);
        let is_ctrl_c = matches!(code, KeyCode::Char('c')) && modifiers.contains(KeyModifiers::CONTROL);
        if !is_escape && !is_ctrl_c {
            return;
        }

        let mut state = state;
        let mut state = state.write();

        let consumed_by_session = if let Some(session_state) = state.session_state() {
            if is_escape {
                session_state.interrupt() || session_state.is_busy()
            } else {
                session_state.is_busy()
            }
        } else {
            false
        };
        if consumed_by_session {
            return;
        }

        if state.flow.is_some() {
            if is_escape {
                if matches!(state.history.last(), Some(HistoryCellType::Command { .. })) {
                    state.history.push(HistoryCellType::CommandResult {
                        result: "Cancelled".to_string(),
                    });
                }
                state.flow = None;
            }
            return;
        }

        if is_escape && sessions::decision::leave_question_mode(&mut state) {
            return;
        }

        state.flow = Some(Box::new(ExitFlow));
    });

    let on_flow_event: Handler<FlowEvent> = Handler::from(move |event: FlowEvent| {
        let mut state = state;
        state.write().history.push(HistoryCellType::CommandResult {
            result: event.result,
        });
        state.write().flow = event.next_flow;
    });

    let input_disabled = state.read().session_state().is_some_and(SessionState::is_busy);
    let input_component: AnyElement<'static> = match state.read().flow.as_ref() {
        Some(flow) => {
            let flow_component = flow.render(on_flow_event);
            element! {
                View(flex_direction: FlexDirection::Column) {
                    View(
                        width: 100pct,
                        height: 1u16,
                        border_style: BorderStyle::Single,
                        border_color: state.read().theme().accent_color,
                        border_edges: Some(Edges::Top),
                    )
                    #(flow_component)
                }
            }
            .into()
        },
        None => element! { CommandInput(disabled: input_disabled, on_submit: on_command) }.into(),
    };

    let history_cell_components: Vec<AnyElement<'static>> = state
        .read()
        .history
        .iter()
        .rev()
        .take(HISTORY_LIMIT)
        .cloned()
        .collect::<Vec<_>>()
        .into_iter()
        .rev()
        .map(|r#type| element! { HistoryCell(r#type: Some(r#type)) }.into())
        .collect();

    let pending_reply = {
        let state = state.read();
        state.session_state().and_then(SessionState::pending_history_cell)
    };
    let pending_reply_component: AnyElement<'static> = match pending_reply {
        Some(history_cell) => element! {
            HistoryCell(r#type: Some(history_cell), live: true)
        }
        .into(),
        None => element! { View }.into(),
    };

    let selected_model_component: AnyElement<'static> = match state.read().model_state.as_ref() {
        Some(model_state) => element! { SelectedModel(key: model_state.model.identifier.clone()) }.into(),
        None => element! { View }.into(),
    };

    element! {
        ContextProvider(value: Context::owned(state)) {
            View(
                flex_direction: FlexDirection::Column,
                width: width,
            ) {
                View(
                    padding_left: state.read().theme().padding(),
                    padding_right: state.read().theme().padding(),
                ) {
                    Logo
                }
                View(
                    flex_direction: FlexDirection::Column
                ) {
                    #(history_cell_components.into_iter())
                }
                #(pending_reply_component)
                View(
                    flex_direction: FlexDirection::Column,
                    column_gap: 0,
                ) {
                    View(height: state.read().theme().padding())
                    #(selected_model_component)
                    #(input_component)
                }
            }
        }
    }
}