A high-performance inference engine for AI models
use std::io::{self, Write};
use anyhow::{Context, Result, bail};
use iocraft::prelude::*;
use nagare::chat::ChatSessionStreamChunk;
use shoji::types::{
basic::ReasoningEffort,
session::chat::{ChatMessage, ChatReplyConfig},
};
use uzu::{
engine::{Engine, EngineConfig},
storage::DownloadPhase,
};
use crate::{
common::thinking::ThinkingSupport,
interactive::{
components::{ApplicationState, TranscriptItem, chat_transcript_component},
sessions::chat::{build_transcript, create_session},
},
};
pub async fn run_non_interactive(
model: String,
message: String,
reasoning_effort: Option<ReasoningEffort>,
seed: Option<i64>,
) -> Result<()> {
let preferences = ApplicationState::load_preferences().unwrap_or_default();
let config = EngineConfig::default();
let engine = Engine::new(config).await?;
let model = engine.model(model.clone()).await?.with_context(|| format!("Model not found: {model}"))?;
if !model.is_chat_capable() {
bail!("--message requires a chat model");
}
if model.is_downloadable() {
let download = engine.download(&model).await?;
let mut announced = false;
while let Some(update) = download.next().await {
if !announced && update.is_in_progress() {
eprintln!("Downloading {}", model.identifier);
announced = true;
}
}
match engine.download_state(&model).await.map(|state| state.phase) {
Some(DownloadPhase::Downloaded {}) => {},
Some(DownloadPhase::Error {
message,
}) => bail!("Model download failed: {}: {message}", model.identifier),
_ => bail!("Model download did not complete: {}", model.identifier),
}
}
let support = ThinkingSupport::for_model(&model);
let effort = match reasoning_effort {
Some(effort) => support
.fulfill_requested_effort(effort)
.map_err(|error| anyhow::anyhow!("Invalid --reasoning-effort: {error}"))?,
None => support.with_preference(&preferences.thinking).reasoning_effort(),
};
let session = create_session(&engine, &model, seed, true).await?;
let mut messages = Vec::new();
if let Some(effort) = effort {
messages.push(ChatMessage::system().with_reasoning_effort(effort));
}
messages.push(ChatMessage::user().with_text(message));
let config = ChatReplyConfig::default().with_sampling_policy(preferences.sampling.policy());
let stream = session.reply_with_stream(messages, config).await;
let cancel_token = stream.cancel_token();
let theme = preferences.theme;
let result = async move {
let mut stdout = io::stdout();
let mut plain = PlainTranscript::default();
let mut stats = None;
while let Some(chunk) = stream.next().await {
match chunk {
ChatSessionStreamChunk::Replies {
replies,
} => {
if let Some(reply) = replies.last() {
stats = Some(reply.stats.clone());
}
},
ChatSessionStreamChunk::ToolResults {
..
} => {},
ChatSessionStreamChunk::Error {
error,
} => return Err(error.into()),
}
let items = build_transcript(&session.messages().await, 0);
plain.write(&mut stdout, items, false)?;
}
let stats = stats.context("No response generated")?;
let items = build_transcript(&session.messages().await, 0);
plain.write(&mut stdout, items, true)?;
writeln!(stdout)?;
chat_transcript_component(
Vec::new(),
Some(stats),
theme.subtitle_color,
theme.overlay_color(),
theme.padding(),
false,
)
.write(&mut stdout)?;
stdout.flush()?;
Ok(())
}
.await;
cancel_token.cancel();
result
}
#[derive(Default)]
struct PlainTranscript {
reasoning: String,
text: String,
last_was_thinking: Option<bool>,
}
impl PlainTranscript {
fn write(
&mut self,
output: &mut impl Write,
items: Vec<TranscriptItem>,
finished: bool,
) -> io::Result<()> {
let mut reasoning = String::new();
let mut text = String::new();
for item in items {
match item {
TranscriptItem::Thinking(value) => reasoning.push_str(&value),
TranscriptItem::Text(value) => text.push_str(&value),
TranscriptItem::ToolCall {
..
} => {},
}
}
if self.last_was_thinking == Some(false)
&& reasoning.trim_end_matches(|ch: char| ch == '\u{fffd}' || ch.is_whitespace()).len()
> self.reasoning.len()
{
self.write_part(output, &text, false, true)?;
}
self.write_part(output, &reasoning, true, finished)?;
if self.last_was_thinking == Some(true) && text.trim_end_matches('\u{fffd}').len() > self.text.len() {
self.write_part(output, &reasoning, true, true)?;
}
self.write_part(output, &text, false, finished)?;
output.flush()
}
fn write_part(
&mut self,
output: &mut impl Write,
text: &str,
is_thinking: bool,
finished: bool,
) -> io::Result<()> {
let previous = if is_thinking {
&mut self.reasoning
} else {
&mut self.text
};
let delta = text.strip_prefix(previous.as_str()).ok_or_else(|| {
io::Error::new(io::ErrorKind::InvalidData, "Model output changed text that was already written")
})?;
let delta = if finished {
delta
} else {
delta.trim_end_matches(|ch: char| ch == '\u{fffd}' || (is_thinking && ch.is_whitespace()))
};
if delta.is_empty() {
return Ok(());
}
if self.last_was_thinking.is_some_and(|last| last != is_thinking) {
writeln!(output)?;
}
output.write_all(delta.as_bytes())?;
previous.push_str(delta);
self.last_was_thinking = Some(is_thinking);
Ok(())
}
}
#[cfg(test)]
#[path = "../../unit/interactive/non_interactive_test.rs"]
mod tests;