Skip to content

armgit / ente

Public Imported from GitHub
0

💚 End-to-end encrypted cloud for everything.

[ensu] Update llama.cpp and optimize Android inference

setalpa75fc16bd0ec
diff --git a/android/apps/ensu/app/build.gradle.kts b/android/apps/ensu/app/build.gradle.kts
index 30ec53f175..f7d0d40b27 100644
--- a/android/apps/ensu/app/build.gradle.kts
+++ b/android/apps/ensu/app/build.gradle.kts
@@ -99,6 +99,7 @@ android {
packaging {
resources { excludes += "/META-INF/{AL2.0,LGPL2.1}" }
jniLibs {
+ useLegacyPackaging = true
pickFirsts +=
setOf(
"lib/arm64-v8a/libc++_shared.so",
diff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/AppViewModel.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/AppViewModel.kt
index 88390d11f7..6a57f9f65c 100644
--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/AppViewModel.kt
+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/AppViewModel.kt
@@ -43,6 +43,7 @@ class AppViewModel(application: Application) : AndroidViewModel(application) {
transcriber = transcriber,
deviceCapabilityProvider = deviceCapabilityProvider,
knowledgeEmbedding = configDefaults.knowledgeEmbedding,
+ nativeLibraryDir = application.applicationInfo.nativeLibraryDir,
)
private val chatRepository = ChatRepository(application, credentialStore)
private val knowledgeProvider = (application as EnsuApplication).knowledgeProvider
diff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatMessages.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatMessages.kt
index deba524767..6ca8152d9d 100644
--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatMessages.kt
+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatMessages.kt
@@ -45,6 +45,7 @@ import androidx.compose.runtime.getValue
import androidx.compose.runtime.mutableIntStateOf
import androidx.compose.runtime.mutableStateOf
import androidx.compose.runtime.remember
+import androidx.compose.runtime.rememberUpdatedState
import androidx.compose.runtime.setValue
import androidx.compose.runtime.snapshotFlow
import androidx.compose.ui.Alignment
@@ -909,6 +910,7 @@ private fun StreamingMessageBubble(
isGenerating: Boolean,
) {
var renderedText by remember { mutableStateOf(text) }
+ val latestText by rememberUpdatedState(text)
var showCursor by remember { mutableStateOf(true) }
val hasText = renderedText.isNotBlank()
val shouldBlink = isGenerating && hasText
@@ -924,18 +926,17 @@ private fun StreamingMessageBubble(
}
}
- LaunchedEffect(text, isGenerating) {
+ LaunchedEffect(isGenerating) {
if (!isGenerating) {
renderedText = ""
return@LaunchedEffect
}
- if (text.isBlank()) {
- renderedText = ""
- return@LaunchedEffect
- }
// Throttle markdown re-rendering while streaming to reduce dropped frames.
- delay(33)
- renderedText = stripHiddenMessageParts(text)
+ snapshotFlow { latestText }
+ .collect { value ->
+ renderedText = stripHiddenMessageParts(value)
+ delay(33)
+ }
}
if (!isGenerating) return
diff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/LlmProvider.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/LlmProvider.kt
index d524e2e297..b3a6024ce6 100644
--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/LlmProvider.kt
+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/LlmProvider.kt
@@ -22,7 +22,7 @@ import io.ente.ensu.bindings.knowledgeEmbeddingModelAsset
import io.ente.ensu.bindings.llmAsset
import io.ente.ensu.bindings.llmCancel
import io.ente.ensu.bindings.llmHasRequiredWorkReserve
-import io.ente.ensu.bindings.llmInitBackend
+import io.ente.ensu.bindings.llmInitBackendFromDirectory
import io.ente.ensu.bindings.llmMemoryBudget
import io.ente.ensu.device.AndroidDeviceCapabilityProvider
import io.ente.ensu.device.requireChatSupported
@@ -55,6 +55,7 @@ class LlmProvider(
private val transcriber: Transcriber,
private val deviceCapabilityProvider: AndroidDeviceCapabilityProvider,
private val knowledgeEmbedding: KnowledgeEmbeddingConfig,
+ private val nativeLibraryDir: String,
private val ioDispatcher: kotlinx.coroutines.CoroutineDispatcher = Dispatchers.IO,
) {
private data class LoadedModelKey(
@@ -394,7 +395,7 @@ class LlmProvider(
model
.newTitleContext(
chatContextSize = checkNotNull(currentContextLength).toUInt(),
- nThreads = max(1, Runtime.getRuntime().availableProcessors() - 1),
+ nThreads = generationThreads(),
)
.use { context ->
coroutine.ensureActive()
@@ -515,7 +516,7 @@ class LlmProvider(
}
}
if (!backendInitialized) {
- llmInitBackend()
+ llmInitBackendFromDirectory(nativeLibraryDir)
backendInitialized = true
}
@@ -568,7 +569,7 @@ class LlmProvider(
val contextParams =
LlmContextParams(
contextSize = currentContextLength,
- nThreads = null,
+ nThreads = generationThreads(),
nBatch = LlmModelSelection(key.id, key.requestedContextLength).batchSize,
)
loadedContext?.destroy()
@@ -616,7 +617,7 @@ class LlmProvider(
deviceCapabilityProvider.chatCapability().requireChatSupported()
val modelKey = LoadedModelKey(selection.id, selection.contextLength)
if (!backendInitialized) {
- llmInitBackend()
+ llmInitBackendFromDirectory(nativeLibraryDir)
backendInitialized = true
}
@@ -678,9 +679,12 @@ class LlmProvider(
fun supportsImages(selection: LlmModelSelection): Boolean =
assetStore.llmMmprojPath(chatAsset(selection)) != null
+ private fun generationThreads(): Int =
+ (Runtime.getRuntime().availableProcessors() - 1).coerceIn(1, 3)
+
private fun loadWithFallbacks(selection: LlmModelSelection, modelFile: File) {
val contexts = selection.contextCandidates()
- val threads = max(1, Runtime.getRuntime().availableProcessors() - 1)
+ val threads = generationThreads()
val batch = selection.batchSize
val modelParams =
diff --git a/android/apps/ensu/rust/build.gradle.kts b/android/apps/ensu/rust/build.gradle.kts
index 59e4c0481b..f2dc740339 100644
--- a/android/apps/ensu/rust/build.gradle.kts
+++ b/android/apps/ensu/rust/build.gradle.kts
@@ -1,3 +1,6 @@
+import groovy.json.JsonOutput
+import groovy.json.JsonSlurper
+import java.io.File
import org.jetbrains.kotlin.gradle.dsl.JvmTarget
plugins {
@@ -46,6 +49,79 @@ fun ndkToolchain(ndkDir: java.io.File): java.io.File =
ndkDir.resolve("toolchains/llvm/prebuilt").listFiles { f -> f.isDirectory }?.singleOrNull()
?: error("Expected exactly one NDK host toolchain in $ndkDir/toolchains/llvm/prebuilt")
+fun packageNativeLibraries(
+ messages: File,
+ output: File,
+ abi: String,
+ readElf: (File, String) -> String,
+) {
+ if (abi == "arm64-v8a") {
+ val cpuBackends =
+ listOf("armv8.0_1", "armv8.2_1", "armv8.2_2", "armv8.6_1")
+ .map { "libggml-cpu-android_$it.so" }
+ .toSet()
+ val packageId = Regex("""(?:^|[/#])llama-cpp-sys-2(?:@| )0\.1\.158(?:$|[ )])""")
+ val parser = JsonSlurper()
+ val builds = messages.useLines { lines ->
+ lines
+ .map {
+ checkNotNull(parser.parseText(it) as? Map<*, *>) { "Invalid Cargo message" }
+ }
+ .filter {
+ it["reason"] == "build-script-executed" &&
+ packageId.containsMatchIn(it["package_id"].toString())
+ }
+ .map {
+ val outDir = checkNotNull(it["out_dir"] as? String) { "Missing Cargo out_dir" }
+ File(outDir)
+ }
+ .toList()
+ }
+ check(builds.size == 1) { "Expected one llama-cpp-sys-2 build, found $builds" }
+ val build = builds.single()
+ for (directory in listOf("lib", "lib64", "backends", "build/common")) {
+ build
+ .resolve(directory)
+ .listFiles { file -> file.extension == "so" }
+ .orEmpty()
+ .filter { !it.name.startsWith("libggml-cpu-") || it.name in cpuBackends }
+ .forEach { it.copyTo(output.resolve(it.name), overwrite = true) }
+ }
+ val required =
+ setOf("llama", "mtmd", "ggml", "ggml-base", "llama-common")
+ .map { "lib$it.so" }
+ .toSet() + cpuBackends
+ val missing = required - output.listFiles().orEmpty().map { it.name }.toSet()
+ check(missing.isEmpty()) { "Missing native libraries: ${missing.sorted()}" }
+ }
+
+ val libraries = output.listFiles { file -> file.extension == "so" }.orEmpty().toList()
+ val names = libraries.map { it.name }.toSet()
+ val system = setOf("libc.so", "libm.so", "libdl.so", "liblog.so", "libandroid.so", "libz.so")
+ val alignment = if (abi in setOf("arm64-v8a", "x86_64")) 16384 else 4096
+ for (library in libraries.sortedBy { it.name }) {
+ val loads =
+ readElf(library, "--program-headers")
+ .lineSequence()
+ .map { it.trim().split(Regex("\\s+")) }
+ .filter { it.first() == "LOAD" }
+ .toList()
+ check(
+ loads.isNotEmpty() && loads.all { it.last().removePrefix("0x").toLong(16) >= alignment }
+ ) {
+ "${library.name}: PT_LOAD alignment is below ${alignment / 1024} KB"
+ }
+ val needed =
+ Regex("""\(NEEDED\).*?\[(.*?)]""")
+ .findAll(readElf(library, "--dynamic"))
+ .map { it.groupValues[1] }
+ .toSet()
+ check(needed.none { "/" in it }) { "${library.name}: absolute DT_NEEDED path" }
+ val missing = needed - names - system
+ check(missing.isEmpty()) { "${library.name}: unpackaged dependencies ${missing.sorted()}" }
+ }
+}
+
fun registerBuildRustJni(
taskName: String,
outputDir: Provider<Directory>,
@@ -82,12 +158,32 @@ fun registerBuildRustJni(
toolchain.absolutePath,
"--out-dir",
outDir.absolutePath,
+ "--messages-dir",
+ temporaryDir.absolutePath,
*abis.toTypedArray(),
)
environment("ANDROID_NDK", ndkDir.absolutePath)
environment("ANDROID_NDK_ROOT", ndkDir.absolutePath)
environment("NDK_ROOT", ndkDir.absolutePath)
}
+
+ doLast {
+ val readelf = ndkToolchain(android.ndkDirectory).resolve("bin/llvm-readelf")
+ for (abi in abis) {
+ packageNativeLibraries(
+ temporaryDir.resolve("$abi.jsonl"),
+ outputDir.get().asFile.resolve(abi),
+ abi,
+ ) { library, option ->
+ providers
+ .exec { commandLine(readelf, option, "--wide", library) }
+ .standardOutput
+ .asText
+ .get()
+ }
+ logger.lifecycle("Packaged and verified native libraries for $abi")
+ }
+ }
}
val buildRustJniDebug =
@@ -138,3 +234,86 @@ dependencies {
api("net.java.dev.jna:jna:5.18.1@aar")
implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.8.0")
}
+
+val checkNativePackaging =
+ tasks.register("checkNativePackaging") {
+ group = "verification"
+ description = "Checks native library selection, dependencies, and page alignment."
+ doLast {
+ val fixture = temporaryDir.apply {
+ deleteRecursively()
+ mkdirs()
+ }
+ fun fixtureFile(path: String, text: String = "test") =
+ fixture.resolve(path).apply {
+ parentFile.mkdirs()
+ writeText(text)
+ }
+ val output = fixture.resolve("jni").apply { mkdirs() }
+ val build = fixture.resolve("current")
+ val messages =
+ fixtureFile(
+ "messages.jsonl",
+ JsonOutput.toJson(
+ mapOf(
+ "reason" to "build-script-executed",
+ "package_id" to
+ "registry+https://github.com/rust-lang/crates.io-index#[email protected]",
+ "out_dir" to build.path,
+ )
+ ),
+ )
+ var alignment = "0x4000"
+ var dependency = "libc.so"
+ fun verify(abi: String, expectedError: String? = null) {
+ val result = runCatching {
+ packageNativeLibraries(messages, output, abi) { _, option ->
+ if (option == "--program-headers")
+ " LOAD 0x0 0x0 0x0 0x1000 0x1000 R E $alignment"
+ else " 0x1 (NEEDED) Shared library: [$dependency]"
+ }
+ }
+ if (expectedError == null) result.getOrThrow()
+ else
+ check(result.exceptionOrNull()?.message?.contains(expectedError) == true) {
+ "Expected '$expectedError', got ${result.exceptionOrNull()}"
+ }
+ }
+ val paths =
+ listOf(
+ "lib/libllama.so",
+ "lib/libmtmd.so",
+ "lib/libggml.so",
+ "lib/libggml-base.so",
+ "build/common/libllama-common.so",
+ ) +
+ listOf("armv8.0_1", "armv8.2_1", "armv8.2_2", "armv8.6_1").map {
+ "backends/libggml-cpu-android_$it.so"
+ }
+ paths.forEach { fixtureFile("current/$it") }
+ fixtureFile("current/backends/libggml-cpu-android_armv9.0_1.so")
+ fixtureFile("stale.so")
+ verify("arm64-v8a")
+ check(
+ output.listFiles().orEmpty().map { it.name }.toSet() ==
+ paths.map { it.substringAfterLast('/') }.toSet()
+ )
+ build.resolve("backends/libggml-cpu-android_armv8.0_1.so").delete()
+ output.resolve("libggml-cpu-android_armv8.0_1.so").delete()
+ verify("arm64-v8a", "Missing native libraries")
+
+ output.deleteRecursively()
+ fixtureFile("jni/libc++_shared.so")
+ alignment = "0x1000"
+ verify("x86_64", "below 16 KB")
+ verify("armeabi-v7a")
+ alignment = "0x4000"
+ dependency = "libggml-base.so"
+ verify("x86_64", "unpackaged dependencies")
+ logger.lifecycle("Native packaging checks passed (4 cases)")
+ }
+ }
+
+tasks
+ .matching { it.name in listOf("check", "lintDebug") }
+ .configureEach { dependsOn(checkNativePackaging) }
diff --git a/android/apps/ensu/rust/scripts/build-rust.sh b/android/apps/ensu/rust/scripts/build-rust.sh
index a74ec89cbf..73394c7465 100755
--- a/android/apps/ensu/rust/scripts/build-rust.sh
+++ b/android/apps/ensu/rust/scripts/build-rust.sh
@@ -9,17 +9,19 @@ REPO_ROOT=$(cd "$(dirname "$0")/../../../../.." && pwd)
TARGET_DIR="$REPO_ROOT/rust/target"
TOOLCHAIN=""
OUT_DIR=""
+MESSAGES_DIR=""
ABIS=()
while [[ $# -gt 0 ]]; do
case $1 in
--toolchain) TOOLCHAIN=$2; shift 2 ;;
--out-dir) OUT_DIR=$2; shift 2 ;;
+ --messages-dir) MESSAGES_DIR=$2; shift 2 ;;
*) ABIS+=("$1"); shift ;;
esac
done
-[[ -n $TOOLCHAIN && -n $OUT_DIR && ${#ABIS[@]} -gt 0 ]] || {
- echo "usage: $(basename "$0") --toolchain DIR --out-dir DIR <abi> [<abi>...]" >&2
+[[ -n $TOOLCHAIN && -n $OUT_DIR && -n $MESSAGES_DIR && ${#ABIS[@]} -gt 0 ]] || {
+ echo "usage: $(basename "$0") --toolchain DIR --out-dir DIR --messages-dir DIR <abi> [<abi>...]" >&2
exit 1
}
@@ -30,10 +32,19 @@ for tool in cargo rustup cmake; do
command -v "$tool" >/dev/null || { echo "error: $tool not on PATH" >&2; exit 1; }
done
+export ANDROID_API_LEVEL=24
+export CMAKE_SHARED_LINKER_FLAGS="${CMAKE_SHARED_LINKER_FLAGS:-} -Wl,-z,max-page-size=16384"
+export CMAKE_MODULE_LINKER_FLAGS="${CMAKE_MODULE_LINKER_FLAGS:-} -Wl,-z,max-page-size=16384"
+mkdir -p "$MESSAGES_DIR"
+
build_abi() {
local abi=$1 target clang_triple libcxx_dir
case $abi in
arm64-v8a)
+ if [[ -n ${GGML_CPU_ARM_ARCH+x} ]]; then
+ echo "error: unset GGML_CPU_ARM_ARCH for runtime-selected ARM64 backends" >&2
+ exit 1
+ fi
target=aarch64-linux-android
clang_triple=aarch64-linux-android24
libcxx_dir=aarch64-linux-android
@@ -68,7 +79,8 @@ build_abi() {
local out=$OUT_DIR/$abi
mkdir -p "$out"
echo "==> $abi"
- (cd "$REPO_ROOT/rust/bindings/uniffi/ensu" && CARGO_TARGET_DIR="$TARGET_DIR" cargo build --release --target "$target")
+ (cd "$REPO_ROOT/rust/bindings/uniffi/ensu" && CARGO_TARGET_DIR="$TARGET_DIR" cargo build --locked --release --target "$target" --message-format=json-render-diagnostics) > "$MESSAGES_DIR/$abi.jsonl"
+ rm -f "$out"/*.so
cp "$TARGET_DIR/$target/release/libensu.so" "$out/"
cp "$TOOLCHAIN/sysroot/usr/lib/$libcxx_dir/libc++_shared.so" "$out/"
}
diff --git a/rust/Cargo.lock b/rust/Cargo.lock
index 85212e12fd..c4a8f5a20c 100644
--- a/rust/Cargo.lock
+++ b/rust/Cargo.lock
@@ -2620,11 +2620,13 @@ dependencies = [
name = "ente-ensu"
version = "0.0.0"
dependencies = [
+ "cc",
"ente-assets",
"ente-core",
"ente-ensu-crypto",
"ente-image",
"llama-cpp-2",
+ "llama-cpp-sys-2",
"ort",
"ort-sys",
"regex",
@@ -5901,11 +5903,10 @@ checksum = "11d3d7f243d5c5a8b9bb5d6dd2b1602c0cb0b9db1621bafc7ed66e35ff9fe092"
[[package]]
name = "llama-cpp-2"
-version = "0.1.144"
+version = "0.1.158"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b2e7a3cc3ed741aa3d5e6e3ff1d84d5ab7ba13bf15c857be06a5d11517767616"
+checksum = "5a89d1d5c09794689aa6190206dd55499d8153b3669c72f2ef24a4e8ddc5ccbe"
dependencies = [
- "encoding_rs",
"enumflags2",
"llama-cpp-sys-2",
"thiserror 2.0.21",
@@ -5915,9 +5916,9 @@ dependencies = [
[[package]]
name = "llama-cpp-sys-2"
-version = "0.1.144"
+version = "0.1.158"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0a2efa5c5675f4e2170c52c4418f71907d39b1e3931fd7566f8382cf9e166282"
+checksum = "608edbeed8bcb8573d8e665f3f047e085202510b7b3d09e7fae2cfe3d80d6bec"
dependencies = [
"bindgen",
"cc",
diff --git a/rust/bindings/uniffi/ensu/src/llm.rs b/rust/bindings/uniffi/ensu/src/llm.rs
index 48d7fb4e4b..64ad6a116a 100644
--- a/rust/bindings/uniffi/ensu/src/llm.rs
+++ b/rust/bindings/uniffi/ensu/src/llm.rs
@@ -346,6 +346,11 @@ pub fn llm_init_backend() -> Result<(), LlmError> {
llm::init_backend().map_err(LlmError::from)
}
+#[uniffi::export]
+pub fn llm_init_backend_from_directory(directory: String) -> Result<(), LlmError> {
+ llm::init_backend_from_directory(&directory).map_err(LlmError::from)
+}
+
#[uniffi::export]
pub fn llm_cancel(job_id: i64) {
llm::cancel(job_id);
diff --git a/rust/checks/lint-exceptions/suppressions.json b/rust/checks/lint-exceptions/suppressions.json
index 97f5c94067..7c169d4f0a 100644
--- a/rust/checks/lint-exceptions/suppressions.json
+++ b/rust/checks/lint-exceptions/suppressions.json
@@ -30,6 +30,7 @@
"rust/crates/core/src/crypto/types.rs": ["clippy::expect_used"],
"rust/crates/ensu/src/config.rs": ["clippy::expect_used"],
"rust/crates/ensu/src/config/knowledge.rs": ["clippy::expect_used"],
+ "rust/crates/ensu/src/llm/chat_template_bridge.rs": ["unsafe_code"],
"rust/crates/ensu/src/llm/generate.rs": ["clippy::expect_used"],
"rust/crates/ensu/src/model/mod.rs": ["clippy::expect_used"],
"rust/crates/ensu/src/platform.rs": ["unsafe_code"],
diff --git a/rust/crates/ensu/Cargo.toml b/rust/crates/ensu/Cargo.toml
index 75695da430..4a3831a9c4 100644
--- a/rust/crates/ensu/Cargo.toml
+++ b/rust/crates/ensu/Cargo.toml
@@ -7,6 +7,7 @@ ente-assets.workspace = true
ente-core.workspace = true
ente-ensu-crypto.workspace = true
ente-image.workspace = true
+llama-cpp-sys-2 = { version = "=0.1.158", default-features = false, features = ["common"] }
regex.workspace = true
rubato = { version = "5.0.0", optional = true }
rusqlite = { workspace = true, default-features = true, optional = true }
@@ -24,20 +25,26 @@ zstd = "0.14.0"
[dev-dependencies]
tempfile.workspace = true
+[build-dependencies]
+cc = "1.2.61"
+
# Keep `mtmd` on every platform for multimodal support.
# macOS disables OpenMP because Apple's clang lacks libomp. llama.cpp uses its
# own thread pool, Accelerate, and Metal on arm64 instead.
[target.'cfg(target_os = "macos")'.dependencies]
-llama-cpp-2 = { version = "=0.1.144", default-features = false, features = ["mtmd"] }
+llama-cpp-2 = { version = "=0.1.158", default-features = false, features = ["common", "mtmd"] }
# Linux Vulkan needs a system Vulkan loader and glslc.
[target.'cfg(target_os = "linux")'.dependencies]
-llama-cpp-2 = { version = "=0.1.144", features = ["mtmd", "vulkan"] }
+llama-cpp-2 = { version = "=0.1.158", features = ["mtmd", "vulkan"] }
# Other platforms stay on CPU. Android keeps the shared C++ runtime; Windows
# avoids Vulkan's VULKAN_SDK requirement.
-[target.'cfg(all(not(target_os = "macos"), not(target_os = "linux")))'.dependencies]
-llama-cpp-2 = { version = "=0.1.144", features = ["mtmd"] }
+[target.'cfg(all(not(target_os = "macos"), not(target_os = "linux"), not(all(target_os = "android", target_arch = "aarch64"))))'.dependencies]
+llama-cpp-2 = { version = "=0.1.158", features = ["mtmd"] }
+
+[target.'cfg(all(target_os = "android", target_arch = "aarch64"))'.dependencies]
+llama-cpp-2 = { version = "=0.1.158", features = ["dynamic-backends", "mtmd"] }
[target.'cfg(target_os = "android")'.dependencies]
ort = { workspace = true, features = ["load-dynamic"], optional = true }
diff --git a/rust/crates/ensu/build.rs b/rust/crates/ensu/build.rs
new file mode 100644
index 0000000000..5df3142f9c
--- /dev/null
+++ b/rust/crates/ensu/build.rs
@@ -0,0 +1,36 @@
+use std::{env, error::Error, fs, path::PathBuf};
+
+fn main() -> Result<(), Box<dyn Error>> {
+ println!("cargo:rerun-if-changed=native/chat.cpp");
+ println!("cargo:rerun-if-env-changed=DEP_LLAMA_GGML_CMAKE_DIR");
+
+ let cmake_dir = PathBuf::from(env::var("DEP_LLAMA_GGML_CMAKE_DIR")?);
+ let output = cmake_dir
+ .parent()
+ .and_then(|path| path.parent())
+ .ok_or("Invalid llama.cpp CMake directory")?;
+ let cache_path = output.join("build/CMakeCache.txt");
+ println!("cargo:rerun-if-changed={}", cache_path.display());
+ let cache = fs::read_to_string(cache_path)?;
+ let source = cache
+ .lines()
+ .find_map(|line| line.strip_prefix("CMAKE_HOME_DIRECTORY:INTERNAL="))
+ .ok_or("llama.cpp source directory missing from CMake cache")?;
+ let source = PathBuf::from(source);
+
+ let mut build = cc::Build::new();
+ build
+ .cpp(true)
+ .std("c++17")
+ .file("native/chat.cpp")
+ .include(source.join("common"))
+ .include(source.join("include"))
+ .include(source.join("ggml/include"))
+ .include(source.join("vendor"))
+ .cpp_link_stdlib(None);
+ if build.get_compiler().is_like_msvc() {
+ build.flag("/EHsc");
+ }
+ build.compile("ensu_chat");
+ Ok(())
+}
diff --git a/rust/crates/ensu/native/chat.cpp b/rust/crates/ensu/native/chat.cpp
new file mode 100644
index 0000000000..08e4f68ec8
--- /dev/null
+++ b/rust/crates/ensu/native/chat.cpp
@@ -0,0 +1,78 @@
+#include "chat.h"
+
+#include <cstdlib>
+#include <cstring>
+#include <exception>
+#include <stdexcept>
+#include <string>
+#include <utility>
+#include <vector>
+
+extern "C" {
+struct ensu_chat_result {
+ char * data;
+ size_t len;
+ int status;
+};
+
+static ensu_chat_result copy_result(const char * text, size_t len, int status) noexcept {
+ auto * data = static_cast<char *>(std::malloc(len + 1));
+ if (!data) {
+ return {nullptr, 0, 2};
+ }
+ std::memcpy(data, text, len);
+ data[len] = '\0';
+ return {data, len, status};
+}
+
+ensu_chat_result ensu_chat_render(const char * source, const char * bos, const char * eos,
+ const char * messages_json, bool add_assistant) noexcept {
+ try {
+ auto messages = common_chat_msgs_parse_oaicompat(common_json::parse(messages_json));
+ std::vector<const char *> builtins(llama_chat_builtin_templates(nullptr, 0));
+ llama_chat_builtin_templates(builtins.data(), builtins.size());
+ for (const auto * builtin : builtins) {
+ if (std::strcmp(source, builtin) != 0 || std::strcmp(source, "chatml") == 0) {
+ continue;
+ }
+ std::vector<llama_chat_message> chat;
+ for (const auto & message : messages) {
+ if (message.role.find('\0') != std::string::npos || message.content.find('\0') != std::string::npos) {
+ throw std::runtime_error("Invalid built-in chat message: embedded NUL");
+ }
+ chat.push_back({message.role.c_str(), message.content.c_str()});
+ }
+ const auto length = llama_chat_apply_template(source, chat.data(), chat.size(), add_assistant, nullptr, 0);
+ if (length < 0) {
+ throw std::runtime_error("Failed to apply built-in chat template");
+ }
+ std::string prompt(length, '\0');
+ const auto written = llama_chat_apply_template(source, chat.data(), chat.size(), add_assistant, prompt.data(), length);
+ if (written != length) {
+ throw std::runtime_error("Invalid built-in chat template output length");
+ }
+ return copy_result(prompt.data(), prompt.size(), 0);
+ }
+ auto templates = common_chat_templates_init(nullptr, source[0] ? source : "chatml", bos, eos);
+ common_chat_templates_inputs inputs;
+ inputs.messages = std::move(messages);
+ inputs.add_generation_prompt = add_assistant;
+ inputs.use_jinja = true;
+ inputs.enable_thinking = false;
+ inputs.chat_template_kwargs["enable_thinking"] = "false";
+ inputs.add_bos = false;
+ inputs.add_eos = false;
+ const auto result = common_chat_templates_apply(templates.get(), inputs);
+ return copy_result(result.prompt.data(), result.prompt.size(), 0);
+ } catch (const std::exception & error) {
+ return copy_result(error.what(), std::strlen(error.what()), 1);
+ } catch (...) {
+ constexpr char error[] = "Unknown chat template error";
+ return copy_result(error, sizeof(error) - 1, 1);
+ }
+}
+
+void ensu_chat_free(char * data) noexcept {
+ std::free(data);
+}
+}
diff --git a/rust/crates/ensu/src/llm/chat_template_bridge.rs b/rust/crates/ensu/src/llm/chat_template_bridge.rs
new file mode 100644
index 0000000000..a6e844b76a
--- /dev/null
+++ b/rust/crates/ensu/src/llm/chat_template_bridge.rs
@@ -0,0 +1,80 @@
+use std::ffi::{CString, c_char};
+
+use super::{Error, format_error};
+
+#[repr(C)]
+struct RenderResult {
+ data: *mut c_char,
+ len: usize,
+ status: i32,
+}
+
+#[expect(
+ unsafe_code,
+ reason = "Declarations for the private, synchronous C++ chat bridge"
+)]
+unsafe extern "C" {
+ fn ensu_chat_render(
+ source: *const c_char,
+ bos: *const c_char,
+ eos: *const c_char,
+ messages: *const c_char,
+ add_assistant: bool,
+ ) -> RenderResult;
+ fn ensu_chat_free(data: *mut c_char);
+}
+
+impl Drop for RenderResult {
+ #[expect(
+ unsafe_code,
+ reason = "The bridge allocated this buffer and frees it with the same allocator; null is allowed"
+ )]
+ fn drop(&mut self) {
+ unsafe { ensu_chat_free(self.data) };
+ }
+}
+
+#[expect(
+ unsafe_code,
+ reason = "CString inputs live for the synchronous call; the result owns len initialized bytes until Drop frees them"
+)]
+pub(super) fn render(
+ source: &str,
+ bos: &str,
+ eos: &str,
+ messages: &str,
+ add_assistant: bool,
+) -> Result<String, Error> {
+ let cstring = |value: &str| {
+ CString::new(value)
+ .map_err(|err| Error::InvalidInput(format_error("Invalid template input", err)))
+ };
+ let source = cstring(source)?;
+ let bos = cstring(bos)?;
+ let eos = cstring(eos)?;
+ let messages = cstring(messages)?;
+ let result = unsafe {
+ ensu_chat_render(
+ source.as_ptr(),
+ bos.as_ptr(),
+ eos.as_ptr(),
+ messages.as_ptr(),
+ add_assistant,
+ )
+ };
+ let failure = |message| Error::Llama {
+ op: "Failed to apply chat template",
+ message,
+ };
+ if result.data.is_null() {
+ return Err(failure("Could not allocate template output".to_owned()));
+ }
+ let bytes = unsafe { std::slice::from_raw_parts(result.data.cast::<u8>(), result.len) };
+ let text = std::str::from_utf8(bytes)
+ .map_err(|err| failure(err.to_string()))?
+ .to_owned();
+ if result.status != 0 {
+ return Err(failure(text));
+ }
+ Ok(text)
+}
diff --git a/rust/crates/ensu/src/llm/embed.rs b/rust/crates/ensu/src/llm/embed.rs
index c81fcd6793..fb67fc4b51 100644
--- a/rust/crates/ensu/src/llm/embed.rs
+++ b/rust/crates/ensu/src/llm/embed.rs
@@ -1,5 +1,4 @@
use llama_cpp_2::llama_batch::LlamaBatch;
-use llama_cpp_2::model::AddBos;
use super::Error;
use super::context::LocalContext;
@@ -41,11 +40,8 @@ impl LocalContext {
self.with_context_and_cache_mut(|context, cached_tokens| {
let tokens = context
.model
- .str_to_token(prompt, AddBos::Always)
- .map_err(|err| Error::Llama {
- op: "Embedding tokenization failed",
- message: err.to_string(),
- })?;
+ .vocab()
+ .tokenize(prompt.as_bytes(), true, true);
if tokens.is_empty() {
return Err(Error::InvalidInput(
"embedding prompt produced no tokens".to_string(),
diff --git a/rust/crates/ensu/src/llm/generate.rs b/rust/crates/ensu/src/llm/generate.rs
index b1f6d07e25..54ed1ac82c 100644
--- a/rust/crates/ensu/src/llm/generate.rs
+++ b/rust/crates/ensu/src/llm/generate.rs
@@ -1,9 +1,7 @@
-use llama_cpp_2::TokenToStringError;
use llama_cpp_2::context::LlamaContext;
use llama_cpp_2::llama_batch::LlamaBatch;
-use llama_cpp_2::model::{AddBos, LlamaChatMessage, LlamaChatTemplate, LlamaModel};
+use llama_cpp_2::model::LlamaModel;
use llama_cpp_2::mtmd::{MtmdBitmap, MtmdInputText, mtmd_default_marker};
-use llama_cpp_2::openai::OpenAIChatTemplateParams;
use llama_cpp_2::sampling::LlamaSampler;
use llama_cpp_2::token::LlamaToken;
use serde::{Deserialize, Serialize};
@@ -16,6 +14,7 @@ use std::time::Instant;
use super::context::LocalContext;
use super::event::{EventSink, FinishReason, GenerationEvent, GenerationSummary, JobId};
+use super::template::build_chat_prompt;
use super::{Error, format_error, lock};
static JOB_COUNTER: AtomicI64 = AtomicI64::new(1);
@@ -93,106 +92,27 @@ pub struct ChatRequest {
pub grammar: Option<String>,
}
-fn token_piece_bytes(model: &LlamaModel, token: LlamaToken) -> Result<Vec<u8>, TokenToStringError> {
- match model.token_to_piece_bytes(token, 8, true, None) {
- #[expect(
- clippy::expect_used,
- reason = "llama.cpp reports insufficient buffer capacity as a negative byte count"
- )]
- Err(TokenToStringError::InsufficientBufferSpace(required)) => model.token_to_piece_bytes(
- token,
- (-required)
- .try_into()
- .expect("error buffer size is positive"),
- true,
- None,
- ),
- result => result,
- }
-}
-
fn token_piece_string(model: &LlamaModel, token: LlamaToken) -> Option<String> {
- String::from_utf8(token_piece_bytes(model, token).ok()?).ok()
-}
-
-fn build_chat_prompt(
- model: &LlamaModel,
- messages: Vec<ChatMessage>,
- template_override: Option<String>,
- add_assistant: bool,
-) -> Result<String, Error> {
- let template_text = match template_override {
- Some(template) => template,
- None => model
- .chat_template(None)
- .ok()
- .and_then(|template| template.to_string().ok())
- .unwrap_or_else(|| "chatml".to_string()),
- };
- let template = LlamaChatTemplate::new(&template_text)
- .map_err(|err| Error::InvalidInput(format_error("Invalid chat template", err)))?;
-
- if template_text.contains("enable_thinking") {
- let messages_json = serde_json::to_string(&messages)
- .map_err(|err| Error::InvalidInput(format_error("Invalid chat messages", err)))?;
- let params = OpenAIChatTemplateParams {
- messages_json: &messages_json,
- tools_json: None,
- tool_choice: None,
- json_schema: None,
- grammar: None,
- reasoning_format: None,
- chat_template_kwargs: Some(r#"{"enable_thinking":false}"#),
- add_generation_prompt: add_assistant,
- use_jinja: true,
- parallel_tool_calls: false,
- enable_thinking: false,
- add_bos: false,
- add_eos: false,
- parse_tool_calls: false,
- };
- let result = model
- .apply_chat_template_oaicompat(&template, &params)
- .map_err(|err| Error::Llama {
- op: "Failed to apply chat template",
- message: err.to_string(),
- })?;
- return Ok(result.prompt);
+ if token.0 < 0 {
+ return None;
}
-
- let chat_messages = messages
- .into_iter()
- .map(|message| {
- LlamaChatMessage::new(message.role, message.content)
- .map_err(|err| Error::InvalidInput(format_error("Invalid chat message", err)))
- })
- .collect::<Result<Vec<_>, Error>>()?;
-
- model
- .apply_chat_template(&template, &chat_messages, add_assistant)
- .map_err(|err| Error::Llama {
- op: "Failed to apply chat template",
- message: err.to_string(),
- })
+ String::from_utf8(model.vocab().token_to_piece(token, true, None)).ok()
}
-fn should_add_bos(model: &LlamaModel, prompt: &str) -> AddBos {
- if let Some(bos) = token_piece_string(model, model.token_bos())
+fn should_add_bos(model: &LlamaModel, prompt: &str) -> bool {
+ if let Some(bos) = token_piece_string(model, model.vocab().bos())
&& !bos.is_empty()
&& prompt.starts_with(&bos)
{
- return AddBos::Never;
+ return false;
}
- AddBos::Always
+ true
}
fn tokenize_text_prompt(model: &LlamaModel, prompt: &str) -> Result<Vec<LlamaToken>, Error> {
let tokens = model
- .str_to_token(prompt, should_add_bos(model, prompt))
- .map_err(|err| Error::Llama {
- op: "Tokenize failed",
- message: err.to_string(),
- })?;
+ .vocab()
+ .tokenize(prompt.as_bytes(), should_add_bos(model, prompt), true);
if tokens.is_empty() {
return Err(Error::InvalidInput("Prompt produced no tokens".to_string()));
}
@@ -334,7 +254,7 @@ impl GenerationJob<'_> {
sampler.accept(token);
check_cancelled(self.cancel_flag)?;
- if ctx.model.is_eog_token(token) {
+ if ctx.model.vocab().is_eog(token) {
self.generated_tokens = self.generated_tokens.saturating_add(1);
self.finish_reason = FinishReason::Eog;
break;
@@ -345,10 +265,7 @@ impl GenerationJob<'_> {
}
self.generated_tokens = self.generated_tokens.saturating_add(1);
- let bytes = token_piece_bytes(ctx.model, token).map_err(|err| Error::Llama {
- op: "Detokenize failed",
- message: err.to_string(),
- })?;
+ let bytes = ctx.model.vocab().token_to_piece(token, true, None);
let step = decoder.push_bytes(&bytes);
if let Some(text) = step.text {
@@ -415,7 +332,11 @@ impl GenerationJob<'_> {
}
}
-fn build_sampler(model: &LlamaModel, request: &SamplingParams) -> Result<LlamaSampler, Error> {
+fn build_sampler(
+ model: &LlamaModel,
+ request: &SamplingParams,
+ context_size: i32,
+) -> Result<LlamaSampler, Error> {
let mut samplers = Vec::new();
let mut repeat_penalty = request.repeat_penalty.unwrap_or(1.0);
@@ -436,7 +357,8 @@ fn build_sampler(model: &LlamaModel, request: &SamplingParams) -> Result<LlamaSa
|| presence_penalty != 0.0
{
samplers.push(LlamaSampler::penalties(
- -1,
+ model.vocab().n_tokens(),
+ context_size,
repeat_penalty,
frequency_penalty,
presence_penalty,
@@ -720,7 +642,7 @@ fn generate_chat_stream(
token_offset = end;
}
- let mut sampler = build_sampler(ctx.model, &sampler_request)?;
+ let mut sampler = build_sampler(ctx.model, &sampler_request, ctx.n_ctx() as i32)?;
sampler.accept_many(prompt_tokens.iter());
let pos = prompt_tokens.len() as i32;
@@ -746,9 +668,11 @@ fn generate_chat_stream(
});
}
let bitmap =
- MtmdBitmap::from_file(&mtmd_ctx, image_path).map_err(|err| Error::Llama {
- op: "Failed to load image",
- message: err.to_string(),
+ MtmdBitmap::from_file(&mtmd_ctx, image_path, false).map_err(|err| {
+ Error::Llama {
+ op: "Failed to load image",
+ message: err.to_string(),
+ }
})?;
if bitmap.is_audio() {
return Err(Error::Unsupported("Audio inputs are not supported"));
@@ -757,7 +681,7 @@ fn generate_chat_stream(
}
let bitmap_refs = bitmaps.iter().collect::<Vec<_>>();
- let add_special = matches!(should_add_bos(ctx.model, &prompt), AddBos::Always);
+ let add_special = should_add_bos(ctx.model, &prompt);
let input_text = MtmdInputText {
text: prompt,
add_special,
@@ -807,7 +731,7 @@ fn generate_chat_stream(
})?;
check_cancelled(&cancel_flag)?;
- let mut sampler = build_sampler(ctx.model, &sampler_request)?;
+ let mut sampler = build_sampler(ctx.model, &sampler_request, ctx.n_ctx() as i32)?;
let mut prompt_tokens = Vec::new();
for index in 0..chunks.len() {
if let Some(chunk) = chunks.get(index)
diff --git a/rust/crates/ensu/src/llm/mod.rs b/rust/crates/ensu/src/llm/mod.rs
index 5ca1de1576..0d65518b0b 100644
--- a/rust/crates/ensu/src/llm/mod.rs
+++ b/rust/crates/ensu/src/llm/mod.rs
@@ -1,3 +1,4 @@
+mod chat_template_bridge;
mod context;
mod embed;
mod event;
@@ -5,6 +6,7 @@ mod generate;
mod history;
pub mod memory;
mod model;
+mod template;
mod worker;
pub use context::*;
@@ -41,7 +43,15 @@ fn lock<T>(mutex: &Mutex<T>) -> MutexGuard<'_, T> {
}
fn backend() -> Result<&'static LlamaBackend, Error> {
- match BACKEND.get_or_init(|| LlamaBackend::init().map_err(|err| err.to_string())) {
+ #[cfg(all(target_os = "android", target_arch = "aarch64"))]
+ let result = BACKEND.get().ok_or_else(|| {
+ Error::InvalidInput(
+ "Initialize CPU backends from nativeLibraryDir before loading a model".to_owned(),
+ )
+ })?;
+ #[cfg(not(all(target_os = "android", target_arch = "aarch64")))]
+ let result = BACKEND.get_or_init(|| LlamaBackend::init().map_err(|err| err.to_string()));
+ match result {
Ok(backend) => Ok(backend),
Err(err) => Err(Error::Llama {
op: "Failed to initialize backend",
@@ -50,6 +60,42 @@ fn backend() -> Result<&'static LlamaBackend, Error> {
}
}
+pub fn init_backend_from_directory(directory: &str) -> Result<(), Error> {
+ #[cfg(all(target_os = "android", target_arch = "aarch64"))]
+ {
+ let path = std::path::Path::new(directory);
+ if directory.contains('\0') || !path.is_absolute() || !path.is_dir() {
+ return Err(Error::InvalidInput(
+ "Invalid native-library directory".to_owned(),
+ ));
+ }
+ if !path.join("libggml-cpu-android_armv8.0_1.so").is_file() {
+ return Err(Error::InvalidInput(
+ "Baseline CPU backend is missing from nativeLibraryDir".to_owned(),
+ ));
+ }
+ BACKEND.get_or_init(|| {
+ llama_cpp_2::llama_backend::load_backends_from_path(path);
+ if !cpu_backend_available() {
+ return Err(
+ "No compatible CPU backend could be loaded from nativeLibraryDir".to_owned(),
+ );
+ }
+ LlamaBackend::init().map_err(|err| err.to_string())
+ });
+ }
+ #[cfg(not(all(target_os = "android", target_arch = "aarch64")))]
+ let _ = directory;
+ backend().map(|_| ())
+}
+
+#[cfg(all(target_os = "android", target_arch = "aarch64"))]
+fn cpu_backend_available() -> bool {
+ llama_cpp_2::list_llama_ggml_backend_devices()
+ .iter()
+ .any(|device| device.device_type == llama_cpp_2::LlamaBackendDeviceType::Cpu)
+}
+
fn format_error(context: &str, err: impl std::fmt::Display) -> String {
format!("{context}: {err}")
}
diff --git a/rust/crates/ensu/src/llm/model.rs b/rust/crates/ensu/src/llm/model.rs
index 8ab2f23a62..ee1000e9ac 100644
--- a/rust/crates/ensu/src/llm/model.rs
+++ b/rust/crates/ensu/src/llm/model.rs
@@ -30,6 +30,10 @@ impl Model {
model_params = model_params.with_n_gpu_layers(layers);
}
+ if let Some(use_mmap) = params.use_mmap {
+ model_params = model_params.with_use_mmap(use_mmap);
+ }
+
if let Some(use_mlock) = params.use_mlock {
model_params = model_params.with_use_mlock(use_mlock);
}
diff --git a/rust/crates/ensu/src/llm/template.rs b/rust/crates/ensu/src/llm/template.rs
new file mode 100644
index 0000000000..48bfcd0559
--- /dev/null
+++ b/rust/crates/ensu/src/llm/template.rs
@@ -0,0 +1,260 @@
+use llama_cpp_2::model::LlamaModel;
+
+use super::{ChatMessage, Error, chat_template_bridge, format_error};
+
+pub(super) fn build_chat_prompt(
+ model: &LlamaModel,
+ messages: Vec<ChatMessage>,
+ template_override: Option<String>,
+ add_assistant: bool,
+) -> Result<String, Error> {
+ let source = match template_override {
+ Some(template) => template,
+ None => model
+ .chat_template(None)
+ .ok()
+ .and_then(|template| template.to_string().ok())
+ .unwrap_or_else(|| "chatml".to_owned()),
+ };
+ let vocab = model.vocab();
+ let special_token = |token: llama_cpp_2::token::LlamaToken| {
+ if token.0 < 0 {
+ return Ok(String::new());
+ }
+ String::from_utf8(vocab.token_to_piece(token, true, None))
+ .map_err(|err| Error::InvalidInput(format_error("Invalid special token", err)))
+ };
+ render(
+ &source,
+ &special_token(vocab.bos())?,
+ &special_token(vocab.eos())?,
+ &messages,
+ add_assistant,
+ )
+}
+
+fn render(
+ source: &str,
+ bos: &str,
+ eos: &str,
+ messages: &[ChatMessage],
+ add_assistant: bool,
+) -> Result<String, Error> {
+ let messages = serde_json::to_string(messages)
+ .map_err(|err| Error::InvalidInput(format_error("Invalid chat messages", err)))?;
+ chat_template_bridge::render(source, bos, eos, &messages, add_assistant)
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ struct Fixture {
+ model_id: &'static str,
+ template: &'static str,
+ bos: &'static str,
+ eos: &'static str,
+ }
+
+ fn message(role: &str, content: &str) -> ChatMessage {
+ ChatMessage {
+ role: role.to_owned(),
+ content: content.to_owned(),
+ }
+ }
+
+ #[test]
+ fn default_templates_preserve_turns_unicode_images_and_assistant_prefixes() {
+ for fixture in DEFAULT_TEMPLATES {
+ let messages = [
+ message("system", "Be concise."),
+ message("user", "नमस्ते 🐈\n<__media__>"),
+ message("assistant", "A cat."),
+ message("user", "What colour?"),
+ ];
+ let gemma = fixture.model_id.starts_with("gemma");
+ let (start, end, assistant) = if gemma {
+ ("<|turn>", "<turn|>", "model")
+ } else {
+ ("<|im_start|>", "<|im_end|>", "assistant")
+ };
+ let mut expected = fixture.bos.to_owned();
+ for msg in &messages {
+ let role = if msg.role == "assistant" {
+ assistant
+ } else {
+ &msg.role
+ };
+ expected.push_str(&format!("{start}{role}\n{}{end}\n", msg.content));
+ }
+ for add_assistant in [false, true] {
+ let prompt = render(
+ fixture.template,
+ fixture.bos,
+ fixture.eos,
+ &messages,
+ add_assistant,
+ )
+ .unwrap();
+ let expected = if add_assistant {
+ format!("{expected}{start}{assistant}\n")
+ } else {
+ expected.clone()
+ };
+ assert_eq!(
+ prompt, expected,
+ "{}: add_assistant={add_assistant}",
+ fixture.model_id
+ );
+ }
+ }
+ }
+
+ #[test]
+ fn overrides_receive_a_boolean_false_and_special_tokens() {
+ assert_eq!(
+ render("llama2", "", "", &[message("user", "Hi")], true).unwrap(),
+ "[INST] Hi [/INST]"
+ );
+ let template = "{{ bos_token }}{% if enable_thinking is not defined or enable_thinking %}THINKING{% else %}DIRECT{% endif %}:{{ messages[0].content }}{{ eos_token }}{% if add_generation_prompt %}ASSISTANT{% endif %}";
+ let prompt = render(
+ template,
+ "<bos>",
+ "<eos>",
+ &[message("user", "你好\n<__media__>")],
+ true,
+ )
+ .unwrap();
+ assert_eq!(prompt, "<bos>DIRECT:你好\n<__media__><eos>ASSISTANT");
+ }
+
+ #[test]
+ fn template_errors_are_returned_and_subsequent_renders_recover() {
+ assert!(render("{% if %}", "", "", &[message("user", "Hi")], true).is_err());
+ assert!(render("bad\0template", "", "", &[], true).is_err());
+ assert!(render("llama2", "", "", &[message("user", "Hi\0there")], true).is_err());
+ let messages = [message("user", "Hello")];
+ for template in ["", "chatml"] {
+ assert_eq!(
+ render(template, "", "", &messages, true).unwrap(),
+ "<|im_start|>user\nHello<|im_end|>\n<|im_start|>assistant\n"
+ );
+ }
+ }
+
+ const DEFAULT_TEMPLATES: &[Fixture] = &[
+ Fixture {
+ model_id: "lfm-vl-1.6b",
+ template: concat!(
+ "{{- bos_token -}}\n{%- set keep_past_thinking = keep_past_thinking | default(false) -%}\n{%- set ns = namespace(system_prompt=\"\") -%}\n{%- if messages[0][\"role\"] == \"system\" -%}\n {%- set sys_content = ",
+ "messages[0][\"content\"] -%}\n {%- if sys_content is not string -%}\n {%- for item in sys_content -%}\n {%- if item[\"type\"] == \"text\" -%}\n {%- set ns.system_prompt = ns.system_prompt + ",
+ "item[\"text\"] -%}\n {%- endif -%}\n {%- endfor -%}\n {%- else -%}\n {%- set ns.system_prompt = sys_content -%}\n {%- endif -%}\n {%- set messages = messages[1:] -%}\n{%- endif -%}\n{%- if tools ",
+ "-%}\n {%- set ns.system_prompt = ns.system_prompt + (\"\\n\" if ns.system_prompt else \"\") + \"List of tools: [\" -%}\n {%- for tool in tools -%}\n {%- if tool is not string -%}\n {%- set tool = tool | ",
+ "tojson -%}\n {%- endif -%}\n {%- set ns.system_prompt = ns.system_prompt + tool -%}\n {%- if not loop.last -%}\n {%- set ns.system_prompt = ns.system_prompt + \", \" -%}\n {%- endif -%}\n ",
+ " {%- endfor -%}\n {%- set ns.system_prompt = ns.system_prompt + \"]\" -%}\n{%- endif -%}\n{%- if ns.system_prompt -%}\n {{- \"<|im_start|>system\\n\" + ns.system_prompt + \"<|im_end|>\\n\" -}}\n{%- endif -%}\n{%- set ",
+ "ns.last_assistant_index = -1 -%}\n{%- for message in messages -%}\n {%- if message[\"role\"] == \"assistant\" -%}\n {%- set ns.last_assistant_index = loop.index0 -%}\n {%- endif -%}\n{%- endfor -%}\n{%- for message ",
+ "in messages -%}\n {{- \"<|im_start|>\" + message[\"role\"] + \"\\n\" -}}\n {%- set content = message[\"content\"] -%}\n {%- if content is not string -%}\n {%- set ns.content = \"\" -%}\n {%- for item in content ",
+ "-%}\n {%- if item[\"type\"] == \"image\" -%}\n {%- set ns.content = ns.content + \"<image>\" -%}\n {%- elif item[\"type\"] == \"text\" -%}\n {%- set ns.content = ns.content + ",
+ "item[\"text\"] -%}\n {%- else -%}\n {%- set ns.content = ns.content + item | tojson -%}\n {%- endif -%}\n {%- endfor -%}\n {%- set content = ns.content -%}\n {%- endif -%}\n ",
+ " {%- if message[\"role\"] == \"assistant\" and not keep_past_thinking and loop.index0 != ns.last_assistant_index -%}\n {%- if \"</think>\" in content -%}\n {%- set content = content.split(\"</think>\")[-1] | ",
+ "trim -%}\n {%- endif -%}\n {%- endif -%}\n {{- content + \"<|im_end|>\\n\" -}}\n{%- endfor -%}\n{%- if add_generation_prompt -%}\n {{- \"<|im_start|>assistant\\n\" -}}\n{%- endif -%}",
+ ),
+ bos: "<|startoftext|>",
+ eos: "<|im_end|>",
+ },
+ Fixture {
+ model_id: "gemma-4-e2b-q4km",
+ template: concat!(
+ "{%- macro format_parameters(properties, required, filter_keys=false) -%}\n {%- set standard_keys = ['description', 'type', 'properties', 'required', 'nullable'] -%}\n {%- set ns = namespace(found_first=false) -%}\n ",
+ " {%- for key, value in properties | dictsort -%}\n {%- set add_comma = false -%}\n {%- if not filter_keys or key not in standard_keys -%}\n {%- if ns.found_first %},{% endif -%}\n {%- set ",
+ "ns.found_first = true -%}\n {{ key }}:{\n {%- if value['description'] -%}\n description:<|\"|>{{ value['description'] }}<|\"|>\n {%- set add_comma = true -%}\n {%- ",
+ "endif -%}\n {%- if value['type'] | upper == 'STRING' -%}\n {%- if value['enum'] -%}\n {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%}\n ",
+ " enum:{{ format_argument(value['enum']) }}\n {%- endif -%}\n {%- elif value['type'] | upper == 'ARRAY' -%}\n {%- if value['items'] is mapping and value['items'] -%}\n ",
+ " {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%}\n items:{\n {%- set ns_items = namespace(found_first=false) -%}\n {%- for item_key, ",
+ "item_value in value['items'] | dictsort -%}\n {%- if item_value is not none -%}\n {%- if ns_items.found_first %},{% endif -%}\n {%- set ",
+ "ns_items.found_first = true -%}\n {%- if item_key == 'properties' -%}\n properties:{\n {%- if item_value is mapping -%}\n ",
+ " {{- format_parameters(item_value, value['items']['required'] | default([])) -}}\n {%- endif -%}\n }\n {%- elif ",
+ "item_key == 'required' -%}\n required:[\n {%- for req_item in item_value -%}\n <|\"|>{{- req_item -}}<|\"|>\n ",
+ " {%- if not loop.last %},{% endif -%}\n {%- endfor -%}\n ]\n {%- elif item_key == 'type' -%}\n ",
+ " {%- if item_value is string -%}\n type:{{ format_argument(item_value | upper) }}\n {%- else -%}\n type:{{ ",
+ "format_argument(item_value | map('upper') | list) }}\n {%- endif -%}\n {%- else -%}\n {{ item_key }}:{{ format_argument(item_value) ",
+ "}}\n {%- endif -%}\n {%- endif -%}\n {%- endfor -%}\n }\n {%- endif -%}\n {%- endif -%}\n {%- if ",
+ "value['nullable'] %}\n {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%}\n nullable:true\n {%- endif -%}\n {%- if value['type'] | upper == ",
+ "'OBJECT' -%}\n {%- if value['properties'] is defined and value['properties'] is mapping -%}\n {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%}\n ",
+ " properties:{\n {{- format_parameters(value['properties'], value['required'] | default([])) -}}\n }\n {%- elif value is mapping -%}\n {%- if add_comma ",
+ "%},{%- else -%} {%- set add_comma = true -%} {% endif -%}\n properties:{\n {{- format_parameters(value, value['required'] | default([]), filter_keys=true) -}}\n }\n ",
+ " {%- endif -%}\n {%- if value['required'] -%}\n {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%}\n required:[\n ",
+ "{%- for item in value['required'] | default([]) -%}\n <|\"|>{{- item -}}<|\"|>\n {%- if not loop.last %},{% endif -%}\n {%- endfor -%}\n ]\n ",
+ " {%- endif -%}\n {%- endif -%}\n {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%}\n type:<|\"|>{{ value['type'] | upper }}<|\"|>}\n {%- endif -%}\n ",
+ " {%- endfor -%}\n{%- endmacro -%}\n{%- macro format_function_declaration(tool_data) -%}\n declaration:{{- tool_data['function']['name'] -}}{description:<|\"|>{{- tool_data['function']['description'] -}}<|\"|>\n {%- set ",
+ "params = tool_data['function']['parameters'] -%}\n {%- if params -%}\n ,parameters:{\n {%- if params['properties'] -%}\n properties:{ {{- format_parameters(params['properties'], ",
+ "params['required']) -}} },\n {%- endif -%}\n {%- if params['required'] -%}\n required:[\n {%- for item in params['required'] -%}\n <|\"|>{{- item -}}<|\"|>\n {{- ",
+ "',' if not loop.last -}}\n {%- endfor -%}\n ],\n {%- endif -%}\n {%- if params['type'] -%}\n type:<|\"|>{{- params['type'] | upper -}}<|\"|>}\n {%- endif -%}\n {%- endif ",
+ "-%}\n {%- if 'response' in tool_data['function'] -%}\n {%- set response_declaration = tool_data['function']['response'] -%}\n ,response:{\n {%- if response_declaration['description'] -%}\n ",
+ "description:<|\"|>{{- response_declaration['description'] -}}<|\"|>,\n {%- endif -%}\n {%- if response_declaration['type'] | upper == 'OBJECT' -%}\n type:<|\"|>{{- response_declaration['type'] | upper ",
+ "-}}<|\"|>}\n {%- endif -%}\n {%- endif -%}\n }\n{%- endmacro -%}\n{%- macro format_argument(argument, escape_keys=True) -%}\n {%- if argument is none -%}\n {{- 'null' -}}\n {%- elif argument is string ",
+ "-%}\n {{- '<|\"|>' + argument + '<|\"|>' -}}\n {%- elif argument is boolean -%}\n {{- 'true' if argument else 'false' -}}\n {%- elif argument is mapping -%}\n {{- '{' -}}\n {%- set ns = ",
+ "namespace(found_first=false) -%}\n {%- for key, value in argument | dictsort -%}\n {%- if ns.found_first %},{% endif -%}\n {%- set ns.found_first = true -%}\n {%- if escape_keys -%}\n ",
+ " {{- '<|\"|>' + key + '<|\"|>' -}}\n {%- else -%}\n {{- key -}}\n {%- endif -%}\n :{{- format_argument(value, escape_keys=escape_keys) -}}\n {%- endfor -%}\n ",
+ " {{- '}' -}}\n {%- elif argument is sequence -%}\n {{- '[' -}}\n {%- for item in argument -%}\n {{- format_argument(item, escape_keys=escape_keys) -}}\n {%- if not loop.last %},{% ",
+ "endif -%}\n {%- endfor -%}\n {{- ']' -}}\n {%- else -%}\n {{- argument -}}\n {%- endif -%}\n{%- endmacro -%}\n{%- macro strip_thinking(text) -%}\n {%- set ns = namespace(result='') -%}\n {%- for ",
+ "part in text.split('<channel|>') -%}\n {%- if '<|channel>' in part -%}\n {%- set ns.result = ns.result + part.split('<|channel>')[0] -%}\n {%- else -%}\n {%- set ns.result = ns.result + ",
+ "part -%}\n {%- endif -%}\n {%- endfor -%}\n {{- ns.result | trim -}}\n{%- endmacro -%}\n\n{%- macro format_tool_response_block(tool_name, response) -%}\n {{- '<|tool_response>' -}}\n {%- if response is ",
+ "mapping -%}\n {{- 'response:' + tool_name + '{' -}}\n {%- for key, value in response | dictsort -%}\n {{- key -}}:{{- format_argument(value, escape_keys=False) -}}\n {%- if not loop.last ",
+ "%},{% endif -%}\n {%- endfor -%}\n {{- '}' -}}\n {%- else -%}\n {{- 'response:' + tool_name + '{value:' + format_argument(response, escape_keys=False) + '}' -}}\n {%- endif -%}\n {{- ",
+ "'<tool_response|>' -}}\n{%- endmacro -%}\n\n{#- ===== SETUP ===== -#}\n{%- set ns = namespace(prev_message_type=None, prev_non_tool_role=None) -%}\n{%- set loop_messages = messages -%}\n{%- set enable_thinking = ",
+ "enable_thinking | default(false) -%}\n{%- set preserve_thinking = preserve_thinking | default(false) -%}\n{{- bos_token -}}\n{#- Handle System/Tool Definitions Block -#}\n{%- if enable_thinking or tools or (messages and ",
+ "messages[0]['role'] in ['system', 'developer']) -%}\n {{- '<|turn>system\\n' -}}\n {#- Inject Thinking token at the very top of the FIRST system turn -#}\n {%- if enable_thinking -%}\n {{- '<|think|>\\n' -}}\n ",
+ " {%- set ns.prev_message_type = 'think' -%}\n {%- endif -%}\n {%- if messages and messages[0]['role'] in ['system', 'developer'] -%}\n {%- if messages[0]['content'] is string -%}\n {{- ",
+ "messages[0]['content'] | trim -}}\n {%- elif messages[0]['content'] is sequence -%}\n {%- for item in messages[0]['content'] -%}\n {{- item['text'] | trim + ' '-}}\n {%- endfor ",
+ "-%}\n {%- endif -%}\n {%- set loop_messages = messages[1:] -%}\n {%- endif -%}\n {%- if tools -%}\n {%- for tool in tools %}\n {{- '<|tool>' -}}\n {{- ",
+ "format_function_declaration(tool) | trim -}}\n {{- '<tool|>' -}}\n {%- endfor %}\n {%- set ns.prev_message_type = 'tool' -%}\n {%- endif -%}\n {{- '<turn|>\\n' -}}\n{%- endif %}\n\n{#- Pre-scan: ",
+ "find last user message index for reasoning guard -#}\n{%- set ns_turn = namespace(last_user_idx=-1) -%}\n{%- for i in range(loop_messages | length) -%}\n {%- if loop_messages[i]['role'] == 'user' -%}\n {%- set ",
+ "ns_turn.last_user_idx = i -%}\n {%- endif -%}\n{%- endfor -%}\n\n{#- Loop through messages -#}\n{%- for message in loop_messages -%}\n {%- if message['role'] != 'tool' -%}\n {%- set ns.prev_message_type = None -%}\n ",
+ "{%- set role = 'model' if message['role'] == 'assistant' else message['role'] -%}\n {#- Detect continuation using tracked state — O(1) instead of O(n) backward scan -#}\n {%- set continue_same_model_turn = (role == ",
+ "'model' and ns.prev_non_tool_role == 'assistant') -%}\n {%- if not continue_same_model_turn -%}\n {{- '<|turn>' + role + '\\n' }}\n {%- endif -%}\n\n {#- Render reasoning/reasoning_content as thinking channel ",
+ "-#}\n {%- set thinking_text = message.get('reasoning') or message.get('reasoning_content') -%}\n {%- set thinking_gate = (loop.index0 > ns_turn.last_user_idx) or (preserve_thinking and message.get('tool_calls')) -%}\n",
+ " {%- if thinking_text and thinking_gate -%}\n {{- '<|channel>thought\\n' + thinking_text + '\\n<channel|>' -}}\n {%- endif -%}\n\n {%- if message.get('tool_calls') -%}\n {%- for tool_call ",
+ "in message.get('tool_calls') -%}\n {%- set function = tool_call['function'] -%}\n {{- '<|tool_call>call:' + function['name'] + '{' -}}\n {%- if function['arguments'] ",
+ "is mapping -%}\n {%- set ns_args = namespace(found_first=false) -%}\n {%- for key, value in function['arguments'] | dictsort -%}\n {%- if ",
+ "ns_args.found_first %},{% endif -%}\n {%- set ns_args.found_first = true -%}\n {{- key -}}:{{- format_argument(value, escape_keys=False) -}}\n ",
+ "{%- endfor -%}\n {%- elif function['arguments'] is none -%}\n {%- elif function['arguments'] is string -%}\n {#- Pre-serialized args (e.g. an OpenAI JSON ",
+ "string). We cannot JSON-parse\n portably in-template, so render non-fatally instead of erroring. Strip an\n outer {...} so it composes with the DSL braces rather than ",
+ "double-wrapping.\n Prefer passing arguments as a mapping for exact Gemma DSL. -#}\n {%- set argstr = function['arguments'] | trim -%}\n {%- if ",
+ "argstr[:1] == '{' and argstr[-1:] == '}' -%}\n {{- argstr[1:-1] -}}\n {%- else -%}\n {{- function['arguments'] -}}\n {%- ",
+ "endif -%}\n {%- endif -%}\n {{- '}<tool_call|>' -}}\n {%- endfor -%}\n {%- set ns.prev_message_type = 'tool_call' -%}\n {%- endif -%}\n\n ",
+ " {%- set ns_tr_out = namespace(flag=false) -%}\n {%- if message.get('tool_responses') -%}\n {#- Legacy: tool_responses embedded on the assistant message (Google/Gemma native) -#}\n ",
+ "{%- for tool_response in message.get('tool_responses') -%}\n {{- format_tool_response_block(tool_response['name'] | default('unknown', true), tool_response['response']) -}}\n {%- set ",
+ "ns_tr_out.flag = true -%}\n {%- set ns.prev_message_type = 'tool_response' -%}\n {%- endfor -%}\n {%- elif message.get('tool_calls') -%}\n {#- OpenAI Chat ",
+ "Completions: forward-scan consecutive role:tool messages -#}\n {%- set ns_tool_scan = namespace(stopped=false) -%}\n {%- for k in range(loop.index0 + 1, loop_messages | length) -%}\n ",
+ " {%- if ns_tool_scan.stopped -%}\n {%- elif loop_messages[k]['role'] != 'tool' -%}\n {%- set ns_tool_scan.stopped = true -%}\n {%- else -%}\n ",
+ " {%- set follow = loop_messages[k] -%}\n {#- Resolve tool_call_id to function name -#}\n {%- set ns_tname = namespace(name=follow.get('name') or 'unknown') -%}\n ",
+ " {%- for tc in message.get('tool_calls') -%}\n {%- if tc.get('id') == follow.get('tool_call_id') -%}\n {%- set ns_tname.name = ",
+ "tc['function']['name'] -%}\n {%- endif -%}\n {%- endfor -%}\n {#- Handle content as string or content-parts array -#}\n {%- set ",
+ "tool_body = follow.get('content') -%}\n {%- if tool_body is string -%}\n {{- format_tool_response_block(ns_tname.name, tool_body) -}}\n {%- elif ",
+ "tool_body is sequence and tool_body is not string -%}\n {%- set ns_txt = namespace(s='') -%}\n {%- for part in tool_body -%}\n {%- if ",
+ "part.get('type') == 'text' -%}\n {%- set ns_txt.s = ns_txt.s + (part.get('text') | default('')) -%}\n {%- endif -%}\n {%- endfor ",
+ "-%}\n {{- format_tool_response_block(ns_tname.name, ns_txt.s) -}}\n {%- for part in tool_body -%}\n {%- if part.get('type') in ['image', ",
+ "'image_url'] -%}\n {{- '<|image|>' -}}\n {%- elif part.get('type') in ['audio', 'input_audio'] -%}\n {{- '<|audio|>' -}}\n ",
+ " {%- elif part.get('type') == 'video' -%}\n {{- '<|video|>' -}}\n {%- endif -%}\n {%- endfor -%}\n ",
+ " {%- else -%}\n {{- format_tool_response_block(ns_tname.name, tool_body) -}}\n {%- endif -%}\n {%- set ns_tr_out.flag = true -%}\n ",
+ " {%- set ns.prev_message_type = 'tool_response' -%}\n {%- endif -%}\n {%- endfor -%}\n {%- endif -%}\n\n {%- set captured_content -%}\n ",
+ "{%- if message.get('content') is string -%}\n {%- if role == 'model' -%}\n {{- strip_thinking(message['content']) -}}\n {%- else -%}\n {{- ",
+ "message['content'] | trim -}}\n {%- endif -%}\n {%- elif message.get('content') is sequence -%}\n {%- for item in message['content'] -%}\n {%- if item.get('type') ",
+ "== 'text' -%}\n {%- if role == 'model' -%}\n {{- strip_thinking(item['text']) -}}\n {%- else -%}\n {{- item['text'] | trim ",
+ "-}}\n {%- endif -%}\n {%- elif item.get('type') in ['image', 'image_url'] -%}\n {{- '<|image|>' -}}\n {%- elif item.get('type') in ",
+ "['audio', 'input_audio'] -%}\n {{- '<|audio|>' -}}\n {%- elif item.get('type') == 'video' -%}\n {{- '<|video|>' -}}\n {%- endif -%}\n ",
+ " {%- endfor -%}\n {%- endif -%}\n {%- endset -%}\n\n {{- captured_content -}}\n {%- set has_content = captured_content | trim | length > 0 -%}\n\n {#- Forward-scan: ",
+ "find next non-tool message role for continuation detection -#}\n {%- set next_nt = namespace(role=None, found=false) -%}\n {%- for j in range(loop.index0 + 1, loop_messages | length) -%}\n {%- if ",
+ "not next_nt.found -%}\n {%- if loop_messages[j]['role'] != 'tool' -%}\n {%- set next_nt.role = loop_messages[j]['role'] -%}\n {%- set next_nt.found = true -%}\n ",
+ " {%- endif -%}\n {%- endif -%}\n {%- endfor -%}\n\n {%- set continues_into_next = (\n role == 'model'\n and next_nt.role == 'assistant'\n and (not ",
+ "message.get('tool_calls') or ns_tr_out.flag)\n ) -%}\n\n {%- if ns.prev_message_type == 'tool_call' and not ns_tr_out.flag -%}\n {{- '<|tool_response>' -}}\n {%- elif continues_into_next -%}\n ",
+ " {%- elif not (ns_tr_out.flag and not has_content and not next_nt.found) -%}\n {{- '<turn|>\\n' -}}\n {%- endif -%}\n\n {#- Track previous non-tool role for next iteration (avoids O(n) backward ",
+ "scan) -#}\n {%- set ns.prev_non_tool_role = message['role'] -%}\n {%- endif -%}\n{%- endfor -%}\n\n{%- if add_generation_prompt -%}\n {%- if ns.prev_message_type != 'tool_response' and ns.prev_message_type != ",
+ "'tool_call' -%}\n {{- '<|turn>model\\n' -}}\n {%- elif ns.prev_message_type == 'tool_response' and enable_thinking -%}\n {{- '<|channel>thought\\n' -}}\n {%- endif -%}\n{%- endif -%}\n",
+ ),
+ bos: "<bos>",
+ eos: "<turn|>",
+ },
+ ];
+}
diff --git a/rust/crates/ensu/tests/llm_worker.rs b/rust/crates/ensu/tests/llm_worker.rs
index 611d407332..a1d3b0727a 100644
--- a/rust/crates/ensu/tests/llm_worker.rs
+++ b/rust/crates/ensu/tests/llm_worker.rs
@@ -6,8 +6,16 @@ use std::sync::Arc;
use std::thread;
use ente_ensu::llm::{self, ChatMessage, ChatRequest, Context, ContextParams, GenerationEvent};
+use ente_image::image_compression::{EncodedImageFormat, encode_rgb};
+
+fn init_test_backend() {
+ if let Ok(directory) = std::env::var("ENSU_TEST_BACKEND_DIR") {
+ llm::init_backend_from_directory(&directory).unwrap();
+ }
+}
fn model() -> llm::ModelRef {
+ init_test_backend();
llm::Model::load(llm::ModelLoadParams {
model_path: std::env::var("ENSU_TEST_MODEL").unwrap(),
n_gpu_layers: Some(99),
@@ -150,6 +158,7 @@ fn concurrent_generations_preserve_output_and_can_drop_caller_model() {
#[test]
#[ignore = "requires ENSU_TEST_EMBED_MODEL"]
fn concurrent_embeddings_match_serial_results() {
+ init_test_backend();
let model = llm::Model::load(llm::ModelLoadParams {
model_path: std::env::var("ENSU_TEST_EMBED_MODEL").unwrap(),
n_gpu_layers: Some(99),
@@ -178,3 +187,110 @@ fn concurrent_embeddings_match_serial_results() {
run.join().unwrap();
}
}
+
+fn collect(context: &llm::ContextRef, request: ChatRequest) -> (String, llm::GenerationSummary) {
+ let mut output = String::new();
+ let summary = context
+ .generate_chat_stream(request, &mut |event| {
+ if let GenerationEvent::Text { text, .. } = event {
+ output.push_str(&text);
+ }
+ })
+ .unwrap();
+ assert!(!output.is_empty());
+ for thinking_marker in ["<think>", "<|think|>", "<|channel>thought"] {
+ assert!(
+ !output.contains(thinking_marker),
+ "Unexpected reasoning: {output}"
+ );
+ }
+ (output, summary)
+}
+
+#[test]
+#[ignore = "requires ENSU_TEST_MODEL set to this device's RAM-tier default"]
+fn default_model_counts_truncates_unicode_and_recovers_from_context_limits() {
+ let context = context();
+ let mut chat = request();
+ chat.messages[0].content = "Reply with a short greeting: नमस्ते, 你好, hello 🐈.".to_owned();
+ let measured = context.measure_text_chat_prompt(&chat).unwrap();
+ let (_, summary) = collect(&context, chat.clone());
+ assert_eq!(summary.prompt_tokens.unwrap() as usize, measured);
+
+ chat.messages[0].content = "नमस्ते 🐈 你好 ".repeat(1_000);
+ assert!(matches!(
+ context.generate_chat_stream(chat.clone(), &mut |_| {}),
+ Err(llm::Error::PromptTooLong { .. })
+ ));
+ let truncated = context
+ .truncate_text_chat_messages(chat.messages.clone(), 128)
+ .unwrap();
+ assert!(chat.messages[0].content.starts_with(&truncated[0].content));
+ chat.messages = truncated;
+ assert!(context.measure_text_chat_prompt(&chat).unwrap() <= 128);
+ collect(&context, chat);
+}
+
+#[test]
+#[ignore = "requires ENSU_TEST_MODEL set to this device's RAM-tier default"]
+fn default_model_reuses_multi_turn_cache_and_reloads() {
+ let owner = model();
+ let params = ContextParams {
+ context_size: Some(512),
+ n_threads: Some(4),
+ n_batch: Some(256),
+ };
+ let warm = Context::new(&owner, params.clone()).unwrap();
+ let first = request();
+ let (answer, _) = collect(&warm, first.clone());
+ let mut followup = first;
+ followup.messages.push(ChatMessage {
+ role: "assistant".to_owned(),
+ content: answer,
+ });
+ followup.messages.push(ChatMessage {
+ role: "user".to_owned(),
+ content: "What comes next?".to_owned(),
+ });
+ let (cached, _) = collect(&warm, followup.clone());
+ drop(warm);
+ let fresh = Context::new(&owner, params).unwrap();
+ let (uncached, _) = collect(&fresh, followup);
+ assert_eq!(cached, uncached);
+ drop(fresh);
+ drop(owner);
+ collect(&context(), request());
+}
+
+#[test]
+#[ignore = "requires ENSU_TEST_MODEL and ENSU_TEST_MMPROJ for this device's RAM-tier default"]
+fn default_model_handles_wide_and_tall_images_and_projector_reloads() {
+ let owner = model();
+ let context = Context::new(
+ &owner,
+ ContextParams {
+ context_size: Some(4096),
+ n_threads: Some(4),
+ n_batch: Some(256),
+ },
+ )
+ .unwrap();
+ let mmproj = std::env::var("ENSU_TEST_MMPROJ").unwrap();
+ let directory = tempfile::tempdir().unwrap();
+ for (name, width, height) in [("wide.png", 1024, 96), ("tall.png", 96, 1024)] {
+ let path = directory.path().join(name);
+ let rgb = [255, 0, 0].repeat((width * height) as usize);
+ let bytes = encode_rgb(&rgb, width, height, EncodedImageFormat::Png).unwrap();
+ std::fs::write(&path, bytes).unwrap();
+ context.prewarm_multimodal(mmproj.clone(), None).unwrap();
+ let mut chat = request();
+ chat.messages[0].content = "Describe the colour in this image.".to_owned();
+ chat.image_paths = Some(vec![path.to_str().unwrap().to_owned()]);
+ chat.mmproj_path = Some(mmproj.clone());
+ collect(&context, chat.clone());
+ context.release_multimodal();
+ chat.messages[0].content.push_str("\n<__media__>");
+ collect(&context, chat);
+ collect(&context, request());
+ }
+}