💚 End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
[ensu] Update llama.cpp and optimize Android inference
setalpa75fc16bd0ec
diff --git a/android/apps/ensu/app/build.gradle.kts b/android/apps/ensu/app/build.gradle.ktsindex 30ec53f175..f7d0d40b27 100644--- a/android/apps/ensu/app/build.gradle.kts+++ b/android/apps/ensu/app/build.gradle.kts@@ -99,6 +99,7 @@ android {packaging {resources { excludes += "/META-INF/{AL2.0,LGPL2.1}" }jniLibs {+ useLegacyPackaging = truepickFirsts +=setOf("lib/arm64-v8a/libc++_shared.so",diff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/AppViewModel.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/AppViewModel.ktindex 88390d11f7..6a57f9f65c 100644--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/AppViewModel.kt+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/AppViewModel.kt@@ -43,6 +43,7 @@ class AppViewModel(application: Application) : AndroidViewModel(application) {transcriber = transcriber,deviceCapabilityProvider = deviceCapabilityProvider,knowledgeEmbedding = configDefaults.knowledgeEmbedding,+ nativeLibraryDir = application.applicationInfo.nativeLibraryDir,)private val chatRepository = ChatRepository(application, credentialStore)private val knowledgeProvider = (application as EnsuApplication).knowledgeProviderdiff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatMessages.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatMessages.ktindex deba524767..6ca8152d9d 100644--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatMessages.kt+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/chat/ChatMessages.kt@@ -45,6 +45,7 @@ import androidx.compose.runtime.getValueimport androidx.compose.runtime.mutableIntStateOfimport androidx.compose.runtime.mutableStateOfimport androidx.compose.runtime.remember+import androidx.compose.runtime.rememberUpdatedStateimport androidx.compose.runtime.setValueimport androidx.compose.runtime.snapshotFlowimport androidx.compose.ui.Alignment@@ -909,6 +910,7 @@ private fun StreamingMessageBubble(isGenerating: Boolean,) {var renderedText by remember { mutableStateOf(text) }+ val latestText by rememberUpdatedState(text)var showCursor by remember { mutableStateOf(true) }val hasText = renderedText.isNotBlank()val shouldBlink = isGenerating && hasText@@ -924,18 +926,17 @@ private fun StreamingMessageBubble(}}- LaunchedEffect(text, isGenerating) {+ LaunchedEffect(isGenerating) {if (!isGenerating) {renderedText = ""return@LaunchedEffect}- if (text.isBlank()) {- renderedText = ""- return@LaunchedEffect- }// Throttle markdown re-rendering while streaming to reduce dropped frames.- delay(33)- renderedText = stripHiddenMessageParts(text)+ snapshotFlow { latestText }+ .collect { value ->+ renderedText = stripHiddenMessageParts(value)+ delay(33)+ }}if (!isGenerating) returndiff --git a/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/LlmProvider.kt b/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/LlmProvider.ktindex d524e2e297..b3a6024ce6 100644--- a/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/LlmProvider.kt+++ b/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/LlmProvider.kt@@ -22,7 +22,7 @@ import io.ente.ensu.bindings.knowledgeEmbeddingModelAssetimport io.ente.ensu.bindings.llmAssetimport io.ente.ensu.bindings.llmCancelimport io.ente.ensu.bindings.llmHasRequiredWorkReserve-import io.ente.ensu.bindings.llmInitBackend+import io.ente.ensu.bindings.llmInitBackendFromDirectoryimport io.ente.ensu.bindings.llmMemoryBudgetimport io.ente.ensu.device.AndroidDeviceCapabilityProviderimport io.ente.ensu.device.requireChatSupported@@ -55,6 +55,7 @@ class LlmProvider(private val transcriber: Transcriber,private val deviceCapabilityProvider: AndroidDeviceCapabilityProvider,private val knowledgeEmbedding: KnowledgeEmbeddingConfig,+ private val nativeLibraryDir: String,private val ioDispatcher: kotlinx.coroutines.CoroutineDispatcher = Dispatchers.IO,) {private data class LoadedModelKey(@@ -394,7 +395,7 @@ class LlmProvider(model.newTitleContext(chatContextSize = checkNotNull(currentContextLength).toUInt(),- nThreads = max(1, Runtime.getRuntime().availableProcessors() - 1),+ nThreads = generationThreads(),).use { context ->coroutine.ensureActive()@@ -515,7 +516,7 @@ class LlmProvider(}}if (!backendInitialized) {- llmInitBackend()+ llmInitBackendFromDirectory(nativeLibraryDir)backendInitialized = true}@@ -568,7 +569,7 @@ class LlmProvider(val contextParams =LlmContextParams(contextSize = currentContextLength,- nThreads = null,+ nThreads = generationThreads(),nBatch = LlmModelSelection(key.id, key.requestedContextLength).batchSize,)loadedContext?.destroy()@@ -616,7 +617,7 @@ class LlmProvider(deviceCapabilityProvider.chatCapability().requireChatSupported()val modelKey = LoadedModelKey(selection.id, selection.contextLength)if (!backendInitialized) {- llmInitBackend()+ llmInitBackendFromDirectory(nativeLibraryDir)backendInitialized = true}@@ -678,9 +679,12 @@ class LlmProvider(fun supportsImages(selection: LlmModelSelection): Boolean =assetStore.llmMmprojPath(chatAsset(selection)) != null+ private fun generationThreads(): Int =+ (Runtime.getRuntime().availableProcessors() - 1).coerceIn(1, 3)+private fun loadWithFallbacks(selection: LlmModelSelection, modelFile: File) {val contexts = selection.contextCandidates()- val threads = max(1, Runtime.getRuntime().availableProcessors() - 1)+ val threads = generationThreads()val batch = selection.batchSizeval modelParams =diff --git a/android/apps/ensu/rust/build.gradle.kts b/android/apps/ensu/rust/build.gradle.ktsindex 59e4c0481b..f2dc740339 100644--- a/android/apps/ensu/rust/build.gradle.kts+++ b/android/apps/ensu/rust/build.gradle.kts@@ -1,3 +1,6 @@+import groovy.json.JsonOutput+import groovy.json.JsonSlurper+import java.io.Fileimport org.jetbrains.kotlin.gradle.dsl.JvmTargetplugins {@@ -46,6 +49,79 @@ fun ndkToolchain(ndkDir: java.io.File): java.io.File =ndkDir.resolve("toolchains/llvm/prebuilt").listFiles { f -> f.isDirectory }?.singleOrNull()?: error("Expected exactly one NDK host toolchain in $ndkDir/toolchains/llvm/prebuilt")+fun packageNativeLibraries(+ messages: File,+ output: File,+ abi: String,+ readElf: (File, String) -> String,+) {+ if (abi == "arm64-v8a") {+ val cpuBackends =+ listOf("armv8.0_1", "armv8.2_1", "armv8.2_2", "armv8.6_1")+ .map { "libggml-cpu-android_$it.so" }+ .toSet()+ val packageId = Regex("""(?:^|[/#])llama-cpp-sys-2(?:@| )0\.1\.158(?:$|[ )])""")+ val parser = JsonSlurper()+ val builds = messages.useLines { lines ->+ lines+ .map {+ checkNotNull(parser.parseText(it) as? Map<*, *>) { "Invalid Cargo message" }+ }+ .filter {+ it["reason"] == "build-script-executed" &&+ packageId.containsMatchIn(it["package_id"].toString())+ }+ .map {+ val outDir = checkNotNull(it["out_dir"] as? String) { "Missing Cargo out_dir" }+ File(outDir)+ }+ .toList()+ }+ check(builds.size == 1) { "Expected one llama-cpp-sys-2 build, found $builds" }+ val build = builds.single()+ for (directory in listOf("lib", "lib64", "backends", "build/common")) {+ build+ .resolve(directory)+ .listFiles { file -> file.extension == "so" }+ .orEmpty()+ .filter { !it.name.startsWith("libggml-cpu-") || it.name in cpuBackends }+ .forEach { it.copyTo(output.resolve(it.name), overwrite = true) }+ }+ val required =+ setOf("llama", "mtmd", "ggml", "ggml-base", "llama-common")+ .map { "lib$it.so" }+ .toSet() + cpuBackends+ val missing = required - output.listFiles().orEmpty().map { it.name }.toSet()+ check(missing.isEmpty()) { "Missing native libraries: ${missing.sorted()}" }+ }++ val libraries = output.listFiles { file -> file.extension == "so" }.orEmpty().toList()+ val names = libraries.map { it.name }.toSet()+ val system = setOf("libc.so", "libm.so", "libdl.so", "liblog.so", "libandroid.so", "libz.so")+ val alignment = if (abi in setOf("arm64-v8a", "x86_64")) 16384 else 4096+ for (library in libraries.sortedBy { it.name }) {+ val loads =+ readElf(library, "--program-headers")+ .lineSequence()+ .map { it.trim().split(Regex("\\s+")) }+ .filter { it.first() == "LOAD" }+ .toList()+ check(+ loads.isNotEmpty() && loads.all { it.last().removePrefix("0x").toLong(16) >= alignment }+ ) {+ "${library.name}: PT_LOAD alignment is below ${alignment / 1024} KB"+ }+ val needed =+ Regex("""\(NEEDED\).*?\[(.*?)]""")+ .findAll(readElf(library, "--dynamic"))+ .map { it.groupValues[1] }+ .toSet()+ check(needed.none { "/" in it }) { "${library.name}: absolute DT_NEEDED path" }+ val missing = needed - names - system+ check(missing.isEmpty()) { "${library.name}: unpackaged dependencies ${missing.sorted()}" }+ }+}+fun registerBuildRustJni(taskName: String,outputDir: Provider<Directory>,@@ -82,12 +158,32 @@ fun registerBuildRustJni(toolchain.absolutePath,"--out-dir",outDir.absolutePath,+ "--messages-dir",+ temporaryDir.absolutePath,*abis.toTypedArray(),)environment("ANDROID_NDK", ndkDir.absolutePath)environment("ANDROID_NDK_ROOT", ndkDir.absolutePath)environment("NDK_ROOT", ndkDir.absolutePath)}++ doLast {+ val readelf = ndkToolchain(android.ndkDirectory).resolve("bin/llvm-readelf")+ for (abi in abis) {+ packageNativeLibraries(+ temporaryDir.resolve("$abi.jsonl"),+ outputDir.get().asFile.resolve(abi),+ abi,+ ) { library, option ->+ providers+ .exec { commandLine(readelf, option, "--wide", library) }+ .standardOutput+ .asText+ .get()+ }+ logger.lifecycle("Packaged and verified native libraries for $abi")+ }+ }}val buildRustJniDebug =@@ -138,3 +234,86 @@ dependencies {api("net.java.dev.jna:jna:5.18.1@aar")implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.8.0")}++val checkNativePackaging =+ tasks.register("checkNativePackaging") {+ group = "verification"+ description = "Checks native library selection, dependencies, and page alignment."+ doLast {+ val fixture = temporaryDir.apply {+ deleteRecursively()+ mkdirs()+ }+ fun fixtureFile(path: String, text: String = "test") =+ fixture.resolve(path).apply {+ parentFile.mkdirs()+ writeText(text)+ }+ val output = fixture.resolve("jni").apply { mkdirs() }+ val build = fixture.resolve("current")+ val messages =+ fixtureFile(+ "messages.jsonl",+ JsonOutput.toJson(+ mapOf(+ "reason" to "build-script-executed",+ "package_id" to+ "registry+https://github.com/rust-lang/crates.io-index#[email protected]",+ "out_dir" to build.path,+ )+ ),+ )+ var alignment = "0x4000"+ var dependency = "libc.so"+ fun verify(abi: String, expectedError: String? = null) {+ val result = runCatching {+ packageNativeLibraries(messages, output, abi) { _, option ->+ if (option == "--program-headers")+ " LOAD 0x0 0x0 0x0 0x1000 0x1000 R E $alignment"+ else " 0x1 (NEEDED) Shared library: [$dependency]"+ }+ }+ if (expectedError == null) result.getOrThrow()+ else+ check(result.exceptionOrNull()?.message?.contains(expectedError) == true) {+ "Expected '$expectedError', got ${result.exceptionOrNull()}"+ }+ }+ val paths =+ listOf(+ "lib/libllama.so",+ "lib/libmtmd.so",+ "lib/libggml.so",+ "lib/libggml-base.so",+ "build/common/libllama-common.so",+ ) ++ listOf("armv8.0_1", "armv8.2_1", "armv8.2_2", "armv8.6_1").map {+ "backends/libggml-cpu-android_$it.so"+ }+ paths.forEach { fixtureFile("current/$it") }+ fixtureFile("current/backends/libggml-cpu-android_armv9.0_1.so")+ fixtureFile("stale.so")+ verify("arm64-v8a")+ check(+ output.listFiles().orEmpty().map { it.name }.toSet() ==+ paths.map { it.substringAfterLast('/') }.toSet()+ )+ build.resolve("backends/libggml-cpu-android_armv8.0_1.so").delete()+ output.resolve("libggml-cpu-android_armv8.0_1.so").delete()+ verify("arm64-v8a", "Missing native libraries")++ output.deleteRecursively()+ fixtureFile("jni/libc++_shared.so")+ alignment = "0x1000"+ verify("x86_64", "below 16 KB")+ verify("armeabi-v7a")+ alignment = "0x4000"+ dependency = "libggml-base.so"+ verify("x86_64", "unpackaged dependencies")+ logger.lifecycle("Native packaging checks passed (4 cases)")+ }+ }++tasks+ .matching { it.name in listOf("check", "lintDebug") }+ .configureEach { dependsOn(checkNativePackaging) }diff --git a/android/apps/ensu/rust/scripts/build-rust.sh b/android/apps/ensu/rust/scripts/build-rust.shindex a74ec89cbf..73394c7465 100755--- a/android/apps/ensu/rust/scripts/build-rust.sh+++ b/android/apps/ensu/rust/scripts/build-rust.sh@@ -9,17 +9,19 @@ REPO_ROOT=$(cd "$(dirname "$0")/../../../../.." && pwd)TARGET_DIR="$REPO_ROOT/rust/target"TOOLCHAIN=""OUT_DIR=""+MESSAGES_DIR=""ABIS=()while [[ $# -gt 0 ]]; docase $1 in--toolchain) TOOLCHAIN=$2; shift 2 ;;--out-dir) OUT_DIR=$2; shift 2 ;;+ --messages-dir) MESSAGES_DIR=$2; shift 2 ;;*) ABIS+=("$1"); shift ;;esacdone-[[ -n $TOOLCHAIN && -n $OUT_DIR && ${#ABIS[@]} -gt 0 ]] || {- echo "usage: $(basename "$0") --toolchain DIR --out-dir DIR <abi> [<abi>...]" >&2+[[ -n $TOOLCHAIN && -n $OUT_DIR && -n $MESSAGES_DIR && ${#ABIS[@]} -gt 0 ]] || {+ echo "usage: $(basename "$0") --toolchain DIR --out-dir DIR --messages-dir DIR <abi> [<abi>...]" >&2exit 1}@@ -30,10 +32,19 @@ for tool in cargo rustup cmake; docommand -v "$tool" >/dev/null || { echo "error: $tool not on PATH" >&2; exit 1; }done+export ANDROID_API_LEVEL=24+export CMAKE_SHARED_LINKER_FLAGS="${CMAKE_SHARED_LINKER_FLAGS:-} -Wl,-z,max-page-size=16384"+export CMAKE_MODULE_LINKER_FLAGS="${CMAKE_MODULE_LINKER_FLAGS:-} -Wl,-z,max-page-size=16384"+mkdir -p "$MESSAGES_DIR"+build_abi() {local abi=$1 target clang_triple libcxx_dircase $abi inarm64-v8a)+ if [[ -n ${GGML_CPU_ARM_ARCH+x} ]]; then+ echo "error: unset GGML_CPU_ARM_ARCH for runtime-selected ARM64 backends" >&2+ exit 1+ fitarget=aarch64-linux-androidclang_triple=aarch64-linux-android24libcxx_dir=aarch64-linux-android@@ -68,7 +79,8 @@ build_abi() {local out=$OUT_DIR/$abimkdir -p "$out"echo "==> $abi"- (cd "$REPO_ROOT/rust/bindings/uniffi/ensu" && CARGO_TARGET_DIR="$TARGET_DIR" cargo build --release --target "$target")+ (cd "$REPO_ROOT/rust/bindings/uniffi/ensu" && CARGO_TARGET_DIR="$TARGET_DIR" cargo build --locked --release --target "$target" --message-format=json-render-diagnostics) > "$MESSAGES_DIR/$abi.jsonl"+ rm -f "$out"/*.socp "$TARGET_DIR/$target/release/libensu.so" "$out/"cp "$TOOLCHAIN/sysroot/usr/lib/$libcxx_dir/libc++_shared.so" "$out/"}diff --git a/rust/Cargo.lock b/rust/Cargo.lockindex 85212e12fd..c4a8f5a20c 100644--- a/rust/Cargo.lock+++ b/rust/Cargo.lock@@ -2620,11 +2620,13 @@ dependencies = [name = "ente-ensu"version = "0.0.0"dependencies = [+ "cc","ente-assets","ente-core","ente-ensu-crypto","ente-image","llama-cpp-2",+ "llama-cpp-sys-2","ort","ort-sys","regex",@@ -5901,11 +5903,10 @@ checksum = "11d3d7f243d5c5a8b9bb5d6dd2b1602c0cb0b9db1621bafc7ed66e35ff9fe092"[[package]]name = "llama-cpp-2"-version = "0.1.144"+version = "0.1.158"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "b2e7a3cc3ed741aa3d5e6e3ff1d84d5ab7ba13bf15c857be06a5d11517767616"+checksum = "5a89d1d5c09794689aa6190206dd55499d8153b3669c72f2ef24a4e8ddc5ccbe"dependencies = [- "encoding_rs","enumflags2","llama-cpp-sys-2","thiserror 2.0.21",@@ -5915,9 +5916,9 @@ dependencies = [[[package]]name = "llama-cpp-sys-2"-version = "0.1.144"+version = "0.1.158"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "0a2efa5c5675f4e2170c52c4418f71907d39b1e3931fd7566f8382cf9e166282"+checksum = "608edbeed8bcb8573d8e665f3f047e085202510b7b3d09e7fae2cfe3d80d6bec"dependencies = ["bindgen","cc",diff --git a/rust/bindings/uniffi/ensu/src/llm.rs b/rust/bindings/uniffi/ensu/src/llm.rsindex 48d7fb4e4b..64ad6a116a 100644--- a/rust/bindings/uniffi/ensu/src/llm.rs+++ b/rust/bindings/uniffi/ensu/src/llm.rs@@ -346,6 +346,11 @@ pub fn llm_init_backend() -> Result<(), LlmError> {llm::init_backend().map_err(LlmError::from)}+#[uniffi::export]+pub fn llm_init_backend_from_directory(directory: String) -> Result<(), LlmError> {+ llm::init_backend_from_directory(&directory).map_err(LlmError::from)+}+#[uniffi::export]pub fn llm_cancel(job_id: i64) {llm::cancel(job_id);diff --git a/rust/checks/lint-exceptions/suppressions.json b/rust/checks/lint-exceptions/suppressions.jsonindex 97f5c94067..7c169d4f0a 100644--- a/rust/checks/lint-exceptions/suppressions.json+++ b/rust/checks/lint-exceptions/suppressions.json@@ -30,6 +30,7 @@"rust/crates/core/src/crypto/types.rs": ["clippy::expect_used"],"rust/crates/ensu/src/config.rs": ["clippy::expect_used"],"rust/crates/ensu/src/config/knowledge.rs": ["clippy::expect_used"],+ "rust/crates/ensu/src/llm/chat_template_bridge.rs": ["unsafe_code"],"rust/crates/ensu/src/llm/generate.rs": ["clippy::expect_used"],"rust/crates/ensu/src/model/mod.rs": ["clippy::expect_used"],"rust/crates/ensu/src/platform.rs": ["unsafe_code"],diff --git a/rust/crates/ensu/Cargo.toml b/rust/crates/ensu/Cargo.tomlindex 75695da430..4a3831a9c4 100644--- a/rust/crates/ensu/Cargo.toml+++ b/rust/crates/ensu/Cargo.toml@@ -7,6 +7,7 @@ ente-assets.workspace = trueente-core.workspace = trueente-ensu-crypto.workspace = trueente-image.workspace = true+llama-cpp-sys-2 = { version = "=0.1.158", default-features = false, features = ["common"] }regex.workspace = truerubato = { version = "5.0.0", optional = true }rusqlite = { workspace = true, default-features = true, optional = true }@@ -24,20 +25,26 @@ zstd = "0.14.0"[dev-dependencies]tempfile.workspace = true+[build-dependencies]+cc = "1.2.61"+# Keep `mtmd` on every platform for multimodal support.# macOS disables OpenMP because Apple's clang lacks libomp. llama.cpp uses its# own thread pool, Accelerate, and Metal on arm64 instead.[target.'cfg(target_os = "macos")'.dependencies]-llama-cpp-2 = { version = "=0.1.144", default-features = false, features = ["mtmd"] }+llama-cpp-2 = { version = "=0.1.158", default-features = false, features = ["common", "mtmd"] }# Linux Vulkan needs a system Vulkan loader and glslc.[target.'cfg(target_os = "linux")'.dependencies]-llama-cpp-2 = { version = "=0.1.144", features = ["mtmd", "vulkan"] }+llama-cpp-2 = { version = "=0.1.158", features = ["mtmd", "vulkan"] }# Other platforms stay on CPU. Android keeps the shared C++ runtime; Windows# avoids Vulkan's VULKAN_SDK requirement.-[target.'cfg(all(not(target_os = "macos"), not(target_os = "linux")))'.dependencies]-llama-cpp-2 = { version = "=0.1.144", features = ["mtmd"] }+[target.'cfg(all(not(target_os = "macos"), not(target_os = "linux"), not(all(target_os = "android", target_arch = "aarch64"))))'.dependencies]+llama-cpp-2 = { version = "=0.1.158", features = ["mtmd"] }++[target.'cfg(all(target_os = "android", target_arch = "aarch64"))'.dependencies]+llama-cpp-2 = { version = "=0.1.158", features = ["dynamic-backends", "mtmd"] }[target.'cfg(target_os = "android")'.dependencies]ort = { workspace = true, features = ["load-dynamic"], optional = true }diff --git a/rust/crates/ensu/build.rs b/rust/crates/ensu/build.rsnew file mode 100644index 0000000000..5df3142f9c--- /dev/null+++ b/rust/crates/ensu/build.rs@@ -0,0 +1,36 @@+use std::{env, error::Error, fs, path::PathBuf};++fn main() -> Result<(), Box<dyn Error>> {+ println!("cargo:rerun-if-changed=native/chat.cpp");+ println!("cargo:rerun-if-env-changed=DEP_LLAMA_GGML_CMAKE_DIR");++ let cmake_dir = PathBuf::from(env::var("DEP_LLAMA_GGML_CMAKE_DIR")?);+ let output = cmake_dir+ .parent()+ .and_then(|path| path.parent())+ .ok_or("Invalid llama.cpp CMake directory")?;+ let cache_path = output.join("build/CMakeCache.txt");+ println!("cargo:rerun-if-changed={}", cache_path.display());+ let cache = fs::read_to_string(cache_path)?;+ let source = cache+ .lines()+ .find_map(|line| line.strip_prefix("CMAKE_HOME_DIRECTORY:INTERNAL="))+ .ok_or("llama.cpp source directory missing from CMake cache")?;+ let source = PathBuf::from(source);++ let mut build = cc::Build::new();+ build+ .cpp(true)+ .std("c++17")+ .file("native/chat.cpp")+ .include(source.join("common"))+ .include(source.join("include"))+ .include(source.join("ggml/include"))+ .include(source.join("vendor"))+ .cpp_link_stdlib(None);+ if build.get_compiler().is_like_msvc() {+ build.flag("/EHsc");+ }+ build.compile("ensu_chat");+ Ok(())+}diff --git a/rust/crates/ensu/native/chat.cpp b/rust/crates/ensu/native/chat.cppnew file mode 100644index 0000000000..08e4f68ec8--- /dev/null+++ b/rust/crates/ensu/native/chat.cpp@@ -0,0 +1,78 @@+#include "chat.h"++#include <cstdlib>+#include <cstring>+#include <exception>+#include <stdexcept>+#include <string>+#include <utility>+#include <vector>++extern "C" {+struct ensu_chat_result {+ char * data;+ size_t len;+ int status;+};++static ensu_chat_result copy_result(const char * text, size_t len, int status) noexcept {+ auto * data = static_cast<char *>(std::malloc(len + 1));+ if (!data) {+ return {nullptr, 0, 2};+ }+ std::memcpy(data, text, len);+ data[len] = '\0';+ return {data, len, status};+}++ensu_chat_result ensu_chat_render(const char * source, const char * bos, const char * eos,+ const char * messages_json, bool add_assistant) noexcept {+ try {+ auto messages = common_chat_msgs_parse_oaicompat(common_json::parse(messages_json));+ std::vector<const char *> builtins(llama_chat_builtin_templates(nullptr, 0));+ llama_chat_builtin_templates(builtins.data(), builtins.size());+ for (const auto * builtin : builtins) {+ if (std::strcmp(source, builtin) != 0 || std::strcmp(source, "chatml") == 0) {+ continue;+ }+ std::vector<llama_chat_message> chat;+ for (const auto & message : messages) {+ if (message.role.find('\0') != std::string::npos || message.content.find('\0') != std::string::npos) {+ throw std::runtime_error("Invalid built-in chat message: embedded NUL");+ }+ chat.push_back({message.role.c_str(), message.content.c_str()});+ }+ const auto length = llama_chat_apply_template(source, chat.data(), chat.size(), add_assistant, nullptr, 0);+ if (length < 0) {+ throw std::runtime_error("Failed to apply built-in chat template");+ }+ std::string prompt(length, '\0');+ const auto written = llama_chat_apply_template(source, chat.data(), chat.size(), add_assistant, prompt.data(), length);+ if (written != length) {+ throw std::runtime_error("Invalid built-in chat template output length");+ }+ return copy_result(prompt.data(), prompt.size(), 0);+ }+ auto templates = common_chat_templates_init(nullptr, source[0] ? source : "chatml", bos, eos);+ common_chat_templates_inputs inputs;+ inputs.messages = std::move(messages);+ inputs.add_generation_prompt = add_assistant;+ inputs.use_jinja = true;+ inputs.enable_thinking = false;+ inputs.chat_template_kwargs["enable_thinking"] = "false";+ inputs.add_bos = false;+ inputs.add_eos = false;+ const auto result = common_chat_templates_apply(templates.get(), inputs);+ return copy_result(result.prompt.data(), result.prompt.size(), 0);+ } catch (const std::exception & error) {+ return copy_result(error.what(), std::strlen(error.what()), 1);+ } catch (...) {+ constexpr char error[] = "Unknown chat template error";+ return copy_result(error, sizeof(error) - 1, 1);+ }+}++void ensu_chat_free(char * data) noexcept {+ std::free(data);+}+}diff --git a/rust/crates/ensu/src/llm/chat_template_bridge.rs b/rust/crates/ensu/src/llm/chat_template_bridge.rsnew file mode 100644index 0000000000..a6e844b76a--- /dev/null+++ b/rust/crates/ensu/src/llm/chat_template_bridge.rs@@ -0,0 +1,80 @@+use std::ffi::{CString, c_char};++use super::{Error, format_error};++#[repr(C)]+struct RenderResult {+ data: *mut c_char,+ len: usize,+ status: i32,+}++#[expect(+ unsafe_code,+ reason = "Declarations for the private, synchronous C++ chat bridge"+)]+unsafe extern "C" {+ fn ensu_chat_render(+ source: *const c_char,+ bos: *const c_char,+ eos: *const c_char,+ messages: *const c_char,+ add_assistant: bool,+ ) -> RenderResult;+ fn ensu_chat_free(data: *mut c_char);+}++impl Drop for RenderResult {+ #[expect(+ unsafe_code,+ reason = "The bridge allocated this buffer and frees it with the same allocator; null is allowed"+ )]+ fn drop(&mut self) {+ unsafe { ensu_chat_free(self.data) };+ }+}++#[expect(+ unsafe_code,+ reason = "CString inputs live for the synchronous call; the result owns len initialized bytes until Drop frees them"+)]+pub(super) fn render(+ source: &str,+ bos: &str,+ eos: &str,+ messages: &str,+ add_assistant: bool,+) -> Result<String, Error> {+ let cstring = |value: &str| {+ CString::new(value)+ .map_err(|err| Error::InvalidInput(format_error("Invalid template input", err)))+ };+ let source = cstring(source)?;+ let bos = cstring(bos)?;+ let eos = cstring(eos)?;+ let messages = cstring(messages)?;+ let result = unsafe {+ ensu_chat_render(+ source.as_ptr(),+ bos.as_ptr(),+ eos.as_ptr(),+ messages.as_ptr(),+ add_assistant,+ )+ };+ let failure = |message| Error::Llama {+ op: "Failed to apply chat template",+ message,+ };+ if result.data.is_null() {+ return Err(failure("Could not allocate template output".to_owned()));+ }+ let bytes = unsafe { std::slice::from_raw_parts(result.data.cast::<u8>(), result.len) };+ let text = std::str::from_utf8(bytes)+ .map_err(|err| failure(err.to_string()))?+ .to_owned();+ if result.status != 0 {+ return Err(failure(text));+ }+ Ok(text)+}diff --git a/rust/crates/ensu/src/llm/embed.rs b/rust/crates/ensu/src/llm/embed.rsindex c81fcd6793..fb67fc4b51 100644--- a/rust/crates/ensu/src/llm/embed.rs+++ b/rust/crates/ensu/src/llm/embed.rs@@ -1,5 +1,4 @@use llama_cpp_2::llama_batch::LlamaBatch;-use llama_cpp_2::model::AddBos;use super::Error;use super::context::LocalContext;@@ -41,11 +40,8 @@ impl LocalContext {self.with_context_and_cache_mut(|context, cached_tokens| {let tokens = context.model- .str_to_token(prompt, AddBos::Always)- .map_err(|err| Error::Llama {- op: "Embedding tokenization failed",- message: err.to_string(),- })?;+ .vocab()+ .tokenize(prompt.as_bytes(), true, true);if tokens.is_empty() {return Err(Error::InvalidInput("embedding prompt produced no tokens".to_string(),diff --git a/rust/crates/ensu/src/llm/generate.rs b/rust/crates/ensu/src/llm/generate.rsindex b1f6d07e25..54ed1ac82c 100644--- a/rust/crates/ensu/src/llm/generate.rs+++ b/rust/crates/ensu/src/llm/generate.rs@@ -1,9 +1,7 @@-use llama_cpp_2::TokenToStringError;use llama_cpp_2::context::LlamaContext;use llama_cpp_2::llama_batch::LlamaBatch;-use llama_cpp_2::model::{AddBos, LlamaChatMessage, LlamaChatTemplate, LlamaModel};+use llama_cpp_2::model::LlamaModel;use llama_cpp_2::mtmd::{MtmdBitmap, MtmdInputText, mtmd_default_marker};-use llama_cpp_2::openai::OpenAIChatTemplateParams;use llama_cpp_2::sampling::LlamaSampler;use llama_cpp_2::token::LlamaToken;use serde::{Deserialize, Serialize};@@ -16,6 +14,7 @@ use std::time::Instant;use super::context::LocalContext;use super::event::{EventSink, FinishReason, GenerationEvent, GenerationSummary, JobId};+use super::template::build_chat_prompt;use super::{Error, format_error, lock};static JOB_COUNTER: AtomicI64 = AtomicI64::new(1);@@ -93,106 +92,27 @@ pub struct ChatRequest {pub grammar: Option<String>,}-fn token_piece_bytes(model: &LlamaModel, token: LlamaToken) -> Result<Vec<u8>, TokenToStringError> {- match model.token_to_piece_bytes(token, 8, true, None) {- #[expect(- clippy::expect_used,- reason = "llama.cpp reports insufficient buffer capacity as a negative byte count"- )]- Err(TokenToStringError::InsufficientBufferSpace(required)) => model.token_to_piece_bytes(- token,- (-required)- .try_into()- .expect("error buffer size is positive"),- true,- None,- ),- result => result,- }-}-fn token_piece_string(model: &LlamaModel, token: LlamaToken) -> Option<String> {- String::from_utf8(token_piece_bytes(model, token).ok()?).ok()-}--fn build_chat_prompt(- model: &LlamaModel,- messages: Vec<ChatMessage>,- template_override: Option<String>,- add_assistant: bool,-) -> Result<String, Error> {- let template_text = match template_override {- Some(template) => template,- None => model- .chat_template(None)- .ok()- .and_then(|template| template.to_string().ok())- .unwrap_or_else(|| "chatml".to_string()),- };- let template = LlamaChatTemplate::new(&template_text)- .map_err(|err| Error::InvalidInput(format_error("Invalid chat template", err)))?;-- if template_text.contains("enable_thinking") {- let messages_json = serde_json::to_string(&messages)- .map_err(|err| Error::InvalidInput(format_error("Invalid chat messages", err)))?;- let params = OpenAIChatTemplateParams {- messages_json: &messages_json,- tools_json: None,- tool_choice: None,- json_schema: None,- grammar: None,- reasoning_format: None,- chat_template_kwargs: Some(r#"{"enable_thinking":false}"#),- add_generation_prompt: add_assistant,- use_jinja: true,- parallel_tool_calls: false,- enable_thinking: false,- add_bos: false,- add_eos: false,- parse_tool_calls: false,- };- let result = model- .apply_chat_template_oaicompat(&template, ¶ms)- .map_err(|err| Error::Llama {- op: "Failed to apply chat template",- message: err.to_string(),- })?;- return Ok(result.prompt);+ if token.0 < 0 {+ return None;}-- let chat_messages = messages- .into_iter()- .map(|message| {- LlamaChatMessage::new(message.role, message.content)- .map_err(|err| Error::InvalidInput(format_error("Invalid chat message", err)))- })- .collect::<Result<Vec<_>, Error>>()?;-- model- .apply_chat_template(&template, &chat_messages, add_assistant)- .map_err(|err| Error::Llama {- op: "Failed to apply chat template",- message: err.to_string(),- })+ String::from_utf8(model.vocab().token_to_piece(token, true, None)).ok()}-fn should_add_bos(model: &LlamaModel, prompt: &str) -> AddBos {- if let Some(bos) = token_piece_string(model, model.token_bos())+fn should_add_bos(model: &LlamaModel, prompt: &str) -> bool {+ if let Some(bos) = token_piece_string(model, model.vocab().bos())&& !bos.is_empty()&& prompt.starts_with(&bos){- return AddBos::Never;+ return false;}- AddBos::Always+ true}fn tokenize_text_prompt(model: &LlamaModel, prompt: &str) -> Result<Vec<LlamaToken>, Error> {let tokens = model- .str_to_token(prompt, should_add_bos(model, prompt))- .map_err(|err| Error::Llama {- op: "Tokenize failed",- message: err.to_string(),- })?;+ .vocab()+ .tokenize(prompt.as_bytes(), should_add_bos(model, prompt), true);if tokens.is_empty() {return Err(Error::InvalidInput("Prompt produced no tokens".to_string()));}@@ -334,7 +254,7 @@ impl GenerationJob<'_> {sampler.accept(token);check_cancelled(self.cancel_flag)?;- if ctx.model.is_eog_token(token) {+ if ctx.model.vocab().is_eog(token) {self.generated_tokens = self.generated_tokens.saturating_add(1);self.finish_reason = FinishReason::Eog;break;@@ -345,10 +265,7 @@ impl GenerationJob<'_> {}self.generated_tokens = self.generated_tokens.saturating_add(1);- let bytes = token_piece_bytes(ctx.model, token).map_err(|err| Error::Llama {- op: "Detokenize failed",- message: err.to_string(),- })?;+ let bytes = ctx.model.vocab().token_to_piece(token, true, None);let step = decoder.push_bytes(&bytes);if let Some(text) = step.text {@@ -415,7 +332,11 @@ impl GenerationJob<'_> {}}-fn build_sampler(model: &LlamaModel, request: &SamplingParams) -> Result<LlamaSampler, Error> {+fn build_sampler(+ model: &LlamaModel,+ request: &SamplingParams,+ context_size: i32,+) -> Result<LlamaSampler, Error> {let mut samplers = Vec::new();let mut repeat_penalty = request.repeat_penalty.unwrap_or(1.0);@@ -436,7 +357,8 @@ fn build_sampler(model: &LlamaModel, request: &SamplingParams) -> Result<LlamaSa|| presence_penalty != 0.0{samplers.push(LlamaSampler::penalties(- -1,+ model.vocab().n_tokens(),+ context_size,repeat_penalty,frequency_penalty,presence_penalty,@@ -720,7 +642,7 @@ fn generate_chat_stream(token_offset = end;}- let mut sampler = build_sampler(ctx.model, &sampler_request)?;+ let mut sampler = build_sampler(ctx.model, &sampler_request, ctx.n_ctx() as i32)?;sampler.accept_many(prompt_tokens.iter());let pos = prompt_tokens.len() as i32;@@ -746,9 +668,11 @@ fn generate_chat_stream(});}let bitmap =- MtmdBitmap::from_file(&mtmd_ctx, image_path).map_err(|err| Error::Llama {- op: "Failed to load image",- message: err.to_string(),+ MtmdBitmap::from_file(&mtmd_ctx, image_path, false).map_err(|err| {+ Error::Llama {+ op: "Failed to load image",+ message: err.to_string(),+ }})?;if bitmap.is_audio() {return Err(Error::Unsupported("Audio inputs are not supported"));@@ -757,7 +681,7 @@ fn generate_chat_stream(}let bitmap_refs = bitmaps.iter().collect::<Vec<_>>();- let add_special = matches!(should_add_bos(ctx.model, &prompt), AddBos::Always);+ let add_special = should_add_bos(ctx.model, &prompt);let input_text = MtmdInputText {text: prompt,add_special,@@ -807,7 +731,7 @@ fn generate_chat_stream(})?;check_cancelled(&cancel_flag)?;- let mut sampler = build_sampler(ctx.model, &sampler_request)?;+ let mut sampler = build_sampler(ctx.model, &sampler_request, ctx.n_ctx() as i32)?;let mut prompt_tokens = Vec::new();for index in 0..chunks.len() {if let Some(chunk) = chunks.get(index)diff --git a/rust/crates/ensu/src/llm/mod.rs b/rust/crates/ensu/src/llm/mod.rsindex 5ca1de1576..0d65518b0b 100644--- a/rust/crates/ensu/src/llm/mod.rs+++ b/rust/crates/ensu/src/llm/mod.rs@@ -1,3 +1,4 @@+mod chat_template_bridge;mod context;mod embed;mod event;@@ -5,6 +6,7 @@ mod generate;mod history;pub mod memory;mod model;+mod template;mod worker;pub use context::*;@@ -41,7 +43,15 @@ fn lock<T>(mutex: &Mutex<T>) -> MutexGuard<'_, T> {}fn backend() -> Result<&'static LlamaBackend, Error> {- match BACKEND.get_or_init(|| LlamaBackend::init().map_err(|err| err.to_string())) {+ #[cfg(all(target_os = "android", target_arch = "aarch64"))]+ let result = BACKEND.get().ok_or_else(|| {+ Error::InvalidInput(+ "Initialize CPU backends from nativeLibraryDir before loading a model".to_owned(),+ )+ })?;+ #[cfg(not(all(target_os = "android", target_arch = "aarch64")))]+ let result = BACKEND.get_or_init(|| LlamaBackend::init().map_err(|err| err.to_string()));+ match result {Ok(backend) => Ok(backend),Err(err) => Err(Error::Llama {op: "Failed to initialize backend",@@ -50,6 +60,42 @@ fn backend() -> Result<&'static LlamaBackend, Error> {}}+pub fn init_backend_from_directory(directory: &str) -> Result<(), Error> {+ #[cfg(all(target_os = "android", target_arch = "aarch64"))]+ {+ let path = std::path::Path::new(directory);+ if directory.contains('\0') || !path.is_absolute() || !path.is_dir() {+ return Err(Error::InvalidInput(+ "Invalid native-library directory".to_owned(),+ ));+ }+ if !path.join("libggml-cpu-android_armv8.0_1.so").is_file() {+ return Err(Error::InvalidInput(+ "Baseline CPU backend is missing from nativeLibraryDir".to_owned(),+ ));+ }+ BACKEND.get_or_init(|| {+ llama_cpp_2::llama_backend::load_backends_from_path(path);+ if !cpu_backend_available() {+ return Err(+ "No compatible CPU backend could be loaded from nativeLibraryDir".to_owned(),+ );+ }+ LlamaBackend::init().map_err(|err| err.to_string())+ });+ }+ #[cfg(not(all(target_os = "android", target_arch = "aarch64")))]+ let _ = directory;+ backend().map(|_| ())+}++#[cfg(all(target_os = "android", target_arch = "aarch64"))]+fn cpu_backend_available() -> bool {+ llama_cpp_2::list_llama_ggml_backend_devices()+ .iter()+ .any(|device| device.device_type == llama_cpp_2::LlamaBackendDeviceType::Cpu)+}+fn format_error(context: &str, err: impl std::fmt::Display) -> String {format!("{context}: {err}")}diff --git a/rust/crates/ensu/src/llm/model.rs b/rust/crates/ensu/src/llm/model.rsindex 8ab2f23a62..ee1000e9ac 100644--- a/rust/crates/ensu/src/llm/model.rs+++ b/rust/crates/ensu/src/llm/model.rs@@ -30,6 +30,10 @@ impl Model {model_params = model_params.with_n_gpu_layers(layers);}+ if let Some(use_mmap) = params.use_mmap {+ model_params = model_params.with_use_mmap(use_mmap);+ }+if let Some(use_mlock) = params.use_mlock {model_params = model_params.with_use_mlock(use_mlock);}diff --git a/rust/crates/ensu/src/llm/template.rs b/rust/crates/ensu/src/llm/template.rsnew file mode 100644index 0000000000..48bfcd0559--- /dev/null+++ b/rust/crates/ensu/src/llm/template.rs@@ -0,0 +1,260 @@+use llama_cpp_2::model::LlamaModel;++use super::{ChatMessage, Error, chat_template_bridge, format_error};++pub(super) fn build_chat_prompt(+ model: &LlamaModel,+ messages: Vec<ChatMessage>,+ template_override: Option<String>,+ add_assistant: bool,+) -> Result<String, Error> {+ let source = match template_override {+ Some(template) => template,+ None => model+ .chat_template(None)+ .ok()+ .and_then(|template| template.to_string().ok())+ .unwrap_or_else(|| "chatml".to_owned()),+ };+ let vocab = model.vocab();+ let special_token = |token: llama_cpp_2::token::LlamaToken| {+ if token.0 < 0 {+ return Ok(String::new());+ }+ String::from_utf8(vocab.token_to_piece(token, true, None))+ .map_err(|err| Error::InvalidInput(format_error("Invalid special token", err)))+ };+ render(+ &source,+ &special_token(vocab.bos())?,+ &special_token(vocab.eos())?,+ &messages,+ add_assistant,+ )+}++fn render(+ source: &str,+ bos: &str,+ eos: &str,+ messages: &[ChatMessage],+ add_assistant: bool,+) -> Result<String, Error> {+ let messages = serde_json::to_string(messages)+ .map_err(|err| Error::InvalidInput(format_error("Invalid chat messages", err)))?;+ chat_template_bridge::render(source, bos, eos, &messages, add_assistant)+}++#[cfg(test)]+mod tests {+ use super::*;++ struct Fixture {+ model_id: &'static str,+ template: &'static str,+ bos: &'static str,+ eos: &'static str,+ }++ fn message(role: &str, content: &str) -> ChatMessage {+ ChatMessage {+ role: role.to_owned(),+ content: content.to_owned(),+ }+ }++ #[test]+ fn default_templates_preserve_turns_unicode_images_and_assistant_prefixes() {+ for fixture in DEFAULT_TEMPLATES {+ let messages = [+ message("system", "Be concise."),+ message("user", "नमस्ते 🐈\n<__media__>"),+ message("assistant", "A cat."),+ message("user", "What colour?"),+ ];+ let gemma = fixture.model_id.starts_with("gemma");+ let (start, end, assistant) = if gemma {+ ("<|turn>", "<turn|>", "model")+ } else {+ ("<|im_start|>", "<|im_end|>", "assistant")+ };+ let mut expected = fixture.bos.to_owned();+ for msg in &messages {+ let role = if msg.role == "assistant" {+ assistant+ } else {+ &msg.role+ };+ expected.push_str(&format!("{start}{role}\n{}{end}\n", msg.content));+ }+ for add_assistant in [false, true] {+ let prompt = render(+ fixture.template,+ fixture.bos,+ fixture.eos,+ &messages,+ add_assistant,+ )+ .unwrap();+ let expected = if add_assistant {+ format!("{expected}{start}{assistant}\n")+ } else {+ expected.clone()+ };+ assert_eq!(+ prompt, expected,+ "{}: add_assistant={add_assistant}",+ fixture.model_id+ );+ }+ }+ }++ #[test]+ fn overrides_receive_a_boolean_false_and_special_tokens() {+ assert_eq!(+ render("llama2", "", "", &[message("user", "Hi")], true).unwrap(),+ "[INST] Hi [/INST]"+ );+ let template = "{{ bos_token }}{% if enable_thinking is not defined or enable_thinking %}THINKING{% else %}DIRECT{% endif %}:{{ messages[0].content }}{{ eos_token }}{% if add_generation_prompt %}ASSISTANT{% endif %}";+ let prompt = render(+ template,+ "<bos>",+ "<eos>",+ &[message("user", "你好\n<__media__>")],+ true,+ )+ .unwrap();+ assert_eq!(prompt, "<bos>DIRECT:你好\n<__media__><eos>ASSISTANT");+ }++ #[test]+ fn template_errors_are_returned_and_subsequent_renders_recover() {+ assert!(render("{% if %}", "", "", &[message("user", "Hi")], true).is_err());+ assert!(render("bad\0template", "", "", &[], true).is_err());+ assert!(render("llama2", "", "", &[message("user", "Hi\0there")], true).is_err());+ let messages = [message("user", "Hello")];+ for template in ["", "chatml"] {+ assert_eq!(+ render(template, "", "", &messages, true).unwrap(),+ "<|im_start|>user\nHello<|im_end|>\n<|im_start|>assistant\n"+ );+ }+ }++ const DEFAULT_TEMPLATES: &[Fixture] = &[+ Fixture {+ model_id: "lfm-vl-1.6b",+ template: concat!(+ "{{- bos_token -}}\n{%- set keep_past_thinking = keep_past_thinking | default(false) -%}\n{%- set ns = namespace(system_prompt=\"\") -%}\n{%- if messages[0][\"role\"] == \"system\" -%}\n {%- set sys_content = ",+ "messages[0][\"content\"] -%}\n {%- if sys_content is not string -%}\n {%- for item in sys_content -%}\n {%- if item[\"type\"] == \"text\" -%}\n {%- set ns.system_prompt = ns.system_prompt + ",+ "item[\"text\"] -%}\n {%- endif -%}\n {%- endfor -%}\n {%- else -%}\n {%- set ns.system_prompt = sys_content -%}\n {%- endif -%}\n {%- set messages = messages[1:] -%}\n{%- endif -%}\n{%- if tools ",+ "-%}\n {%- set ns.system_prompt = ns.system_prompt + (\"\\n\" if ns.system_prompt else \"\") + \"List of tools: [\" -%}\n {%- for tool in tools -%}\n {%- if tool is not string -%}\n {%- set tool = tool | ",+ "tojson -%}\n {%- endif -%}\n {%- set ns.system_prompt = ns.system_prompt + tool -%}\n {%- if not loop.last -%}\n {%- set ns.system_prompt = ns.system_prompt + \", \" -%}\n {%- endif -%}\n ",+ " {%- endfor -%}\n {%- set ns.system_prompt = ns.system_prompt + \"]\" -%}\n{%- endif -%}\n{%- if ns.system_prompt -%}\n {{- \"<|im_start|>system\\n\" + ns.system_prompt + \"<|im_end|>\\n\" -}}\n{%- endif -%}\n{%- set ",+ "ns.last_assistant_index = -1 -%}\n{%- for message in messages -%}\n {%- if message[\"role\"] == \"assistant\" -%}\n {%- set ns.last_assistant_index = loop.index0 -%}\n {%- endif -%}\n{%- endfor -%}\n{%- for message ",+ "in messages -%}\n {{- \"<|im_start|>\" + message[\"role\"] + \"\\n\" -}}\n {%- set content = message[\"content\"] -%}\n {%- if content is not string -%}\n {%- set ns.content = \"\" -%}\n {%- for item in content ",+ "-%}\n {%- if item[\"type\"] == \"image\" -%}\n {%- set ns.content = ns.content + \"<image>\" -%}\n {%- elif item[\"type\"] == \"text\" -%}\n {%- set ns.content = ns.content + ",+ "item[\"text\"] -%}\n {%- else -%}\n {%- set ns.content = ns.content + item | tojson -%}\n {%- endif -%}\n {%- endfor -%}\n {%- set content = ns.content -%}\n {%- endif -%}\n ",+ " {%- if message[\"role\"] == \"assistant\" and not keep_past_thinking and loop.index0 != ns.last_assistant_index -%}\n {%- if \"</think>\" in content -%}\n {%- set content = content.split(\"</think>\")[-1] | ",+ "trim -%}\n {%- endif -%}\n {%- endif -%}\n {{- content + \"<|im_end|>\\n\" -}}\n{%- endfor -%}\n{%- if add_generation_prompt -%}\n {{- \"<|im_start|>assistant\\n\" -}}\n{%- endif -%}",+ ),+ bos: "<|startoftext|>",+ eos: "<|im_end|>",+ },+ Fixture {+ model_id: "gemma-4-e2b-q4km",+ template: concat!(+ "{%- macro format_parameters(properties, required, filter_keys=false) -%}\n {%- set standard_keys = ['description', 'type', 'properties', 'required', 'nullable'] -%}\n {%- set ns = namespace(found_first=false) -%}\n ",+ " {%- for key, value in properties | dictsort -%}\n {%- set add_comma = false -%}\n {%- if not filter_keys or key not in standard_keys -%}\n {%- if ns.found_first %},{% endif -%}\n {%- set ",+ "ns.found_first = true -%}\n {{ key }}:{\n {%- if value['description'] -%}\n description:<|\"|>{{ value['description'] }}<|\"|>\n {%- set add_comma = true -%}\n {%- ",+ "endif -%}\n {%- if value['type'] | upper == 'STRING' -%}\n {%- if value['enum'] -%}\n {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%}\n ",+ " enum:{{ format_argument(value['enum']) }}\n {%- endif -%}\n {%- elif value['type'] | upper == 'ARRAY' -%}\n {%- if value['items'] is mapping and value['items'] -%}\n ",+ " {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%}\n items:{\n {%- set ns_items = namespace(found_first=false) -%}\n {%- for item_key, ",+ "item_value in value['items'] | dictsort -%}\n {%- if item_value is not none -%}\n {%- if ns_items.found_first %},{% endif -%}\n {%- set ",+ "ns_items.found_first = true -%}\n {%- if item_key == 'properties' -%}\n properties:{\n {%- if item_value is mapping -%}\n ",+ " {{- format_parameters(item_value, value['items']['required'] | default([])) -}}\n {%- endif -%}\n }\n {%- elif ",+ "item_key == 'required' -%}\n required:[\n {%- for req_item in item_value -%}\n <|\"|>{{- req_item -}}<|\"|>\n ",+ " {%- if not loop.last %},{% endif -%}\n {%- endfor -%}\n ]\n {%- elif item_key == 'type' -%}\n ",+ " {%- if item_value is string -%}\n type:{{ format_argument(item_value | upper) }}\n {%- else -%}\n type:{{ ",+ "format_argument(item_value | map('upper') | list) }}\n {%- endif -%}\n {%- else -%}\n {{ item_key }}:{{ format_argument(item_value) ",+ "}}\n {%- endif -%}\n {%- endif -%}\n {%- endfor -%}\n }\n {%- endif -%}\n {%- endif -%}\n {%- if ",+ "value['nullable'] %}\n {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%}\n nullable:true\n {%- endif -%}\n {%- if value['type'] | upper == ",+ "'OBJECT' -%}\n {%- if value['properties'] is defined and value['properties'] is mapping -%}\n {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%}\n ",+ " properties:{\n {{- format_parameters(value['properties'], value['required'] | default([])) -}}\n }\n {%- elif value is mapping -%}\n {%- if add_comma ",+ "%},{%- else -%} {%- set add_comma = true -%} {% endif -%}\n properties:{\n {{- format_parameters(value, value['required'] | default([]), filter_keys=true) -}}\n }\n ",+ " {%- endif -%}\n {%- if value['required'] -%}\n {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%}\n required:[\n ",+ "{%- for item in value['required'] | default([]) -%}\n <|\"|>{{- item -}}<|\"|>\n {%- if not loop.last %},{% endif -%}\n {%- endfor -%}\n ]\n ",+ " {%- endif -%}\n {%- endif -%}\n {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%}\n type:<|\"|>{{ value['type'] | upper }}<|\"|>}\n {%- endif -%}\n ",+ " {%- endfor -%}\n{%- endmacro -%}\n{%- macro format_function_declaration(tool_data) -%}\n declaration:{{- tool_data['function']['name'] -}}{description:<|\"|>{{- tool_data['function']['description'] -}}<|\"|>\n {%- set ",+ "params = tool_data['function']['parameters'] -%}\n {%- if params -%}\n ,parameters:{\n {%- if params['properties'] -%}\n properties:{ {{- format_parameters(params['properties'], ",+ "params['required']) -}} },\n {%- endif -%}\n {%- if params['required'] -%}\n required:[\n {%- for item in params['required'] -%}\n <|\"|>{{- item -}}<|\"|>\n {{- ",+ "',' if not loop.last -}}\n {%- endfor -%}\n ],\n {%- endif -%}\n {%- if params['type'] -%}\n type:<|\"|>{{- params['type'] | upper -}}<|\"|>}\n {%- endif -%}\n {%- endif ",+ "-%}\n {%- if 'response' in tool_data['function'] -%}\n {%- set response_declaration = tool_data['function']['response'] -%}\n ,response:{\n {%- if response_declaration['description'] -%}\n ",+ "description:<|\"|>{{- response_declaration['description'] -}}<|\"|>,\n {%- endif -%}\n {%- if response_declaration['type'] | upper == 'OBJECT' -%}\n type:<|\"|>{{- response_declaration['type'] | upper ",+ "-}}<|\"|>}\n {%- endif -%}\n {%- endif -%}\n }\n{%- endmacro -%}\n{%- macro format_argument(argument, escape_keys=True) -%}\n {%- if argument is none -%}\n {{- 'null' -}}\n {%- elif argument is string ",+ "-%}\n {{- '<|\"|>' + argument + '<|\"|>' -}}\n {%- elif argument is boolean -%}\n {{- 'true' if argument else 'false' -}}\n {%- elif argument is mapping -%}\n {{- '{' -}}\n {%- set ns = ",+ "namespace(found_first=false) -%}\n {%- for key, value in argument | dictsort -%}\n {%- if ns.found_first %},{% endif -%}\n {%- set ns.found_first = true -%}\n {%- if escape_keys -%}\n ",+ " {{- '<|\"|>' + key + '<|\"|>' -}}\n {%- else -%}\n {{- key -}}\n {%- endif -%}\n :{{- format_argument(value, escape_keys=escape_keys) -}}\n {%- endfor -%}\n ",+ " {{- '}' -}}\n {%- elif argument is sequence -%}\n {{- '[' -}}\n {%- for item in argument -%}\n {{- format_argument(item, escape_keys=escape_keys) -}}\n {%- if not loop.last %},{% ",+ "endif -%}\n {%- endfor -%}\n {{- ']' -}}\n {%- else -%}\n {{- argument -}}\n {%- endif -%}\n{%- endmacro -%}\n{%- macro strip_thinking(text) -%}\n {%- set ns = namespace(result='') -%}\n {%- for ",+ "part in text.split('<channel|>') -%}\n {%- if '<|channel>' in part -%}\n {%- set ns.result = ns.result + part.split('<|channel>')[0] -%}\n {%- else -%}\n {%- set ns.result = ns.result + ",+ "part -%}\n {%- endif -%}\n {%- endfor -%}\n {{- ns.result | trim -}}\n{%- endmacro -%}\n\n{%- macro format_tool_response_block(tool_name, response) -%}\n {{- '<|tool_response>' -}}\n {%- if response is ",+ "mapping -%}\n {{- 'response:' + tool_name + '{' -}}\n {%- for key, value in response | dictsort -%}\n {{- key -}}:{{- format_argument(value, escape_keys=False) -}}\n {%- if not loop.last ",+ "%},{% endif -%}\n {%- endfor -%}\n {{- '}' -}}\n {%- else -%}\n {{- 'response:' + tool_name + '{value:' + format_argument(response, escape_keys=False) + '}' -}}\n {%- endif -%}\n {{- ",+ "'<tool_response|>' -}}\n{%- endmacro -%}\n\n{#- ===== SETUP ===== -#}\n{%- set ns = namespace(prev_message_type=None, prev_non_tool_role=None) -%}\n{%- set loop_messages = messages -%}\n{%- set enable_thinking = ",+ "enable_thinking | default(false) -%}\n{%- set preserve_thinking = preserve_thinking | default(false) -%}\n{{- bos_token -}}\n{#- Handle System/Tool Definitions Block -#}\n{%- if enable_thinking or tools or (messages and ",+ "messages[0]['role'] in ['system', 'developer']) -%}\n {{- '<|turn>system\\n' -}}\n {#- Inject Thinking token at the very top of the FIRST system turn -#}\n {%- if enable_thinking -%}\n {{- '<|think|>\\n' -}}\n ",+ " {%- set ns.prev_message_type = 'think' -%}\n {%- endif -%}\n {%- if messages and messages[0]['role'] in ['system', 'developer'] -%}\n {%- if messages[0]['content'] is string -%}\n {{- ",+ "messages[0]['content'] | trim -}}\n {%- elif messages[0]['content'] is sequence -%}\n {%- for item in messages[0]['content'] -%}\n {{- item['text'] | trim + ' '-}}\n {%- endfor ",+ "-%}\n {%- endif -%}\n {%- set loop_messages = messages[1:] -%}\n {%- endif -%}\n {%- if tools -%}\n {%- for tool in tools %}\n {{- '<|tool>' -}}\n {{- ",+ "format_function_declaration(tool) | trim -}}\n {{- '<tool|>' -}}\n {%- endfor %}\n {%- set ns.prev_message_type = 'tool' -%}\n {%- endif -%}\n {{- '<turn|>\\n' -}}\n{%- endif %}\n\n{#- Pre-scan: ",+ "find last user message index for reasoning guard -#}\n{%- set ns_turn = namespace(last_user_idx=-1) -%}\n{%- for i in range(loop_messages | length) -%}\n {%- if loop_messages[i]['role'] == 'user' -%}\n {%- set ",+ "ns_turn.last_user_idx = i -%}\n {%- endif -%}\n{%- endfor -%}\n\n{#- Loop through messages -#}\n{%- for message in loop_messages -%}\n {%- if message['role'] != 'tool' -%}\n {%- set ns.prev_message_type = None -%}\n ",+ "{%- set role = 'model' if message['role'] == 'assistant' else message['role'] -%}\n {#- Detect continuation using tracked state — O(1) instead of O(n) backward scan -#}\n {%- set continue_same_model_turn = (role == ",+ "'model' and ns.prev_non_tool_role == 'assistant') -%}\n {%- if not continue_same_model_turn -%}\n {{- '<|turn>' + role + '\\n' }}\n {%- endif -%}\n\n {#- Render reasoning/reasoning_content as thinking channel ",+ "-#}\n {%- set thinking_text = message.get('reasoning') or message.get('reasoning_content') -%}\n {%- set thinking_gate = (loop.index0 > ns_turn.last_user_idx) or (preserve_thinking and message.get('tool_calls')) -%}\n",+ " {%- if thinking_text and thinking_gate -%}\n {{- '<|channel>thought\\n' + thinking_text + '\\n<channel|>' -}}\n {%- endif -%}\n\n {%- if message.get('tool_calls') -%}\n {%- for tool_call ",+ "in message.get('tool_calls') -%}\n {%- set function = tool_call['function'] -%}\n {{- '<|tool_call>call:' + function['name'] + '{' -}}\n {%- if function['arguments'] ",+ "is mapping -%}\n {%- set ns_args = namespace(found_first=false) -%}\n {%- for key, value in function['arguments'] | dictsort -%}\n {%- if ",+ "ns_args.found_first %},{% endif -%}\n {%- set ns_args.found_first = true -%}\n {{- key -}}:{{- format_argument(value, escape_keys=False) -}}\n ",+ "{%- endfor -%}\n {%- elif function['arguments'] is none -%}\n {%- elif function['arguments'] is string -%}\n {#- Pre-serialized args (e.g. an OpenAI JSON ",+ "string). We cannot JSON-parse\n portably in-template, so render non-fatally instead of erroring. Strip an\n outer {...} so it composes with the DSL braces rather than ",+ "double-wrapping.\n Prefer passing arguments as a mapping for exact Gemma DSL. -#}\n {%- set argstr = function['arguments'] | trim -%}\n {%- if ",+ "argstr[:1] == '{' and argstr[-1:] == '}' -%}\n {{- argstr[1:-1] -}}\n {%- else -%}\n {{- function['arguments'] -}}\n {%- ",+ "endif -%}\n {%- endif -%}\n {{- '}<tool_call|>' -}}\n {%- endfor -%}\n {%- set ns.prev_message_type = 'tool_call' -%}\n {%- endif -%}\n\n ",+ " {%- set ns_tr_out = namespace(flag=false) -%}\n {%- if message.get('tool_responses') -%}\n {#- Legacy: tool_responses embedded on the assistant message (Google/Gemma native) -#}\n ",+ "{%- for tool_response in message.get('tool_responses') -%}\n {{- format_tool_response_block(tool_response['name'] | default('unknown', true), tool_response['response']) -}}\n {%- set ",+ "ns_tr_out.flag = true -%}\n {%- set ns.prev_message_type = 'tool_response' -%}\n {%- endfor -%}\n {%- elif message.get('tool_calls') -%}\n {#- OpenAI Chat ",+ "Completions: forward-scan consecutive role:tool messages -#}\n {%- set ns_tool_scan = namespace(stopped=false) -%}\n {%- for k in range(loop.index0 + 1, loop_messages | length) -%}\n ",+ " {%- if ns_tool_scan.stopped -%}\n {%- elif loop_messages[k]['role'] != 'tool' -%}\n {%- set ns_tool_scan.stopped = true -%}\n {%- else -%}\n ",+ " {%- set follow = loop_messages[k] -%}\n {#- Resolve tool_call_id to function name -#}\n {%- set ns_tname = namespace(name=follow.get('name') or 'unknown') -%}\n ",+ " {%- for tc in message.get('tool_calls') -%}\n {%- if tc.get('id') == follow.get('tool_call_id') -%}\n {%- set ns_tname.name = ",+ "tc['function']['name'] -%}\n {%- endif -%}\n {%- endfor -%}\n {#- Handle content as string or content-parts array -#}\n {%- set ",+ "tool_body = follow.get('content') -%}\n {%- if tool_body is string -%}\n {{- format_tool_response_block(ns_tname.name, tool_body) -}}\n {%- elif ",+ "tool_body is sequence and tool_body is not string -%}\n {%- set ns_txt = namespace(s='') -%}\n {%- for part in tool_body -%}\n {%- if ",+ "part.get('type') == 'text' -%}\n {%- set ns_txt.s = ns_txt.s + (part.get('text') | default('')) -%}\n {%- endif -%}\n {%- endfor ",+ "-%}\n {{- format_tool_response_block(ns_tname.name, ns_txt.s) -}}\n {%- for part in tool_body -%}\n {%- if part.get('type') in ['image', ",+ "'image_url'] -%}\n {{- '<|image|>' -}}\n {%- elif part.get('type') in ['audio', 'input_audio'] -%}\n {{- '<|audio|>' -}}\n ",+ " {%- elif part.get('type') == 'video' -%}\n {{- '<|video|>' -}}\n {%- endif -%}\n {%- endfor -%}\n ",+ " {%- else -%}\n {{- format_tool_response_block(ns_tname.name, tool_body) -}}\n {%- endif -%}\n {%- set ns_tr_out.flag = true -%}\n ",+ " {%- set ns.prev_message_type = 'tool_response' -%}\n {%- endif -%}\n {%- endfor -%}\n {%- endif -%}\n\n {%- set captured_content -%}\n ",+ "{%- if message.get('content') is string -%}\n {%- if role == 'model' -%}\n {{- strip_thinking(message['content']) -}}\n {%- else -%}\n {{- ",+ "message['content'] | trim -}}\n {%- endif -%}\n {%- elif message.get('content') is sequence -%}\n {%- for item in message['content'] -%}\n {%- if item.get('type') ",+ "== 'text' -%}\n {%- if role == 'model' -%}\n {{- strip_thinking(item['text']) -}}\n {%- else -%}\n {{- item['text'] | trim ",+ "-}}\n {%- endif -%}\n {%- elif item.get('type') in ['image', 'image_url'] -%}\n {{- '<|image|>' -}}\n {%- elif item.get('type') in ",+ "['audio', 'input_audio'] -%}\n {{- '<|audio|>' -}}\n {%- elif item.get('type') == 'video' -%}\n {{- '<|video|>' -}}\n {%- endif -%}\n ",+ " {%- endfor -%}\n {%- endif -%}\n {%- endset -%}\n\n {{- captured_content -}}\n {%- set has_content = captured_content | trim | length > 0 -%}\n\n {#- Forward-scan: ",+ "find next non-tool message role for continuation detection -#}\n {%- set next_nt = namespace(role=None, found=false) -%}\n {%- for j in range(loop.index0 + 1, loop_messages | length) -%}\n {%- if ",+ "not next_nt.found -%}\n {%- if loop_messages[j]['role'] != 'tool' -%}\n {%- set next_nt.role = loop_messages[j]['role'] -%}\n {%- set next_nt.found = true -%}\n ",+ " {%- endif -%}\n {%- endif -%}\n {%- endfor -%}\n\n {%- set continues_into_next = (\n role == 'model'\n and next_nt.role == 'assistant'\n and (not ",+ "message.get('tool_calls') or ns_tr_out.flag)\n ) -%}\n\n {%- if ns.prev_message_type == 'tool_call' and not ns_tr_out.flag -%}\n {{- '<|tool_response>' -}}\n {%- elif continues_into_next -%}\n ",+ " {%- elif not (ns_tr_out.flag and not has_content and not next_nt.found) -%}\n {{- '<turn|>\\n' -}}\n {%- endif -%}\n\n {#- Track previous non-tool role for next iteration (avoids O(n) backward ",+ "scan) -#}\n {%- set ns.prev_non_tool_role = message['role'] -%}\n {%- endif -%}\n{%- endfor -%}\n\n{%- if add_generation_prompt -%}\n {%- if ns.prev_message_type != 'tool_response' and ns.prev_message_type != ",+ "'tool_call' -%}\n {{- '<|turn>model\\n' -}}\n {%- elif ns.prev_message_type == 'tool_response' and enable_thinking -%}\n {{- '<|channel>thought\\n' -}}\n {%- endif -%}\n{%- endif -%}\n",+ ),+ bos: "<bos>",+ eos: "<turn|>",+ },+ ];+}diff --git a/rust/crates/ensu/tests/llm_worker.rs b/rust/crates/ensu/tests/llm_worker.rsindex 611d407332..a1d3b0727a 100644--- a/rust/crates/ensu/tests/llm_worker.rs+++ b/rust/crates/ensu/tests/llm_worker.rs@@ -6,8 +6,16 @@ use std::sync::Arc;use std::thread;use ente_ensu::llm::{self, ChatMessage, ChatRequest, Context, ContextParams, GenerationEvent};+use ente_image::image_compression::{EncodedImageFormat, encode_rgb};++fn init_test_backend() {+ if let Ok(directory) = std::env::var("ENSU_TEST_BACKEND_DIR") {+ llm::init_backend_from_directory(&directory).unwrap();+ }+}fn model() -> llm::ModelRef {+ init_test_backend();llm::Model::load(llm::ModelLoadParams {model_path: std::env::var("ENSU_TEST_MODEL").unwrap(),n_gpu_layers: Some(99),@@ -150,6 +158,7 @@ fn concurrent_generations_preserve_output_and_can_drop_caller_model() {#[test]#[ignore = "requires ENSU_TEST_EMBED_MODEL"]fn concurrent_embeddings_match_serial_results() {+ init_test_backend();let model = llm::Model::load(llm::ModelLoadParams {model_path: std::env::var("ENSU_TEST_EMBED_MODEL").unwrap(),n_gpu_layers: Some(99),@@ -178,3 +187,110 @@ fn concurrent_embeddings_match_serial_results() {run.join().unwrap();}}++fn collect(context: &llm::ContextRef, request: ChatRequest) -> (String, llm::GenerationSummary) {+ let mut output = String::new();+ let summary = context+ .generate_chat_stream(request, &mut |event| {+ if let GenerationEvent::Text { text, .. } = event {+ output.push_str(&text);+ }+ })+ .unwrap();+ assert!(!output.is_empty());+ for thinking_marker in ["<think>", "<|think|>", "<|channel>thought"] {+ assert!(+ !output.contains(thinking_marker),+ "Unexpected reasoning: {output}"+ );+ }+ (output, summary)+}++#[test]+#[ignore = "requires ENSU_TEST_MODEL set to this device's RAM-tier default"]+fn default_model_counts_truncates_unicode_and_recovers_from_context_limits() {+ let context = context();+ let mut chat = request();+ chat.messages[0].content = "Reply with a short greeting: नमस्ते, 你好, hello 🐈.".to_owned();+ let measured = context.measure_text_chat_prompt(&chat).unwrap();+ let (_, summary) = collect(&context, chat.clone());+ assert_eq!(summary.prompt_tokens.unwrap() as usize, measured);++ chat.messages[0].content = "नमस्ते 🐈 你好 ".repeat(1_000);+ assert!(matches!(+ context.generate_chat_stream(chat.clone(), &mut |_| {}),+ Err(llm::Error::PromptTooLong { .. })+ ));+ let truncated = context+ .truncate_text_chat_messages(chat.messages.clone(), 128)+ .unwrap();+ assert!(chat.messages[0].content.starts_with(&truncated[0].content));+ chat.messages = truncated;+ assert!(context.measure_text_chat_prompt(&chat).unwrap() <= 128);+ collect(&context, chat);+}++#[test]+#[ignore = "requires ENSU_TEST_MODEL set to this device's RAM-tier default"]+fn default_model_reuses_multi_turn_cache_and_reloads() {+ let owner = model();+ let params = ContextParams {+ context_size: Some(512),+ n_threads: Some(4),+ n_batch: Some(256),+ };+ let warm = Context::new(&owner, params.clone()).unwrap();+ let first = request();+ let (answer, _) = collect(&warm, first.clone());+ let mut followup = first;+ followup.messages.push(ChatMessage {+ role: "assistant".to_owned(),+ content: answer,+ });+ followup.messages.push(ChatMessage {+ role: "user".to_owned(),+ content: "What comes next?".to_owned(),+ });+ let (cached, _) = collect(&warm, followup.clone());+ drop(warm);+ let fresh = Context::new(&owner, params).unwrap();+ let (uncached, _) = collect(&fresh, followup);+ assert_eq!(cached, uncached);+ drop(fresh);+ drop(owner);+ collect(&context(), request());+}++#[test]+#[ignore = "requires ENSU_TEST_MODEL and ENSU_TEST_MMPROJ for this device's RAM-tier default"]+fn default_model_handles_wide_and_tall_images_and_projector_reloads() {+ let owner = model();+ let context = Context::new(+ &owner,+ ContextParams {+ context_size: Some(4096),+ n_threads: Some(4),+ n_batch: Some(256),+ },+ )+ .unwrap();+ let mmproj = std::env::var("ENSU_TEST_MMPROJ").unwrap();+ let directory = tempfile::tempdir().unwrap();+ for (name, width, height) in [("wide.png", 1024, 96), ("tall.png", 96, 1024)] {+ let path = directory.path().join(name);+ let rgb = [255, 0, 0].repeat((width * height) as usize);+ let bytes = encode_rgb(&rgb, width, height, EncodedImageFormat::Png).unwrap();+ std::fs::write(&path, bytes).unwrap();+ context.prewarm_multimodal(mmproj.clone(), None).unwrap();+ let mut chat = request();+ chat.messages[0].content = "Describe the colour in this image.".to_owned();+ chat.image_paths = Some(vec![path.to_str().unwrap().to_owned()]);+ chat.mmproj_path = Some(mmproj.clone());+ collect(&context, chat.clone());+ context.release_multimodal();+ chat.messages[0].content.push_str("\n<__media__>");+ collect(&context, chat);+ collect(&context, request());+ }+}