π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
Complete the token and id-based selection sweep across the app layers
Manav Rathie8dea98d58c6
diff --git a/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/LlmModels.kt b/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/LlmModels.ktindex a44e11de52..0b76ba6cf6 100644--- a/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/LlmModels.kt+++ b/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/LlmModels.kt@@ -1,19 +1,13 @@package io.ente.ensu.llm-import io.ente.ensu.bindings.ConfigModelPresetimport io.ente.ensu.bindings.ModelTarget-import io.ente.ensu.bindings.mobileLlmTargetdata class LlmModelSelection(- val preset: ConfigModelPreset,+ val id: String,+ val modelTarget: ModelTarget,val contextLength: Int? = null,val maxTokens: Int? = null-) {- val id: String get() = preset.id-- val modelTarget: ModelTarget- get() = mobileLlmTarget(preset.id)-}+)data class DownloadProgress(val percent: Int?,diff --git a/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/ModelSettingsActions.kt b/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/ModelSettingsActions.ktindex c09cf2e72c..368c76fbd7 100644--- a/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/ModelSettingsActions.kt+++ b/mobile/native/android/apps/ensu/app/src/main/java/io/ente/ensu/llm/ModelSettingsActions.kt@@ -6,6 +6,7 @@ import io.ente.ensu.AppStateimport io.ente.ensu.bindings.ConfigDefaultsimport io.ente.ensu.bindings.DownloadErrorimport io.ente.ensu.bindings.LlmException+import io.ente.ensu.bindings.mobileLlmTargetimport io.ente.ensu.device.isChatSupportedimport io.ente.ensu.logging.FileLogRepositoryimport io.ente.ensu.logging.LogLevel@@ -319,7 +320,8 @@ internal class ModelSettingsActions(val maxTokens = settings.maxTokens.toIntOrNull()?.takeIf { it > 0 }return LlmModelSelection(- preset = preset,+ id = preset.id,+ modelTarget = mobileLlmTarget(preset.id),contextLength = contextLength,maxTokens = maxTokens)diff --git a/mobile/native/apple/apps/ensu/Ensu/Chat/ChatViewModel.swift b/mobile/native/apple/apps/ensu/Ensu/Chat/ChatViewModel.swiftindex 6de35ee63f..1a316a5358 100644--- a/mobile/native/apple/apps/ensu/Ensu/Chat/ChatViewModel.swift+++ b/mobile/native/apple/apps/ensu/Ensu/Chat/ChatViewModel.swift@@ -1574,7 +1574,7 @@ final class ChatViewModel: ObservableObject {fallback: summaryInput.fallback,existingSummary: existingSummary,provider: provider,- selection+ selection: selection)guard let summary else { return }diff --git a/mobile/native/apple/apps/ensu/Ensu/Services/LlmProvider.swift b/mobile/native/apple/apps/ensu/Ensu/Services/LlmProvider.swiftindex c4385cca8c..820dedda9c 100644--- a/mobile/native/apple/apps/ensu/Ensu/Services/LlmProvider.swift+++ b/mobile/native/apple/apps/ensu/Ensu/Services/LlmProvider.swift@@ -1,15 +1,10 @@import Foundationstruct LlmModelSelection: Equatable {- let preset: ConfigModelPreset+ let id: String+ let modelTarget: ModelTargetlet contextLength: Int?let maxTokens: Int?-- var id: String { preset.id }-- var modelTarget: ModelTarget {- try! mobileLlmTarget(modelId: preset.id)- }}struct DownloadProgress: Equatable {diff --git a/mobile/native/apple/apps/ensu/Ensu/Services/ModelDownloader.swift b/mobile/native/apple/apps/ensu/Ensu/Services/ModelDownloader.swiftindex ec439bb454..13f050e605 100644--- a/mobile/native/apple/apps/ensu/Ensu/Services/ModelDownloader.swift+++ b/mobile/native/apple/apps/ensu/Ensu/Services/ModelDownloader.swift@@ -51,15 +51,15 @@ final class ModelDownloader {}func modelDir(_ target: ModelTarget) -> URL {- URL(fileURLWithPath: core.modelDir(target))+ URL(fileURLWithPath: core.modelDir(target: target))}func llmModelPath(_ target: ModelTarget) -> URL? {- core.llmModelPath(target).map { URL(fileURLWithPath: $0) }+ core.llmModelPath(target: target).map { URL(fileURLWithPath: $0) }}func llmMmprojPath(_ target: ModelTarget) -> URL? {- core.llmMmprojPath(target).map { URL(fileURLWithPath: $0) }+ core.llmMmprojPath(target: target).map { URL(fileURLWithPath: $0) }}func voiceActivityModelPath() -> URL {@@ -67,11 +67,11 @@ final class ModelDownloader {}func isDownloaded(_ target: ModelTarget) -> Bool {- core.isDownloaded(target)+ core.isDownloaded(target: target)}func removeDownloaded(_ target: ModelTarget) -> Bool {- core.removeDownloaded(target)+ core.removeDownloaded(target: target)}func cancel() {@@ -97,7 +97,7 @@ final class ModelDownloader {guard started else { throw DownloadAlreadyActiveError() }defer { activeLock.withLock { activeToken = nil } }- if targets.allSatisfy({ self.core.isDownloaded($0) }) {+ if targets.allSatisfy({ self.core.isDownloaded(target: $0) }) {return}diff --git a/mobile/native/apple/apps/ensu/Ensu/Services/ModelSettingsStore.swift b/mobile/native/apple/apps/ensu/Ensu/Services/ModelSettingsStore.swiftindex 62fe957a70..776ead4909 100644--- a/mobile/native/apple/apps/ensu/Ensu/Services/ModelSettingsStore.swift+++ b/mobile/native/apple/apps/ensu/Ensu/Services/ModelSettingsStore.swift@@ -99,7 +99,8 @@ final class ModelSettingsStore: ObservableObject {let presets = [defaultModel] + defaults.mobileModelPresetslet preset = presets.first(where: { $0.id == modelId }) ?? defaultModelreturn LlmModelSelection(- preset: preset,+ id: preset.id,+ modelTarget: try! mobileLlmTarget(modelId: preset.id),contextLength: Int(contextLength),maxTokens: Int(maxTokens).flatMap { $0 > 0 ? $0 : nil })diff --git a/rust/apps/ensu/src-tauri/src/commands/llm.rs b/rust/apps/ensu/src-tauri/src/commands/llm.rsindex bd138dbedf..7c3e7e1f5a 100644--- a/rust/apps/ensu/src-tauri/src/commands/llm.rs+++ b/rust/apps/ensu/src-tauri/src/commands/llm.rs@@ -1,6 +1,6 @@use std::panic::{AssertUnwindSafe, catch_unwind};use std::path::PathBuf;-use std::sync::{Arc, Mutex};+use std::sync::{Arc, Mutex, PoisonError};use std::time::{Duration, Instant};use ente_ensu::llm;@@ -22,14 +22,16 @@ pub struct State {pub struct ModelDownloadState {downloader: Arc<ModelDownloader>,models_dir: PathBuf,+ active_token: Mutex<Option<download::CancellationToken>>,}impl ModelDownloadState {- pub fn new(models_dir: PathBuf) -> Result<Self, download::Error> {- Ok(Self {- downloader: Arc::new(ModelDownloader::new(&models_dir)?),+ pub fn new(models_dir: PathBuf) -> Self {+ Self {+ downloader: Arc::new(ModelDownloader::new(&models_dir)),models_dir,- })+ active_token: Mutex::new(None),+ }}}@@ -280,31 +282,62 @@ pub async fn llm_download_model() -> Result<(), ApiError> {let downloader = Arc::clone(&state.downloader);let target = desktop_target(&model_id)?;- async_runtime::spawn_blocking(move || {+ let token = {+ let mut slot = state+ .active_token+ .lock()+ .unwrap_or_else(PoisonError::into_inner);+ if slot.is_some() {+ return Err(ApiError::new(+ "download_active",+ "A model download is already in progress",+ ));+ }+ let token = download::CancellationToken::new();+ *slot = Some(token.clone());+ token+ };+ let result = async_runtime::spawn_blocking(move || {downloader- .download(std::slice::from_ref(&target), |progress| {- if let Some(line) = &progress.log_line {- logging::log("LLMDownload", line.clone());- }- let _ = window.emit(- "llm-download-progress",- DownloadProgress {- percent: progress.percent,- status: progress.status,- bytes_downloaded: progress.downloaded_bytes,- total_bytes: progress.total_bytes,- },- );- })+ .download(+ std::slice::from_ref(&target),+ |progress| {+ if let Some(line) = &progress.log_line {+ logging::log("LLMDownload", line.clone());+ }+ let _ = window.emit(+ "llm-download-progress",+ DownloadProgress {+ percent: progress.percent,+ status: progress.status,+ bytes_downloaded: progress.downloaded_bytes,+ total_bytes: progress.total_bytes,+ },+ );+ },+ &token,+ ).map_err(|err| ApiError::new(download_code(&err), err.to_string()))}).await- .map_err(|_| fs_thread_error())?+ .map_err(|_| fs_thread_error());+ *state+ .active_token+ .lock()+ .unwrap_or_else(PoisonError::into_inner) = None;+ result?}#[tauri::command]pub fn llm_cancel_model_download(state: TauriState<'_, ModelDownloadState>) {- state.downloader.cancel();+ if let Some(token) = state+ .active_token+ .lock()+ .unwrap_or_else(PoisonError::into_inner)+ .as_ref()+ {+ token.cancel();+ }}#[tauri::command]diff --git a/rust/crates/ensu/src/model/migrations.rs b/rust/crates/ensu/src/model/migrations.rsindex 1cfd291881..a4495b0d30 100644--- a/rust/crates/ensu/src/model/migrations.rs+++ b/rust/crates/ensu/src/model/migrations.rs@@ -47,7 +47,10 @@ pub fn migrate_desktop_models(let defaults = config::defaults();let selected = model_url.and_then(|model_url| {legacy_selected_preset_id(- std::iter::once(&defaults.desktop_default_model).chain(&defaults.desktop_model_presets),+ std::iter::once(&defaults.mobile_default_model)+ .chain(&defaults.mobile_model_presets)+ .chain(std::iter::once(&defaults.desktop_default_model))+ .chain(&defaults.desktop_model_presets),model_url,mmproj_url,)diff --git a/rust/crates/ensu/src/model/mod.rs b/rust/crates/ensu/src/model/mod.rsindex 045d4b7ddc..97e4e83089 100644--- a/rust/crates/ensu/src/model/mod.rs+++ b/rust/crates/ensu/src/model/mod.rs@@ -56,9 +56,9 @@ pub fn mobile_llm_target(model_id: &str) -> Result<ModelTarget, InvalidPreset> {pub fn desktop_llm_target(model_id: &str) -> Result<ModelTarget, InvalidPreset> {let defaults = config::defaults();- let preset = defaults- .desktop_model_presets- .iter()+ let preset = std::iter::once(&defaults.mobile_default_model)+ .chain(&defaults.mobile_model_presets)+ .chain(&defaults.desktop_model_presets).find(|preset| preset.id == model_id).unwrap_or(&defaults.desktop_default_model);llm_target(preset)@@ -168,9 +168,13 @@ mod tests {}#[test]- fn desktop_llm_target_resolves_or_falls_back_to_default() {+ fn desktop_llm_target_resolves_union_or_falls_back_to_default() {let defaults = crate::config::defaults();- for preset in &defaults.desktop_model_presets {+ for preset in std::iter::once(&defaults.mobile_default_model)+ .chain(&defaults.mobile_model_presets)+ .chain(std::iter::once(&defaults.desktop_default_model))+ .chain(&defaults.desktop_model_presets)+ {assert_eq!(desktop_llm_target(&preset.id).unwrap(),llm_target(preset).unwrap(),@@ -179,10 +183,6 @@ mod tests {);}let default = llm_target(&defaults.desktop_default_model).unwrap();- assert_eq!(- desktop_llm_target(&defaults.desktop_default_model.id).unwrap(),- default- );assert_eq!(desktop_llm_target("").unwrap(), default);assert_eq!(desktop_llm_target("no-such-model").unwrap(), default);}diff --git a/rust/crates/photos/Cargo.toml b/rust/crates/photos/Cargo.tomlindex 78b4f27c1b..8f6af9bcf2 100644--- a/rust/crates/photos/Cargo.toml+++ b/rust/crates/photos/Cargo.toml@@ -62,6 +62,7 @@ serde = { version = "1", features = ["derive"] }serde_json.workspace = truetar = "0.4"tempfile = "3.12"+tokio = { version = "1", features = ["rt"] }url = "2.5.8"[[test]]diff --git a/rust/crates/photos/tests/support/ml_indexing.rs b/rust/crates/photos/tests/support/ml_indexing.rsindex ef9a24aeeb..61a4a04cca 100644--- a/rust/crates/photos/tests/support/ml_indexing.rs+++ b/rust/crates/photos/tests/support/ml_indexing.rs@@ -169,27 +169,64 @@ pub(crate) struct MlIndexingTestContext {runtime_config: MlRuntimeConfig,}+struct AssetFetcher {+ runtime: tokio::runtime::Runtime,+ downloader: download::Downloader,+}++impl AssetFetcher {+ fn new() -> Result<Self> {+ Ok(Self {+ runtime: tokio::runtime::Builder::new_current_thread()+ .enable_all()+ .build()+ .context("build download runtime")?,+ downloader: download::Downloader::new().context("build downloader")?,+ })+ }++ fn fetch(&self, targets: Vec<download::Target>) -> Result<()> {+ self.runtime.block_on(self.downloader.download(+ targets,+ |_| {},+ download::CancellationToken::default(),+ ))?;+ Ok(())+ }+}+impl MlIndexingTestContext {pub(crate) fn load() -> Result<Self> {let repo_root = repo_root()?;let asset_lock = load_asset_lock(&repo_root)?;let cache_dir = cache_dir(&repo_root);-- let manifest_path = resolve_document_asset(&cache_dir, "manifest", &asset_lock.manifest)?;- let golden_path =- resolve_document_asset(&cache_dir, "python-golden", &asset_lock.python_golden)?;+ let fetcher = AssetFetcher::new()?;++ let manifest_path =+ resolve_document_asset(&fetcher, &cache_dir, "manifest", &asset_lock.manifest)?;+ let golden_path = resolve_document_asset(+ &fetcher,+ &cache_dir,+ "python-golden",+ &asset_lock.python_golden,+ )?;let manifest = load_manifest(&manifest_path)?;let golden_results = load_golden_results(&golden_path)?;- fetch_fixtures(&cache_dir, &asset_lock.fixture_base_url, &manifest)?;+ fetch_fixtures(+ &fetcher,+ &cache_dir,+ &asset_lock.fixture_base_url,+ &manifest,+ )?;let onnx_runtime_library =- resolve_onnx_runtime_library(&cache_dir, &asset_lock.onnx_runtime)?;+ resolve_onnx_runtime_library(&fetcher, &cache_dir, &asset_lock.onnx_runtime)?;let _ = ort::init_from(&onnx_runtime_library).context("load ONNX Runtime dynamic library")?.commit();let runtime_config = MlRuntimeConfig {- model_paths: resolve_model_paths(&cache_dir, &asset_lock.models)?,+ model_paths: resolve_model_paths(&fetcher, &cache_dir, &asset_lock.models)?,provider_policy: ExecutionProviderPolicy {prefer_coreml: false,prefer_nnapi: false,@@ -520,9 +557,14 @@ fn file_id_for_manifest_path(path: &str) -> Result<String> {.with_context(|| format!("derive file_id from manifest path '{path}'"))}-fn resolve_document_asset(cache_dir: &Path, label: &str, asset: &DocumentAsset) -> Result<PathBuf> {+fn resolve_document_asset(+ fetcher: &AssetFetcher,+ cache_dir: &Path,+ label: &str,+ asset: &DocumentAsset,+) -> Result<PathBuf> {let target = cache_path_for(cache_dir, "documents", &asset.path, &asset.sha256)?;- ensure_remote_asset(&asset.url, &asset.sha256, &target, label)+ ensure_remote_asset(fetcher, &asset.url, &asset.sha256, &target, label)}fn fixture_url(fixture_base_url: &str, path: &str) -> Result<String> {@@ -534,6 +576,7 @@ fn fixture_url(fixture_base_url: &str, path: &str) -> Result<String> {}fn fetch_fixtures(+ fetcher: &AssetFetcher,cache_dir: &Path,fixture_base_url: &str,manifest: &FixtureManifest,@@ -545,22 +588,19 @@ fn fetch_fixtures(Ok(download::Target {label: file_id_for_manifest_path(&fixture.path)?,url: fixture_url(fixture_base_url, &fixture.path)?,- destination_path: cache_path_for(- cache_dir,- "fixtures",- &fixture.path,- &fixture.sha256,- )?- .display()- .to_string(),sha256: normalize_sha256(&fixture.sha256),+ destination: cache_path_for(cache_dir, "fixtures", &fixture.path, &fixture.sha256)?,})}).collect::<Result<Vec<_>>>()?;- download::fetch(targets, |_| {}, || false).context("fetch fixtures")+ fetcher.fetch(targets).context("fetch fixtures")}-fn resolve_onnx_runtime_library(cache_dir: &Path, assets: &OnnxRuntimeAssets) -> Result<PathBuf> {+fn resolve_onnx_runtime_library(+ fetcher: &AssetFetcher,+ cache_dir: &Path,+ assets: &OnnxRuntimeAssets,+) -> Result<PathBuf> {let target_key = onnx_runtime_target_key()?;let archive = assets.archives.get(&target_key).with_context(|| {let supported = assets@@ -582,6 +622,7 @@ fn resolve_onnx_runtime_library(cache_dir: &Path, assets: &OnnxRuntimeAssets) ->&archive.sha256,)?;let archive_path = ensure_remote_asset(+ fetcher,&archive.url,&archive.sha256,&archive_path,@@ -672,10 +713,16 @@ fn extract_tgz_member(archive_path: &Path, member_path: &str, target: &Path) ->);}-fn resolve_model_paths(cache_dir: &Path, models: &ModelAssets) -> Result<ModelPaths> {- let face_detection = resolve_model_asset(cache_dir, "face-detection", &models.face_detection)?;- let face_embedding = resolve_model_asset(cache_dir, "face-embedding", &models.face_embedding)?;- let clip_image = resolve_model_asset(cache_dir, "clip-image", &models.clip_image)?;+fn resolve_model_paths(+ fetcher: &AssetFetcher,+ cache_dir: &Path,+ models: &ModelAssets,+) -> Result<ModelPaths> {+ let face_detection =+ resolve_model_asset(fetcher, cache_dir, "face-detection", &models.face_detection)?;+ let face_embedding =+ resolve_model_asset(fetcher, cache_dir, "face-embedding", &models.face_embedding)?;+ let clip_image = resolve_model_asset(fetcher, cache_dir, "clip-image", &models.clip_image)?;Ok(ModelPaths {face_detection: face_detection.to_string_lossy().into_owned(),@@ -691,28 +738,31 @@ fn resolve_model_paths(cache_dir: &Path, models: &ModelAssets) -> Result<ModelPa})}-fn resolve_model_asset(cache_dir: &Path, label: &str, asset: &ModelAsset) -> Result<PathBuf> {+fn resolve_model_asset(+ fetcher: &AssetFetcher,+ cache_dir: &Path,+ label: &str,+ asset: &ModelAsset,+) -> Result<PathBuf> {let target = cache_path_for(cache_dir, "models", &asset.file_name, &asset.sha256)?;- ensure_remote_asset(&asset.url, &asset.sha256, &target, label)+ ensure_remote_asset(fetcher, &asset.url, &asset.sha256, &target, label)}fn ensure_remote_asset(+ fetcher: &AssetFetcher,url: &str,expected_sha256: &str,target: &Path,label: &str,) -> Result<PathBuf> {- download::fetch(- vec![download::Target {+ fetcher+ .fetch(vec![download::Target {label: label.to_string(),url: url.to_string(),- destination_path: target.display().to_string(),sha256: normalize_sha256(expected_sha256),- }],- |_| {},- || false,- )- .with_context(|| format!("download {label} from {url}"))?;+ destination: target.to_path_buf(),+ }])+ .with_context(|| format!("download {label} from {url}"))?;Ok(target.to_path_buf())}diff --git a/web/apps/ensu/src/services/llm/provider.ts b/web/apps/ensu/src/services/llm/provider.tsindex 0b24d5e10a..487ef436ef 100644--- a/web/apps/ensu/src/services/llm/provider.ts+++ b/web/apps/ensu/src/services/llm/provider.ts@@ -73,14 +73,6 @@ interface TauriLlmModelDownloadProgress {totalBytes?: number;}-interface ModelTarget {- id: string;- url: string;- sha256: string;- mmprojUrl?: string | null;- mmprojSha256?: string | null;-}-interface TauriModelStatus {modelPath: string;mmprojPath?: string | null;@@ -259,7 +251,7 @@ export class LlmProvider {};}- const status = await this.modelStatus(this.resolveTarget(model));+ const status = await this.modelStatus(model.id);return {model,modelPath: status.modelPath,@@ -279,11 +271,8 @@ export class LlmProvider {const { model, contextSize } = this.resolveRuntimeSettings(settings);const contextKey = JSON.stringify({ contextSize });- const target =- this.backend.kind === "tauri"- ? this.resolveTarget(model)- : undefined;- const status = target ? await this.modelStatus(target) : undefined;+ const modelId = this.backend.kind === "tauri" ? model.id : undefined;+ const status = modelId ? await this.modelStatus(modelId) : undefined;const modelPath = status?.modelPath ?? model.url;const mmprojPath = status?.mmprojPath ?? undefined;@@ -350,8 +339,8 @@ export class LlmProvider {this.currentMmprojPath = undefined;this.currentContextKey = undefined;- if (target && !(await this.modelStatus(target)).downloaded) {- await this.downloadModelNative(target);+ if (modelId && !(await this.modelStatus(modelId)).downloaded) {+ await this.downloadModelNative(modelId);}if (emitProgress) {@@ -573,28 +562,18 @@ export class LlmProvider {].find((preset) => preset.id == modelId);}- private resolveTarget(model: ModelInfo): ModelTarget {- return {- id: model.id,- url: model.url,- sha256: model.sha256,- mmprojUrl: model.mmprojUrl ?? null,- mmprojSha256: model.mmprojSha256 ?? null,- };- }-- private async modelStatus(target: ModelTarget): Promise<TauriModelStatus> {+ private async modelStatus(modelId: string): Promise<TauriModelStatus> {const { invoke } = await import("@tauri-apps/api/core");- return invoke<TauriModelStatus>("llm_model_status", { target });+ return invoke<TauriModelStatus>("llm_model_status", { modelId });}- private async downloadModelNative(target: ModelTarget) {+ private async downloadModelNative(modelId: string) {const [{ invoke }, { listen }] = await Promise.all([import("@tauri-apps/api/core"),import("@tauri-apps/api/event"),]);- log.info("LLM native download start", { target });+ log.info("LLM native download start", { modelId });this.emitProgress({percent: 0,status: "Starting download...",@@ -616,8 +595,8 @@ export class LlmProvider {);try {- await invoke("llm_download_model", { target });- log.info("LLM native download complete", { target });+ await invoke("llm_download_model", { modelId });+ log.info("LLM native download complete", { modelId });} finally {this.downloadActive = false;unlisten();