π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
Use named errors for native ML failures
Manav Rathi211d97f4c2d8
diff --git a/desktop/src/main/services/ml-worker.ts b/desktop/src/main/services/ml-worker.tsindex 06f17a9379..81f8573426 100644--- a/desktop/src/main/services/ml-worker.ts+++ b/desktop/src/main/services/ml-worker.ts@@ -50,27 +50,6 @@ const parseInitData = (data: unknown) => {type MLNative = typeof MLNativeModule;-type MLWorkerAnalyzeImageErrorKind = "init" | "ort" | "image" | "misc";--interface MLWorkerAnalyzeImageError {- kind: MLWorkerAnalyzeImageErrorKind;- message: string;-}--type MLWorkerAnalyzeImageResponse =- | { ok: true; result: MLNativeModule.AnalyzeImageResult }- | { ok: false; error: MLWorkerAnalyzeImageError };--class CategorizedMLWorkerError extends Error {- constructor(- public readonly kind: MLWorkerAnalyzeImageErrorKind,- message: string,- ) {- super(message);- this.name = "CategorizedMLWorkerError";- }-}-let _native: MLNative | undefined;let _assetStore: MLNativeModule.AssetStore | undefined;let _nativeLoadError: string | undefined;@@ -111,20 +90,22 @@ const logRustEntry = (level: string, target: string, message: string) => {const mlNative = () => {if (_native) return _native;- throw new CategorizedMLWorkerError(- "init",- `ML is unavailable: ${_nativeLoadError ?? "not loaded"}`,- );+ throw mlInitError(`ML is unavailable: ${_nativeLoadError ?? "not loaded"}`);};const assetStore = () => {if (_assetStore) return _assetStore;- throw new CategorizedMLWorkerError(- "init",+ throw mlInitError(`ML is unavailable: ${_nativeLoadError ?? "asset store not loaded"}`,);};+const mlInitError = (message: string) => {+ const error = new Error(message);+ error.name = "ml_init";+ return error;+};+const _indexingModelPaths = new Map<number,Promise<MLNativeModule.ModelPaths>@@ -177,52 +158,15 @@ export interface MLWorkerAnalyzeImageRequest {generateFaceCrops: boolean;}-export const analyzeImage = async (- req: MLWorkerAnalyzeImageRequest,-): Promise<MLWorkerAnalyzeImageResponse> => {- try {- return { ok: true, result: await analyzeImageOrThrow(req) };- } catch (e) {- return { ok: false, error: categorizeAnalyzeImageError(e) };- }-};--const analyzeImageOrThrow = async (req: MLWorkerAnalyzeImageRequest) => {+export const analyzeImage = async (req: MLWorkerAnalyzeImageRequest) => {const native = mlNative();- return analyzeImageOnce(native, req);-};--// A corrupt on-disk model will fail every retry, so report an init failure that-// suspends indexing until restart.-const categorizeAnalyzeImageError = (e: unknown): MLWorkerAnalyzeImageError => {- if (e instanceof CategorizedMLWorkerError) {- return { kind: e.kind, message: e.message };- }-- const message = e instanceof Error ? e.message : String(e);- const kind: MLWorkerAnalyzeImageErrorKind = message.startsWith(- "CorruptModel: ",- )- ? "init"- : message.startsWith("Decode: ") || message.startsWith("Image: ")- ? "image"- : message.startsWith("Ort: ") || message.startsWith("Runtime: ")- ? "ort"- : "misc";- return { kind, message };-};--const analyzeImageOnce = async (- native: MLNative,- req: MLWorkerAnalyzeImageRequest,-) => {const modelPaths = await indexingModelPaths(req.runFaces,req.runClip,req.runPets,);ensureMLRuntime(native, modelPaths);- return await native.analyzeImage({+ return native.analyzeImage({fileId: req.fileID,imageBytes: uint8ArrayToBuffer(req.bytes),runFaces: req.runFaces,diff --git a/rust/bindings/napi/photos/src/lib.rs b/rust/bindings/napi/photos/src/lib.rsindex 0d9926da1b..87bdec5d29 100644--- a/rust/bindings/napi/photos/src/lib.rs+++ b/rust/bindings/napi/photos/src/lib.rs@@ -190,7 +190,8 @@ pub struct AnalyzeImageTask {}impl Task for AnalyzeImageTask {- type Output = shared_indexing::AnalyzeImageResult;+ // Retain MlError until resolve, where the JS thread can assign Error.name.+ type Output = std::result::Result<shared_indexing::AnalyzeImageResult, MlError>;type JsValue = AnalyzeImageResult;fn compute(&mut self) -> Result<Self::Output> {@@ -198,11 +199,14 @@ impl Task for AnalyzeImageTask {.req.take().ok_or_else(|| Error::from_reason("Runtime: analyze task computed twice"))?;- shared_indexing::analyze_image(req).map_err(ml_error_to_napi)+ Ok(shared_indexing::analyze_image(req))}- fn resolve(&mut self, _env: Env, output: Self::Output) -> Result<Self::JsValue> {- Ok(to_napi_analyze_image_result(output))+ fn resolve(&mut self, env: Env, output: Self::Output) -> Result<Self::JsValue> {+ match output {+ Ok(output) => Ok(to_napi_analyze_image_result(output)),+ Err(error) => Err(ml_error_to_napi(&env, error)?),+ }}}@@ -246,7 +250,7 @@ impl Task for RunClipTextTask {.req.take().ok_or_else(|| Error::from_reason("Runtime: clip text task computed twice"))?;- shared_indexing::run_clip_text(req).map_err(ml_error_to_napi)+ shared_indexing::run_clip_text(req).map_err(|error| Error::from_reason(error.to_string()))}fn resolve(&mut self, _env: Env, output: Self::Output) -> Result<Self::JsValue> {@@ -269,32 +273,20 @@ pub fn run_clip_text(req: RunClipTextRequest) -> AsyncTask<RunClipTextTask> {#[napi]pub fn tokenize_clip_text(text: String, vocab_path: String) -> Result<Vec<i32>> {- shared_indexing::tokenize_clip_text(&text, &vocab_path).map_err(ml_error_to_napi)-}--// Error kind prefixes survive Comlink's MessagePort serialization.-fn ml_error_to_napi(error: MlError) -> Error {- let kind = match &error {- MlError::InvalidRequest(_) => "InvalidRequest",- MlError::Decode(_) => "Decode",- MlError::Image(_) => "Image",- MlError::Preprocess(_) => "Preprocess",- MlError::Ort(_) => "Ort",- MlError::CorruptModel(_) => "CorruptModel",- MlError::Postprocess(_) => "Postprocess",- MlError::Runtime(_) => "Runtime",- };- let detail = match error {- MlError::InvalidRequest(message)- | MlError::Decode(message)- | MlError::Image(message)- | MlError::Preprocess(message)- | MlError::Ort(message)- | MlError::CorruptModel(message)- | MlError::Postprocess(message)- | MlError::Runtime(message) => message,+ shared_indexing::tokenize_clip_text(&text, &vocab_path)+ .map_err(|error| Error::from_reason(error.to_string()))+}++fn ml_error_to_napi(env: &Env, error: MlError) -> Result<Error> {+ let name = match &error {+ MlError::CorruptModel(_) => "ml_init",+ MlError::Decode(_) => "ml_decode",+ MlError::Image(_) => "ml_image",+ _ => return Ok(Error::from_reason(error.to_string())),};- Error::from_reason(format!("{kind}: {detail}"))+ let mut value = env.create_error(Error::from_reason(error.to_string()))?;+ value.set_named_property("name", name)?;+ Ok(Error::from(value.into_unknown(env)?))}fn asset_error_to_napi(error: ente_assets::download::Error) -> Error {diff --git a/web/packages/base/types/ipc.ts b/web/packages/base/types/ipc.tsindex 5ed8f3702f..633a58aaad 100644--- a/web/packages/base/types/ipc.ts+++ b/web/packages/base/types/ipc.ts@@ -131,7 +131,7 @@ export interface ElectronMLWorker {fsStatMtime: (path: string) => Promise<number>;analyzeImage: (request: MLWorkerAnalyzeImageRequest,- ) => Promise<MLWorkerAnalyzeImageResponse>;+ ) => Promise<MLWorkerAnalyzeImageResult>;releaseMLRuntime: () => Promise<void>;computeCLIPTextEmbeddingIfAvailable: (text: string,@@ -147,17 +147,6 @@ export interface MLWorkerAnalyzeImageRequest {generateFaceCrops: boolean;}-export type MLWorkerAnalyzeImageErrorKind = "init" | "ort" | "image" | "misc";--export interface MLWorkerAnalyzeImageError {- kind: MLWorkerAnalyzeImageErrorKind;- message: string;-}--export type MLWorkerAnalyzeImageResponse =- | { ok: true; result: MLWorkerAnalyzeImageResult }- | { ok: false; error: MLWorkerAnalyzeImageError };-export interface MLWorkerFaceResult {faceId: string;detection: { score: number; boxXyxy: number[]; keypoints: number[][] };diff --git a/web/packages/new/photos/services/ml/worker.ts b/web/packages/new/photos/services/ml/worker.tsindex cc47ddcea8..1614668797 100644--- a/web/packages/new/photos/services/ml/worker.ts+++ b/web/packages/new/photos/services/ml/worker.ts@@ -7,7 +7,6 @@ import log from "ente-base/log";import { logUnhandledErrorsAndRejectionsInWorker } from "ente-base/log-web";import type {ElectronMLWorker,- MLWorkerAnalyzeImageErrorKind,MLWorkerAnalyzeImageRequest,MLWorkerAnalyzeImageResult,} from "ente-base/types/ipc";@@ -194,7 +193,7 @@ export class MLWorker {this.resolvePendingIndexRequests();}- private suspendForMLInitError(error: MLAnalyzeError) {+ private suspendForMLInitError(error: Error) {log.error("Suspending ML indexing because the native runtime failed to initialize",error,@@ -519,53 +518,37 @@ const analyzeImageWithConversionFallback = async (conversionError instanceof Error? conversionError.message: String(conversionError);- throw new MLAnalyzeError(- "image",+ const error = new Error(`JPEG conversion fallback failed: ${message}`,);+ error.name = "ml_image";+ throw error;}return await enqueueAnalyzeImage(electron, { ...request, bytes });}};-const isMLImageError = (e: unknown): e is MLAnalyzeError =>- e instanceof MLAnalyzeError && e.kind == "image";+const isMLImageError = (e: unknown): e is Error =>+ e instanceof Error && (e.name == "ml_decode" || e.name == "ml_image");const isMLDecodeError = (e: unknown) =>- isMLImageError(e) && e.message.startsWith("Decode: ");+ e instanceof Error && e.name == "ml_decode";-const isMLInitError = (e: unknown): e is MLAnalyzeError =>- e instanceof MLAnalyzeError && e.kind == "init";--class MLAnalyzeError extends Error {- constructor(- public readonly kind: MLWorkerAnalyzeImageErrorKind,- message: string,- ) {- super(message);- this.name = "MLAnalyzeError";- }-}+const isMLInitError = (e: unknown): e is Error =>+ e instanceof Error && e.name == "ml_init";// Native model sessions serialize anyway; keep only one inference outstanding.let _analyzeImageQueue: Promise<unknown> = Promise.resolve();-let _blockingMLInitError: MLAnalyzeError | undefined;+let _blockingMLInitError: Error | undefined;const enqueueAnalyzeImage = (electron: ElectronMLWorker,request: MLWorkerAnalyzeImageRequest,): Promise<MLWorkerAnalyzeImageResult> => {- const result = _analyzeImageQueue.then(async () => {- const response = await electron.analyzeImage(request);- if (!response.ok) {- throw new MLAnalyzeError(- response.error.kind,- response.error.message,- );- }- return response.result;- });+ const result = _analyzeImageQueue.then(() =>+ electron.analyzeImage(request),+ );_analyzeImageQueue = result.catch(() => undefined);return result;};