Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

Use named errors for native ML failures

Manav Rathi211d97f4c2d8
diff --git a/desktop/src/main/services/ml-worker.ts b/desktop/src/main/services/ml-worker.ts
index 06f17a9379..81f8573426 100644
--- a/desktop/src/main/services/ml-worker.ts
+++ b/desktop/src/main/services/ml-worker.ts
@@ -50,27 +50,6 @@ const parseInitData = (data: unknown) => {
type MLNative = typeof MLNativeModule;
-type MLWorkerAnalyzeImageErrorKind = "init" | "ort" | "image" | "misc";
-
-interface MLWorkerAnalyzeImageError {
- kind: MLWorkerAnalyzeImageErrorKind;
- message: string;
-}
-
-type MLWorkerAnalyzeImageResponse =
- | { ok: true; result: MLNativeModule.AnalyzeImageResult }
- | { ok: false; error: MLWorkerAnalyzeImageError };
-
-class CategorizedMLWorkerError extends Error {
- constructor(
- public readonly kind: MLWorkerAnalyzeImageErrorKind,
- message: string,
- ) {
- super(message);
- this.name = "CategorizedMLWorkerError";
- }
-}
-
let _native: MLNative | undefined;
let _assetStore: MLNativeModule.AssetStore | undefined;
let _nativeLoadError: string | undefined;
@@ -111,20 +90,22 @@ const logRustEntry = (level: string, target: string, message: string) => {
const mlNative = () => {
if (_native) return _native;
- throw new CategorizedMLWorkerError(
- "init",
- `ML is unavailable: ${_nativeLoadError ?? "not loaded"}`,
- );
+ throw mlInitError(`ML is unavailable: ${_nativeLoadError ?? "not loaded"}`);
};
const assetStore = () => {
if (_assetStore) return _assetStore;
- throw new CategorizedMLWorkerError(
- "init",
+ throw mlInitError(
`ML is unavailable: ${_nativeLoadError ?? "asset store not loaded"}`,
);
};
+const mlInitError = (message: string) => {
+ const error = new Error(message);
+ error.name = "ml_init";
+ return error;
+};
+
const _indexingModelPaths = new Map<
number,
Promise<MLNativeModule.ModelPaths>
@@ -177,52 +158,15 @@ export interface MLWorkerAnalyzeImageRequest {
generateFaceCrops: boolean;
}
-export const analyzeImage = async (
- req: MLWorkerAnalyzeImageRequest,
-): Promise<MLWorkerAnalyzeImageResponse> => {
- try {
- return { ok: true, result: await analyzeImageOrThrow(req) };
- } catch (e) {
- return { ok: false, error: categorizeAnalyzeImageError(e) };
- }
-};
-
-const analyzeImageOrThrow = async (req: MLWorkerAnalyzeImageRequest) => {
+export const analyzeImage = async (req: MLWorkerAnalyzeImageRequest) => {
const native = mlNative();
- return analyzeImageOnce(native, req);
-};
-
-// A corrupt on-disk model will fail every retry, so report an init failure that
-// suspends indexing until restart.
-const categorizeAnalyzeImageError = (e: unknown): MLWorkerAnalyzeImageError => {
- if (e instanceof CategorizedMLWorkerError) {
- return { kind: e.kind, message: e.message };
- }
-
- const message = e instanceof Error ? e.message : String(e);
- const kind: MLWorkerAnalyzeImageErrorKind = message.startsWith(
- "CorruptModel: ",
- )
- ? "init"
- : message.startsWith("Decode: ") || message.startsWith("Image: ")
- ? "image"
- : message.startsWith("Ort: ") || message.startsWith("Runtime: ")
- ? "ort"
- : "misc";
- return { kind, message };
-};
-
-const analyzeImageOnce = async (
- native: MLNative,
- req: MLWorkerAnalyzeImageRequest,
-) => {
const modelPaths = await indexingModelPaths(
req.runFaces,
req.runClip,
req.runPets,
);
ensureMLRuntime(native, modelPaths);
- return await native.analyzeImage({
+ return native.analyzeImage({
fileId: req.fileID,
imageBytes: uint8ArrayToBuffer(req.bytes),
runFaces: req.runFaces,
diff --git a/rust/bindings/napi/photos/src/lib.rs b/rust/bindings/napi/photos/src/lib.rs
index 0d9926da1b..87bdec5d29 100644
--- a/rust/bindings/napi/photos/src/lib.rs
+++ b/rust/bindings/napi/photos/src/lib.rs
@@ -190,7 +190,8 @@ pub struct AnalyzeImageTask {
}
impl Task for AnalyzeImageTask {
- type Output = shared_indexing::AnalyzeImageResult;
+ // Retain MlError until resolve, where the JS thread can assign Error.name.
+ type Output = std::result::Result<shared_indexing::AnalyzeImageResult, MlError>;
type JsValue = AnalyzeImageResult;
fn compute(&mut self) -> Result<Self::Output> {
@@ -198,11 +199,14 @@ impl Task for AnalyzeImageTask {
.req
.take()
.ok_or_else(|| Error::from_reason("Runtime: analyze task computed twice"))?;
- shared_indexing::analyze_image(req).map_err(ml_error_to_napi)
+ Ok(shared_indexing::analyze_image(req))
}
- fn resolve(&mut self, _env: Env, output: Self::Output) -> Result<Self::JsValue> {
- Ok(to_napi_analyze_image_result(output))
+ fn resolve(&mut self, env: Env, output: Self::Output) -> Result<Self::JsValue> {
+ match output {
+ Ok(output) => Ok(to_napi_analyze_image_result(output)),
+ Err(error) => Err(ml_error_to_napi(&env, error)?),
+ }
}
}
@@ -246,7 +250,7 @@ impl Task for RunClipTextTask {
.req
.take()
.ok_or_else(|| Error::from_reason("Runtime: clip text task computed twice"))?;
- shared_indexing::run_clip_text(req).map_err(ml_error_to_napi)
+ shared_indexing::run_clip_text(req).map_err(|error| Error::from_reason(error.to_string()))
}
fn resolve(&mut self, _env: Env, output: Self::Output) -> Result<Self::JsValue> {
@@ -269,32 +273,20 @@ pub fn run_clip_text(req: RunClipTextRequest) -> AsyncTask<RunClipTextTask> {
#[napi]
pub fn tokenize_clip_text(text: String, vocab_path: String) -> Result<Vec<i32>> {
- shared_indexing::tokenize_clip_text(&text, &vocab_path).map_err(ml_error_to_napi)
-}
-
-// Error kind prefixes survive Comlink's MessagePort serialization.
-fn ml_error_to_napi(error: MlError) -> Error {
- let kind = match &error {
- MlError::InvalidRequest(_) => "InvalidRequest",
- MlError::Decode(_) => "Decode",
- MlError::Image(_) => "Image",
- MlError::Preprocess(_) => "Preprocess",
- MlError::Ort(_) => "Ort",
- MlError::CorruptModel(_) => "CorruptModel",
- MlError::Postprocess(_) => "Postprocess",
- MlError::Runtime(_) => "Runtime",
- };
- let detail = match error {
- MlError::InvalidRequest(message)
- | MlError::Decode(message)
- | MlError::Image(message)
- | MlError::Preprocess(message)
- | MlError::Ort(message)
- | MlError::CorruptModel(message)
- | MlError::Postprocess(message)
- | MlError::Runtime(message) => message,
+ shared_indexing::tokenize_clip_text(&text, &vocab_path)
+ .map_err(|error| Error::from_reason(error.to_string()))
+}
+
+fn ml_error_to_napi(env: &Env, error: MlError) -> Result<Error> {
+ let name = match &error {
+ MlError::CorruptModel(_) => "ml_init",
+ MlError::Decode(_) => "ml_decode",
+ MlError::Image(_) => "ml_image",
+ _ => return Ok(Error::from_reason(error.to_string())),
};
- Error::from_reason(format!("{kind}: {detail}"))
+ let mut value = env.create_error(Error::from_reason(error.to_string()))?;
+ value.set_named_property("name", name)?;
+ Ok(Error::from(value.into_unknown(env)?))
}
fn asset_error_to_napi(error: ente_assets::download::Error) -> Error {
diff --git a/web/packages/base/types/ipc.ts b/web/packages/base/types/ipc.ts
index 5ed8f3702f..633a58aaad 100644
--- a/web/packages/base/types/ipc.ts
+++ b/web/packages/base/types/ipc.ts
@@ -131,7 +131,7 @@ export interface ElectronMLWorker {
fsStatMtime: (path: string) => Promise<number>;
analyzeImage: (
request: MLWorkerAnalyzeImageRequest,
- ) => Promise<MLWorkerAnalyzeImageResponse>;
+ ) => Promise<MLWorkerAnalyzeImageResult>;
releaseMLRuntime: () => Promise<void>;
computeCLIPTextEmbeddingIfAvailable: (
text: string,
@@ -147,17 +147,6 @@ export interface MLWorkerAnalyzeImageRequest {
generateFaceCrops: boolean;
}
-export type MLWorkerAnalyzeImageErrorKind = "init" | "ort" | "image" | "misc";
-
-export interface MLWorkerAnalyzeImageError {
- kind: MLWorkerAnalyzeImageErrorKind;
- message: string;
-}
-
-export type MLWorkerAnalyzeImageResponse =
- | { ok: true; result: MLWorkerAnalyzeImageResult }
- | { ok: false; error: MLWorkerAnalyzeImageError };
-
export interface MLWorkerFaceResult {
faceId: string;
detection: { score: number; boxXyxy: number[]; keypoints: number[][] };
diff --git a/web/packages/new/photos/services/ml/worker.ts b/web/packages/new/photos/services/ml/worker.ts
index cc47ddcea8..1614668797 100644
--- a/web/packages/new/photos/services/ml/worker.ts
+++ b/web/packages/new/photos/services/ml/worker.ts
@@ -7,7 +7,6 @@ import log from "ente-base/log";
import { logUnhandledErrorsAndRejectionsInWorker } from "ente-base/log-web";
import type {
ElectronMLWorker,
- MLWorkerAnalyzeImageErrorKind,
MLWorkerAnalyzeImageRequest,
MLWorkerAnalyzeImageResult,
} from "ente-base/types/ipc";
@@ -194,7 +193,7 @@ export class MLWorker {
this.resolvePendingIndexRequests();
}
- private suspendForMLInitError(error: MLAnalyzeError) {
+ private suspendForMLInitError(error: Error) {
log.error(
"Suspending ML indexing because the native runtime failed to initialize",
error,
@@ -519,53 +518,37 @@ const analyzeImageWithConversionFallback = async (
conversionError instanceof Error
? conversionError.message
: String(conversionError);
- throw new MLAnalyzeError(
- "image",
+ const error = new Error(
`JPEG conversion fallback failed: ${message}`,
);
+ error.name = "ml_image";
+ throw error;
}
return await enqueueAnalyzeImage(electron, { ...request, bytes });
}
};
-const isMLImageError = (e: unknown): e is MLAnalyzeError =>
- e instanceof MLAnalyzeError && e.kind == "image";
+const isMLImageError = (e: unknown): e is Error =>
+ e instanceof Error && (e.name == "ml_decode" || e.name == "ml_image");
const isMLDecodeError = (e: unknown) =>
- isMLImageError(e) && e.message.startsWith("Decode: ");
+ e instanceof Error && e.name == "ml_decode";
-const isMLInitError = (e: unknown): e is MLAnalyzeError =>
- e instanceof MLAnalyzeError && e.kind == "init";
-
-class MLAnalyzeError extends Error {
- constructor(
- public readonly kind: MLWorkerAnalyzeImageErrorKind,
- message: string,
- ) {
- super(message);
- this.name = "MLAnalyzeError";
- }
-}
+const isMLInitError = (e: unknown): e is Error =>
+ e instanceof Error && e.name == "ml_init";
// Native model sessions serialize anyway; keep only one inference outstanding.
let _analyzeImageQueue: Promise<unknown> = Promise.resolve();
-let _blockingMLInitError: MLAnalyzeError | undefined;
+let _blockingMLInitError: Error | undefined;
const enqueueAnalyzeImage = (
electron: ElectronMLWorker,
request: MLWorkerAnalyzeImageRequest,
): Promise<MLWorkerAnalyzeImageResult> => {
- const result = _analyzeImageQueue.then(async () => {
- const response = await electron.analyzeImage(request);
- if (!response.ok) {
- throw new MLAnalyzeError(
- response.error.kind,
- response.error.message,
- );
- }
- return response.result;
- });
+ const result = _analyzeImageQueue.then(() =>
+ electron.analyzeImage(request),
+ );
_analyzeImageQueue = result.catch(() => undefined);
return result;
};