π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
Warm up the CLIP text encoder at ML worker startup
laurenspriem647b712de4b5
diff --git a/desktop/src/main/services/ml-worker.ts b/desktop/src/main/services/ml-worker.tsindex e718c91361..e85cbe6e7e 100644--- a/desktop/src/main/services/ml-worker.ts+++ b/desktop/src/main/services/ml-worker.ts@@ -39,6 +39,7 @@ process.parentPort.once("message", (e) => {// Initialize ourselves with the data we got from our parent.const { mlNativePaths } = parseInitData(e.data);loadMLNative(mlNativePaths);+ if (_native) void warmUpClipTextEncoder(_native);// Expose an instance of `ElectronMLWorker` on the port we got from our// parent.expose(@@ -578,6 +579,32 @@ const evictModelIfReportedCorrupt = async (e: unknown) => {const uint8ArrayToBuffer = (bytes: Uint8Array) =>Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength);+/**+ * Run a throwaway CLIP text query so that the Rust side builds and caches the+ * text encoder session before the user's first search. Mirrors the mobile+ * app's warm up (see ml_computer.dart).+ *+ * Best effort - on failure, queries build the session on demand as before.+ */+const warmUpClipTextEncoder = async (native: MLNative) => {+ try {+ const [model, vocab] = await Promise.all([+ modelPath(modelSpecs.clipText),+ modelPath(modelSpecs.clipTextVocab),+ ]);+ await native.runClipText({+ text: "warm up text encoder",+ modelPath: model,+ vocabPath: vocab,+ });+ } catch (e) {+ if (!(await evictModelIfReportedCorrupt(e)))+ log.warn("Failed to warm up the CLIP text encoder", e);+ } finally {+ logMLRuntimeEvents(native);+ }+};+/*** Compute CLIP embeddings for a text snippet, if the model is available.*