Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

Warm up the CLIP text encoder at ML worker startup

laurenspriem647b712de4b5
diff --git a/desktop/src/main/services/ml-worker.ts b/desktop/src/main/services/ml-worker.ts
index e718c91361..e85cbe6e7e 100644
--- a/desktop/src/main/services/ml-worker.ts
+++ b/desktop/src/main/services/ml-worker.ts
@@ -39,6 +39,7 @@ process.parentPort.once("message", (e) => {
// Initialize ourselves with the data we got from our parent.
const { mlNativePaths } = parseInitData(e.data);
loadMLNative(mlNativePaths);
+ if (_native) void warmUpClipTextEncoder(_native);
// Expose an instance of `ElectronMLWorker` on the port we got from our
// parent.
expose(
@@ -578,6 +579,32 @@ const evictModelIfReportedCorrupt = async (e: unknown) => {
const uint8ArrayToBuffer = (bytes: Uint8Array) =>
Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+/**
+ * Run a throwaway CLIP text query so that the Rust side builds and caches the
+ * text encoder session before the user's first search. Mirrors the mobile
+ * app's warm up (see ml_computer.dart).
+ *
+ * Best effort - on failure, queries build the session on demand as before.
+ */
+const warmUpClipTextEncoder = async (native: MLNative) => {
+ try {
+ const [model, vocab] = await Promise.all([
+ modelPath(modelSpecs.clipText),
+ modelPath(modelSpecs.clipTextVocab),
+ ]);
+ await native.runClipText({
+ text: "warm up text encoder",
+ modelPath: model,
+ vocabPath: vocab,
+ });
+ } catch (e) {
+ if (!(await evictModelIfReportedCorrupt(e)))
+ log.warn("Failed to warm up the CLIP text encoder", e);
+ } finally {
+ logMLRuntimeEvents(native);
+ }
+};
+
/**
* Compute CLIP embeddings for a text snippet, if the model is available.
*