Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

[rust] Bind ONNX sessions to model identity

laurenspriem9ae8d361fcb1
diff --git a/rust/crates/ml/src/onnx.rs b/rust/crates/ml/src/onnx.rs
index 1204669e53..256424e54c 100644
--- a/rust/crates/ml/src/onnx.rs
+++ b/rust/crates/ml/src/onnx.rs
@@ -54,6 +54,8 @@ impl ProviderUsage {
#[derive(Debug)]
pub(crate) struct OnnxSession {
+ model_path: String,
+ model_namespace: String,
mode: ExecutionMode,
validation: AccelerationValidation,
provider_plan: Option<ProviderPlan>,
@@ -62,8 +64,10 @@ pub(crate) struct OnnxSession {
}
impl OnnxSession {
- pub(crate) fn new(mode: ExecutionMode) -> Self {
+ pub(crate) fn new(model_path: &str, model_namespace: &str, mode: ExecutionMode) -> Self {
Self {
+ model_path: model_path.to_string(),
+ model_namespace: model_namespace.to_string(),
mode,
validation: AccelerationValidation::GoldenRequired,
provider_plan: None,
@@ -79,9 +83,14 @@ impl OnnxSession {
self
}
- pub(crate) fn clear(&mut self) {
+ pub(crate) fn model_path(&self) -> &str {
+ &self.model_path
+ }
+
+ pub(crate) fn unload(&mut self) {
self.provider_plan = None;
self.session = None;
+ self.disarm_first_run_canary();
}
#[cfg(test)]
@@ -89,14 +98,26 @@ impl OnnxSession {
self.session.is_some()
}
+ #[cfg(test)]
+ pub(crate) fn has_load_state(&self) -> bool {
+ self.provider_plan.is_some() || self.session.is_some() || self.first_run_canary.is_some()
+ }
+
+ #[cfg(test)]
+ pub(crate) fn initialize_load_state(&mut self) {
+ self.provider_plan = Some(ProviderPlan::new(
+ self.mode,
+ &self.model_path,
+ self.validation,
+ ));
+ }
+
pub(crate) fn run<T>(
&mut self,
- model_path: &str,
- model_namespace: &str,
mut operation: impl FnMut(&mut Session) -> SessionRunResult<T>,
) -> MlResult<(T, ProviderUsage)> {
loop {
- self.ensure_loaded(model_path, model_namespace)?;
+ self.ensure_loaded()?;
let execution_provider = self
.provider_plan
@@ -127,11 +148,13 @@ impl OnnxSession {
}
}
- fn ensure_loaded(&mut self, model_path: &str, model_namespace: &str) -> MlResult<()> {
+ fn ensure_loaded(&mut self) -> MlResult<()> {
if self.session.is_some() {
return Ok(());
}
+ let model_path = self.model_path.as_str();
+ let model_namespace = self.model_namespace.as_str();
let provider_plan = self
.provider_plan
.get_or_insert_with(|| ProviderPlan::new(self.mode, model_path, self.validation));
@@ -628,7 +651,7 @@ mod tests {
#[test]
fn successful_first_run_disarms_the_canary() {
let temp = tempfile::tempdir().unwrap();
- let mut session = OnnxSession::new(ExecutionMode::CpuOnly);
+ let mut session = OnnxSession::new("model.onnx", "scanner", ExecutionMode::CpuOnly);
session.first_run_canary = Some(first_run_canary(&temp));
session.disarm_first_run_canary();
@@ -639,7 +662,7 @@ mod tests {
#[test]
fn retryable_first_run_failure_leaves_the_canary_armed() {
let temp = tempfile::tempdir().unwrap();
- let mut session = OnnxSession::new(ExecutionMode::CpuOnly);
+ let mut session = OnnxSession::new("model.onnx", "scanner", ExecutionMode::CpuOnly);
session.first_run_canary = Some(first_run_canary(&temp));
session.leave_first_run_canary_armed();
@@ -647,6 +670,23 @@ mod tests {
assert!(has_canary(&temp));
}
+ #[test]
+ fn unload_preserves_model_identity() {
+ let mut session = OnnxSession::new(
+ "/models/document.onnx",
+ "document-segmentation",
+ ExecutionMode::CpuOnly,
+ );
+ session.initialize_load_state();
+
+ session.unload();
+
+ assert_eq!(session.model_path, "/models/document.onnx");
+ assert_eq!(session.model_namespace, "document-segmentation");
+ assert!(!session.is_loaded());
+ assert!(!session.has_load_state());
+ }
+
#[test]
fn provider_failure_detection_is_limited_to_ort_execution_errors() {
assert!(is_execution_provider_run_failure(&super::MlError::Ort(
diff --git a/rust/crates/ml/src/runtime.rs b/rust/crates/ml/src/runtime.rs
index 913b9d7a76..f239fdda6c 100644
--- a/rust/crates/ml/src/runtime.rs
+++ b/rust/crates/ml/src/runtime.rs
@@ -11,23 +11,18 @@ use crate::{
onnx,
};
-#[derive(Debug)]
-struct ModelSlotState {
- path: String,
- onnx_session: onnx::OnnxSession,
-}
-
#[cfg(test)]
#[derive(Debug, PartialEq, Eq)]
-struct ModelSlotSnapshot {
+struct ModelRuntimeSnapshot {
path: String,
session_loaded: bool,
+ has_load_state: bool,
}
#[derive(Debug)]
-struct ModelSlot {
+struct ModelRuntime {
model: Model,
- state: Mutex<ModelSlotState>,
+ session: Mutex<Option<onnx::OnnxSession>>,
}
pub(crate) struct MlRuntimeView<'a> {
@@ -36,30 +31,35 @@ pub(crate) struct MlRuntimeView<'a> {
provider_usage: Cell<onnx::ProviderUsage>,
}
-impl ModelSlot {
+impl ModelRuntime {
fn new(model: Model) -> Self {
Self {
model,
- state: Mutex::new(ModelSlotState {
- path: String::new(),
- onnx_session: onnx::OnnxSession::new(default_execution_mode(model)),
- }),
+ session: Mutex::new(None),
}
}
- fn lock_state(&self) -> MutexGuard<'_, ModelSlotState> {
- match self.state.lock() {
+ fn lock_session(&self) -> MutexGuard<'_, Option<onnx::OnnxSession>> {
+ match self.session.lock() {
Ok(guard) => guard,
Err(poisoned) => poisoned.into_inner(),
}
}
#[cfg(test)]
- fn snapshot(&self) -> ModelSlotSnapshot {
- let state = self.lock_state();
- ModelSlotSnapshot {
- path: state.path.clone(),
- session_loaded: state.onnx_session.is_loaded(),
+ fn snapshot(&self) -> ModelRuntimeSnapshot {
+ let session = self.lock_session();
+ match session.as_ref() {
+ Some(session) => ModelRuntimeSnapshot {
+ path: session.model_path().to_string(),
+ session_loaded: session.is_loaded(),
+ has_load_state: session.has_load_state(),
+ },
+ None => ModelRuntimeSnapshot {
+ path: String::new(),
+ session_loaded: false,
+ has_load_state: false,
+ },
}
}
@@ -67,23 +67,25 @@ impl ModelSlot {
if path.trim().is_empty() {
return;
}
- let mut state = self.lock_state();
- Self::set_config_locked(&mut state, path);
+ let mut session = self.lock_session();
+ self.configure_locked(&mut session, path);
}
fn sync_indexing_residency(&self, path: &str) {
- let mut state = self.lock_state();
+ let mut session = self.lock_session();
if path.trim().is_empty() {
- Self::reset_slot_locked(&mut state);
+ *session = None;
return;
}
- Self::set_config_locked(&mut state, path);
+ self.configure_locked(&mut session, path);
}
fn release_residency(&self) {
- let mut state = self.lock_state();
- state.onnx_session.clear();
+ let mut session = self.lock_session();
+ if let Some(session) = session.as_mut() {
+ session.unload();
+ }
}
fn run<T>(
@@ -95,29 +97,32 @@ impl ModelSlot {
return Err(MlError::InvalidRequest(self.model.missing_path_error()));
}
- let mut state = self.lock_state();
- Self::set_config_locked(&mut state, path);
- let ModelSlotState { path, onnx_session } = &mut *state;
- onnx_session.run(path, self.model.namespace(), operation)
+ let mut session = self.lock_session();
+ self.configure_locked(&mut session, path);
+ session
+ .as_mut()
+ .expect("non-empty model path must configure a session")
+ .run(operation)
}
- fn set_config_locked(state: &mut ModelSlotState, path: &str) {
- if state.path == path {
+ fn configure_locked(&self, session: &mut Option<onnx::OnnxSession>, path: &str) {
+ if session
+ .as_ref()
+ .is_some_and(|session| session.model_path() == path)
+ {
return;
}
- state.path = path.to_string();
- state.onnx_session.clear();
- }
-
- fn reset_slot_locked(state: &mut ModelSlotState) {
- state.path.clear();
- state.onnx_session.clear();
+ *session = Some(onnx::OnnxSession::new(
+ path,
+ self.model.namespace(),
+ default_execution_mode(self.model),
+ ));
}
}
#[derive(Debug)]
struct MlRuntime {
- slots: [ModelSlot; Model::COUNT],
+ models: [ModelRuntime; Model::COUNT],
}
static GLOBAL_RUNTIME: Lazy<MlRuntime> = Lazy::new(MlRuntime::new);
@@ -125,31 +130,31 @@ static GLOBAL_RUNTIME: Lazy<MlRuntime> = Lazy::new(MlRuntime::new);
impl MlRuntime {
fn new() -> Self {
Self {
- slots: Model::ALL.map(ModelSlot::new),
+ models: Model::ALL.map(ModelRuntime::new),
}
}
- fn slot(&self, model: Model) -> &ModelSlot {
- &self.slots[model.index()]
+ fn model(&self, model: Model) -> &ModelRuntime {
+ &self.models[model.index()]
}
fn configure_requested_models(&self, model_paths: &ModelPaths) {
for model in Model::ALL {
- self.slot(model)
+ self.model(model)
.configure_if_requested(model_paths.get(model));
}
}
fn prepare_indexing_models(&self, model_paths: &ModelPaths) {
for model in Model::INDEXING {
- self.slot(model)
+ self.model(model)
.sync_indexing_residency(model_paths.get(model));
}
}
fn release_indexing_models(&self) {
for model in Model::INDEXING {
- self.slot(model).release_residency();
+ self.model(model).release_residency();
}
}
}
@@ -166,7 +171,7 @@ impl<'a> MlRuntimeView<'a> {
) -> MlResult<T> {
let (value, provider_usage) = self
.runtime
- .slot(model)
+ .model(model)
.run(self.model_paths.get(model), operation)?;
self.provider_usage
.set(self.provider_usage.get().merge(provider_usage));
@@ -218,7 +223,7 @@ mod tests {
use super::*;
#[test]
- fn configure_requested_models_preserves_unrequested_slots() {
+ fn configure_requested_models_preserves_unrequested_models() {
let runtime = MlRuntime::new();
runtime.configure_requested_models(&ModelPaths {
@@ -232,11 +237,11 @@ mod tests {
});
assert_eq!(
- runtime.slot(Model::ClipText).snapshot().path,
+ runtime.model(Model::ClipText).snapshot().path,
"clip_text.onnx"
);
assert_eq!(
- runtime.slot(Model::FaceDetection).snapshot().path,
+ runtime.model(Model::FaceDetection).snapshot().path,
"face.onnx"
);
}
@@ -263,17 +268,19 @@ mod tests {
runtime.release_indexing_models();
assert_eq!(
- runtime.slot(Model::ClipText).snapshot(),
- ModelSlotSnapshot {
+ runtime.model(Model::ClipText).snapshot(),
+ ModelRuntimeSnapshot {
path: "clip_text.onnx".to_string(),
session_loaded: false,
+ has_load_state: false,
}
);
assert_eq!(
- runtime.slot(Model::FaceDetection).snapshot(),
- ModelSlotSnapshot {
+ runtime.model(Model::FaceDetection).snapshot(),
+ ModelRuntimeSnapshot {
path: "face.onnx".to_string(),
session_loaded: false,
+ has_load_state: false,
}
);
}
@@ -290,21 +297,21 @@ mod tests {
});
assert_eq!(
- runtime.slot(Model::FaceDetection).snapshot(),
+ runtime.model(Model::FaceDetection).snapshot(),
unloaded_snapshot("face.onnx")
);
assert_eq!(
- runtime.slot(Model::FaceEmbedding).snapshot(),
+ runtime.model(Model::FaceEmbedding).snapshot(),
unloaded_snapshot("embed.onnx")
);
assert_eq!(
- runtime.slot(Model::ClipImage).snapshot(),
+ runtime.model(Model::ClipImage).snapshot(),
unloaded_snapshot("clip.onnx")
);
}
#[test]
- fn disabling_an_indexing_model_resets_its_slot() {
+ fn disabling_an_indexing_model_clears_its_runtime() {
let runtime = MlRuntime::new();
runtime.prepare_indexing_models(&ModelPaths {
clip_image: "clip.onnx".to_string(),
@@ -314,11 +321,35 @@ mod tests {
runtime.prepare_indexing_models(&ModelPaths::default());
assert_eq!(
- runtime.slot(Model::ClipImage).snapshot(),
+ runtime.model(Model::ClipImage).snapshot(),
unloaded_snapshot("")
);
}
+ #[test]
+ fn changing_model_path_discards_stale_session_state() {
+ let runtime = MlRuntime::new();
+ let model_runtime = runtime.model(Model::ClipText);
+ let first_path = "first.onnx";
+ let second_path = "second.onnx";
+ model_runtime.configure_if_requested(first_path);
+ model_runtime
+ .lock_session()
+ .as_mut()
+ .unwrap()
+ .initialize_load_state();
+
+ assert!(model_runtime.snapshot().has_load_state);
+
+ model_runtime.configure_if_requested(first_path);
+
+ assert!(model_runtime.snapshot().has_load_state);
+
+ model_runtime.configure_if_requested(second_path);
+
+ assert_eq!(model_runtime.snapshot(), unloaded_snapshot(second_path));
+ }
+
#[test]
fn model_execution_modes_match_platform_policy() {
let accelerated = [Model::FaceDetection, Model::FaceEmbedding, Model::ClipImage];
@@ -333,10 +364,11 @@ mod tests {
}
}
- fn unloaded_snapshot(path: &str) -> ModelSlotSnapshot {
- ModelSlotSnapshot {
+ fn unloaded_snapshot(path: &str) -> ModelRuntimeSnapshot {
+ ModelRuntimeSnapshot {
path: path.to_string(),
session_loaded: false,
+ has_load_state: false,
}
}
}
diff --git a/rust/crates/ml/src/scan/segmentation.rs b/rust/crates/ml/src/scan/segmentation.rs
index 7e628a1786..ba38a7792e 100644
--- a/rust/crates/ml/src/scan/segmentation.rs
+++ b/rust/crates/ml/src/scan/segmentation.rs
@@ -13,16 +13,15 @@ const MODEL_NAMESPACE: &str = "document-segmentation";
pub(crate) struct Segmenter {
session: Mutex<OnnxSession>,
- model_path: String,
}
impl Segmenter {
pub(crate) fn new(model_path: &str) -> Result<Self, ScanError> {
let segmenter = Self {
session: Mutex::new(
- OnnxSession::new(ExecutionMode::PlatformDefault).with_unvalidated_acceleration(),
+ OnnxSession::new(model_path, MODEL_NAMESPACE, ExecutionMode::PlatformDefault)
+ .with_unvalidated_acceleration(),
),
- model_path: model_path.to_string(),
};
let samples = (MASK_SIDE * MASK_SIDE * 3) as usize;
segmenter
@@ -40,7 +39,7 @@ impl Segmenter {
Err(poisoned) => poisoned.into_inner(),
};
let (data, _usage) = guard
- .run(&self.model_path, MODEL_NAMESPACE, |session| {
+ .run(|session| {
let (shape, values) = run_f32(session, &input, [1i64, side, side, 3])?;
if values.len() != expected {
return Err(SessionRunError::from(MlError::Ort(format!(