World's first open-source enterprise world model.
Rust agent-memorybitemporalgraphragknowledge-baseknowledge-graphllmontologypgvectorpostgresqlragrustself-hostedsemantic-searchtemporal-knowledge-graphworld-model
Raising a model's concurrency limit admits the tasks already waiting on it (#1305)
Quintin Botesb34e237980c9
diff --git a/crates/utopia-server/src/api/admin_routes.rs b/crates/utopia-server/src/api/admin_routes.rsindex ebbf2843..68fcdbe0 100644--- a/crates/utopia-server/src/api/admin_routes.rs+++ b/crates/utopia-server/src/api/admin_routes.rs@@ -79,10 +79,10 @@ pub async fn put_deployment(.worker_concurrency.store(n as usize, std::sync::atomic::Ordering::Relaxed);}- // 按模型的限额即时生效:闸门每次调用前读库,发现限额变了就换一把新信号量if let Some(l) = &req.default_ontology_lang {utopia_store::access::set_default_ontology_lang(&state.pool, l).await?;}+ let limits_changed = req.default_model_concurrency.is_some() || req.model_limit.is_some();if let Some(n) = req.default_model_concurrency {utopia_store::model_limits::set_default(&state.pool, n).await?;}@@ -90,6 +90,14 @@ pub async fn put_deployment(utopia_store::model_limits::set(&state.pool, &m.base_url, &m.model, m.max_concurrent).await?;}+ // 按模型的限额在这里就地生效:闸门调大放行、调小收紧,不等下一个任务来取许可时才读+ // ——那时可能所有 worker 都等在闸门上,唯一在飞的还是一次复读(#1302)。限额已经存下了,+ // 调闸门失败只记一笔:下一次取许可照样读到新限额+ if limits_changed {+ if let Err(e) = state.model_gates.refresh(&state.pool).await {+ tracing::warn!(error = %e, "限额已保存,闸门要等下一次取许可才按新限额调");+ }+ }Ok(Json(json!({ "ok": true })))}diff --git a/crates/utopia-server/src/llm_util.rs b/crates/utopia-server/src/llm_util.rsindex 448c7125..15428c63 100644--- a/crates/utopia-server/src/llm_util.rs+++ b/crates/utopia-server/src/llm_util.rs@@ -81,24 +81,84 @@ pub fn embed_client(s: &LlmSettings) -> Option<LlmClient> {))}-/// 按模型的信号量注册表。限额变了就换一把新的——旧的在飞许可自然跑完,-/// 换的瞬间可能短暂超出新限额,可接受;换来的是"改完即时生效"而不必做-/// 缓存失效,也不必和 tokio Semaphore 不能缩容的限制搏斗。+/// 按模型的信号量注册表:每个模型一把,限额变了就地调整(#1302)。+///+/// 原来限额一变就换一把新的。等在旧的那把上的任务不会挪过去,每个都要拿旧限额再做+/// 完一次调用才换到新的上——抽取一次调用几分钟,八个 worker 排在一张许可后面,限额+/// 从 1 调到 4 之后好几个小时还是一次只跑一个,看起来就是「改了不生效」。+///+/// 调高用 `add_permits`,已经在等的立刻放进来。调低时空着的许可当场收走+/// (`forget_permits`);正被在飞调用占着的记成欠账,由等着的任务拿到时收掉。没用+/// 「另起一个任务把差额 acquire 下来再 forget」:信号量先来先得,那个任务排在已经在等的+/// 任务后面,还回来的许可先发给它们,它们全在调低之后的限额之上进去了,它才拿到。+///+/// 限额在两处读:取许可时,和管理员改了限额时([`ModelGates::refresh`])。只靠前一处+/// 的话,所有 worker 都等在闸门上、唯一在飞的又是一次复读的调用时,没人来读新限额#[derive(Default)]pub struct ModelGates {- inner: std::sync::Mutex<HashMap<String, (usize, Arc<Semaphore>)>>,+ inner: std::sync::Mutex<HashMap<(String, String), Gate>>,+}++struct Gate {+ sem: Arc<Semaphore>,+ limit: usize,+ /// 调低之后还该收走、当时正被在飞调用占着的许可数+ owed: usize,}impl ModelGates {- fn gate(&self, key: &str, limit: usize) -> Arc<Semaphore> {- let mut m = self.inner.lock().unwrap();- match m.get(key) {- Some((n, sem)) if *n == limit => sem.clone(),- _ => {- let sem = Arc::new(Semaphore::new(limit));- m.insert(key.to_string(), (limit, sem.clone()));- sem+ async fn admit(+ &self,+ base_url: &str,+ model: &str,+ limit: usize,+ ) -> Option<OwnedSemaphorePermit> {+ let key = (base_url.to_string(), model.to_string());+ let sem = self.resize(&key, limit);+ loop {+ let permit = sem.clone().acquire_owned().await.ok()?;+ if !self.repay(&key) {+ return Some(permit);}+ permit.forget();+ }+ }++ /// 按库里的限额把认识的每个模型调一遍:管理员改了限额就调,等着的任务当场放行或收紧+ pub async fn refresh(&self, pool: &sqlx::PgPool) -> utopia_core::AppResult<()> {+ let keys: Vec<(String, String)> = self.inner.lock().unwrap().keys().cloned().collect();+ for key in keys {+ let limit = utopia_store::model_limits::limit_for(pool, &key.0, &key.1).await?;+ self.resize(&key, limit);+ }+ Ok(())+ }++ fn resize(&self, key: &(String, String), limit: usize) -> Arc<Semaphore> {+ let mut gates = self.inner.lock().unwrap();+ let gate = gates.entry(key.clone()).or_insert_with(|| Gate {+ sem: Arc::new(Semaphore::new(limit)),+ limit,+ owed: 0,+ });+ if limit > gate.limit {+ gate.sem.add_permits(limit - gate.limit);+ } else if limit < gate.limit {+ let lower = gate.limit - limit;+ gate.owed += lower - gate.sem.forget_permits(lower);+ }+ gate.limit = limit;+ gate.sem.clone()+ }++ /// 还欠着许可就该收掉刚拿到的这张+ fn repay(&self, key: &(String, String)) -> bool {+ match self.inner.lock().unwrap().get_mut(key) {+ Some(gate) if gate.owed > 0 => {+ gate.owed -= 1;+ true+ }+ _ => false,}}}@@ -119,13 +179,7 @@ pub async fn acquire(let limit = utopia_store::model_limits::limit_for(&state.pool, base_url, model).await.ok()?;- let key = format!("{base_url}|{model}");- state- .model_gates- .gate(&key, limit)- .acquire_owned()- .await- .ok()+ state.model_gates.admit(base_url, model, limit).await}/// `acquire` 的便捷形式:直接从工作区设置取 chat 模型的身份。@@ -139,3 +193,7 @@ pub async fn acquire_embed(state: &AppState, s: &LlmSettings) -> Option<OwnedSemlet (base, model) = (s.embed_base_url.as_deref()?, s.embed_model.as_deref()?);acquire(state, base, model).await}++#[cfg(test)]+#[path = "llm_util_tests.rs"]+mod tests;diff --git a/crates/utopia-server/src/llm_util_tests.rs b/crates/utopia-server/src/llm_util_tests.rsnew file mode 100644index 00000000..e27179e5--- /dev/null+++ b/crates/utopia-server/src/llm_util_tests.rs@@ -0,0 +1,100 @@+//! 限额就地调整(#1302):管理员调高,已经在等的任务当场放行;调低要等在飞的调用还回来才生效。++use super::ModelGates;+use axum::body::Body;+use axum::http::{Request, StatusCode};+use serde_json::json;+use std::sync::Arc;+use std::time::Duration;+use tokio::time::timeout;+use tower::ServiceExt;+use uuid::Uuid;++const SOON: Duration = Duration::from_millis(100);++/// 调高不靠下一个来取许可的任务:所有 worker 都等着时,管理员存下新限额的那一下就得把它们放进来+#[tokio::test]+async fn an_admin_raising_a_limit_admits_a_task_already_waiting() -> anyhow::Result<()> {+ let Some(url) = utopia_store::test_db::url() else {+ return Ok(());+ };+ let pool = sqlx::PgPool::connect(&url).await?;+ utopia_store::db::migrate(&pool).await?;+ let (org, admin) = (Uuid::now_v7(), Uuid::now_v7());+ let base = format!("http://gate-{}", Uuid::now_v7());+ let dir = tempfile::tempdir()?;+ let cfg = utopia_core::config::AppConfig {+ data_dir: dir.path().to_string_lossy().into_owned(),+ ..Default::default()+ };+ let state = crate::state::AppState::new(+ pool.clone(),+ &cfg,+ Arc::new(utopia_search::SearchIndex::open(+ &dir.path().join("search"),+ )?),+ "test-only".into(),+ );+ sqlx::raw_sql(&format!(+ "INSERT INTO organizations(id,name) VALUES('{org}','model-gate-test');+ INSERT INTO users(id,org_id,email,password_hash,display_name,is_admin)+ VALUES('{admin}','{org}','{admin}@example.test','unused','Admin',TRUE);"+ ))+ .execute(&pool)+ .await?;+ utopia_store::model_limits::set(&pool, &base, "m", Some(1)).await?;+ let result = async {+ let _in_flight = super::acquire(&state, &base, "m").await;+ let (s, b) = (state.clone(), base.clone());+ let mut waiting = tokio::spawn(async move { super::acquire(&s, &b, "m").await.is_some() });+ let parked = timeout(SOON, &mut waiting).await.is_err();++ // 这个接口每次都写 open_registration,原样送回去,部署设置那一行不变+ let open = utopia_store::access::open_registration(&pool).await?;+ let body = json!({+ "open_registration": open,+ "model_limit": {"base_url": base, "model": "m", "max_concurrent": 2},+ });+ let request = Request::builder()+ .method("PUT")+ .uri("/api/v1/admin/deployment")+ .header(+ "authorization",+ format!("Bearer {}", crate::auth::issue_token(&state, admin)?),+ )+ .header("content-type", "application/json")+ .body(Body::from(body.to_string()))?;+ let status = crate::api::router(state.clone(), &Default::default())+ .oneshot(request)+ .await?+ .status();+ let admitted = timeout(SOON, &mut waiting).await.is_ok();+ anyhow::ensure!(+ status == StatusCode::OK && parked && admitted,+ "status: {status}, parked: {parked}, admitted: {admitted}"+ );+ anyhow::Ok(())+ }+ .await;+ utopia_store::model_limits::set(&pool, &base, "m", None).await?;+ sqlx::query("DELETE FROM organizations WHERE id=$1")+ .bind(org)+ .execute(&pool)+ .await?;+ result+}++#[tokio::test]+async fn lowering_the_limit_takes_effect_as_in_flight_calls_return() {+ let gates = Arc::new(ModelGates::default());+ let first = gates.admit("u", "m", 2).await.unwrap();+ let second = gates.admit("u", "m", 2).await.unwrap();+ let g = gates.clone();+ let mut waiting = tokio::spawn(async move { g.admit("u", "m", 1).await });++ // 两个在飞降到一个:还回来的第一张被收走,第二个调用还没完就仍满额+ drop(first);+ assert!(timeout(SOON, &mut waiting).await.is_err());+ drop(second);+ assert!(timeout(SOON, &mut waiting).await.is_ok());+}diff --git a/crates/utopia-store/src/model_limits.rs b/crates/utopia-store/src/model_limits.rsindex 2f30b1df..01d3b956 100644--- a/crates/utopia-store/src/model_limits.rs+++ b/crates/utopia-store/src/model_limits.rs@@ -13,7 +13,8 @@ use utopia_core::AppResult;/// 这个模型允许多少并发。没有专属配置就走部署缺省。////// 每次 LLM 调用前查一次——相对于一次动辄二十几秒的调用,这个查询的成本可以-/// 忽略,换来的是"管理员改完即时生效",不必做缓存失效。+/// 忽略。闸门手里也存着一份:管理员改限额时 `ModelGates::refresh` 再查一遍、当场+/// 调闸门(#1302),等在闸门上的任务不必等下一次调用来读。pub async fn limit_for(pool: &PgPool, base_url: &str, model: &str) -> AppResult<usize> {let row: Option<(i32,)> = sqlx::query_as("SELECT max_concurrent FROM model_concurrency WHERE base_url = $1 AND model = $2",