Skip to content

World's first open-source enterprise world model.

Raising a model's concurrency limit admits the tasks already waiting on it (#1305)

Quintin Botesb34e237980c9
diff --git a/crates/utopia-server/src/api/admin_routes.rs b/crates/utopia-server/src/api/admin_routes.rs
index ebbf2843..68fcdbe0 100644
--- a/crates/utopia-server/src/api/admin_routes.rs
+++ b/crates/utopia-server/src/api/admin_routes.rs
@@ -79,10 +79,10 @@ pub async fn put_deployment(
.worker_concurrency
.store(n as usize, std::sync::atomic::Ordering::Relaxed);
}
- // 按模型的限额即时生效:闸门每次调用前读库,发现限额变了就换一把新信号量
if let Some(l) = &req.default_ontology_lang {
utopia_store::access::set_default_ontology_lang(&state.pool, l).await?;
}
+ let limits_changed = req.default_model_concurrency.is_some() || req.model_limit.is_some();
if let Some(n) = req.default_model_concurrency {
utopia_store::model_limits::set_default(&state.pool, n).await?;
}
@@ -90,6 +90,14 @@ pub async fn put_deployment(
utopia_store::model_limits::set(&state.pool, &m.base_url, &m.model, m.max_concurrent)
.await?;
}
+ // 按模型的限额在这里就地生效:闸门调大放行、调小收紧,不等下一个任务来取许可时才读
+ // ——那时可能所有 worker 都等在闸门上,唯一在飞的还是一次复读(#1302)。限额已经存下了,
+ // 调闸门失败只记一笔:下一次取许可照样读到新限额
+ if limits_changed {
+ if let Err(e) = state.model_gates.refresh(&state.pool).await {
+ tracing::warn!(error = %e, "限额已保存,闸门要等下一次取许可才按新限额调");
+ }
+ }
Ok(Json(json!({ "ok": true })))
}
diff --git a/crates/utopia-server/src/llm_util.rs b/crates/utopia-server/src/llm_util.rs
index 448c7125..15428c63 100644
--- a/crates/utopia-server/src/llm_util.rs
+++ b/crates/utopia-server/src/llm_util.rs
@@ -81,24 +81,84 @@ pub fn embed_client(s: &LlmSettings) -> Option<LlmClient> {
))
}
-/// 按模型的信号量注册表。限额变了就换一把新的——旧的在飞许可自然跑完,
-/// 换的瞬间可能短暂超出新限额,可接受;换来的是"改完即时生效"而不必做
-/// 缓存失效,也不必和 tokio Semaphore 不能缩容的限制搏斗。
+/// 按模型的信号量注册表:每个模型一把,限额变了就地调整(#1302)。
+///
+/// 原来限额一变就换一把新的。等在旧的那把上的任务不会挪过去,每个都要拿旧限额再做
+/// 完一次调用才换到新的上——抽取一次调用几分钟,八个 worker 排在一张许可后面,限额
+/// 从 1 调到 4 之后好几个小时还是一次只跑一个,看起来就是「改了不生效」。
+///
+/// 调高用 `add_permits`,已经在等的立刻放进来。调低时空着的许可当场收走
+/// (`forget_permits`);正被在飞调用占着的记成欠账,由等着的任务拿到时收掉。没用
+/// 「另起一个任务把差额 acquire 下来再 forget」:信号量先来先得,那个任务排在已经在等的
+/// 任务后面,还回来的许可先发给它们,它们全在调低之后的限额之上进去了,它才拿到。
+///
+/// 限额在两处读:取许可时,和管理员改了限额时([`ModelGates::refresh`])。只靠前一处
+/// 的话,所有 worker 都等在闸门上、唯一在飞的又是一次复读的调用时,没人来读新限额
#[derive(Default)]
pub struct ModelGates {
- inner: std::sync::Mutex<HashMap<String, (usize, Arc<Semaphore>)>>,
+ inner: std::sync::Mutex<HashMap<(String, String), Gate>>,
+}
+
+struct Gate {
+ sem: Arc<Semaphore>,
+ limit: usize,
+ /// 调低之后还该收走、当时正被在飞调用占着的许可数
+ owed: usize,
}
impl ModelGates {
- fn gate(&self, key: &str, limit: usize) -> Arc<Semaphore> {
- let mut m = self.inner.lock().unwrap();
- match m.get(key) {
- Some((n, sem)) if *n == limit => sem.clone(),
- _ => {
- let sem = Arc::new(Semaphore::new(limit));
- m.insert(key.to_string(), (limit, sem.clone()));
- sem
+ async fn admit(
+ &self,
+ base_url: &str,
+ model: &str,
+ limit: usize,
+ ) -> Option<OwnedSemaphorePermit> {
+ let key = (base_url.to_string(), model.to_string());
+ let sem = self.resize(&key, limit);
+ loop {
+ let permit = sem.clone().acquire_owned().await.ok()?;
+ if !self.repay(&key) {
+ return Some(permit);
}
+ permit.forget();
+ }
+ }
+
+ /// 按库里的限额把认识的每个模型调一遍:管理员改了限额就调,等着的任务当场放行或收紧
+ pub async fn refresh(&self, pool: &sqlx::PgPool) -> utopia_core::AppResult<()> {
+ let keys: Vec<(String, String)> = self.inner.lock().unwrap().keys().cloned().collect();
+ for key in keys {
+ let limit = utopia_store::model_limits::limit_for(pool, &key.0, &key.1).await?;
+ self.resize(&key, limit);
+ }
+ Ok(())
+ }
+
+ fn resize(&self, key: &(String, String), limit: usize) -> Arc<Semaphore> {
+ let mut gates = self.inner.lock().unwrap();
+ let gate = gates.entry(key.clone()).or_insert_with(|| Gate {
+ sem: Arc::new(Semaphore::new(limit)),
+ limit,
+ owed: 0,
+ });
+ if limit > gate.limit {
+ gate.sem.add_permits(limit - gate.limit);
+ } else if limit < gate.limit {
+ let lower = gate.limit - limit;
+ gate.owed += lower - gate.sem.forget_permits(lower);
+ }
+ gate.limit = limit;
+ gate.sem.clone()
+ }
+
+ /// 还欠着许可就该收掉刚拿到的这张
+ fn repay(&self, key: &(String, String)) -> bool {
+ match self.inner.lock().unwrap().get_mut(key) {
+ Some(gate) if gate.owed > 0 => {
+ gate.owed -= 1;
+ true
+ }
+ _ => false,
}
}
}
@@ -119,13 +179,7 @@ pub async fn acquire(
let limit = utopia_store::model_limits::limit_for(&state.pool, base_url, model)
.await
.ok()?;
- let key = format!("{base_url}|{model}");
- state
- .model_gates
- .gate(&key, limit)
- .acquire_owned()
- .await
- .ok()
+ state.model_gates.admit(base_url, model, limit).await
}
/// `acquire` 的便捷形式:直接从工作区设置取 chat 模型的身份。
@@ -139,3 +193,7 @@ pub async fn acquire_embed(state: &AppState, s: &LlmSettings) -> Option<OwnedSem
let (base, model) = (s.embed_base_url.as_deref()?, s.embed_model.as_deref()?);
acquire(state, base, model).await
}
+
+#[cfg(test)]
+#[path = "llm_util_tests.rs"]
+mod tests;
diff --git a/crates/utopia-server/src/llm_util_tests.rs b/crates/utopia-server/src/llm_util_tests.rs
new file mode 100644
index 00000000..e27179e5
--- /dev/null
+++ b/crates/utopia-server/src/llm_util_tests.rs
@@ -0,0 +1,100 @@
+//! 限额就地调整(#1302):管理员调高,已经在等的任务当场放行;调低要等在飞的调用还回来才生效。
+
+use super::ModelGates;
+use axum::body::Body;
+use axum::http::{Request, StatusCode};
+use serde_json::json;
+use std::sync::Arc;
+use std::time::Duration;
+use tokio::time::timeout;
+use tower::ServiceExt;
+use uuid::Uuid;
+
+const SOON: Duration = Duration::from_millis(100);
+
+/// 调高不靠下一个来取许可的任务:所有 worker 都等着时,管理员存下新限额的那一下就得把它们放进来
+#[tokio::test]
+async fn an_admin_raising_a_limit_admits_a_task_already_waiting() -> anyhow::Result<()> {
+ let Some(url) = utopia_store::test_db::url() else {
+ return Ok(());
+ };
+ let pool = sqlx::PgPool::connect(&url).await?;
+ utopia_store::db::migrate(&pool).await?;
+ let (org, admin) = (Uuid::now_v7(), Uuid::now_v7());
+ let base = format!("http://gate-{}", Uuid::now_v7());
+ let dir = tempfile::tempdir()?;
+ let cfg = utopia_core::config::AppConfig {
+ data_dir: dir.path().to_string_lossy().into_owned(),
+ ..Default::default()
+ };
+ let state = crate::state::AppState::new(
+ pool.clone(),
+ &cfg,
+ Arc::new(utopia_search::SearchIndex::open(
+ &dir.path().join("search"),
+ )?),
+ "test-only".into(),
+ );
+ sqlx::raw_sql(&format!(
+ "INSERT INTO organizations(id,name) VALUES('{org}','model-gate-test');
+ INSERT INTO users(id,org_id,email,password_hash,display_name,is_admin)
+ VALUES('{admin}','{org}','{admin}@example.test','unused','Admin',TRUE);"
+ ))
+ .execute(&pool)
+ .await?;
+ utopia_store::model_limits::set(&pool, &base, "m", Some(1)).await?;
+ let result = async {
+ let _in_flight = super::acquire(&state, &base, "m").await;
+ let (s, b) = (state.clone(), base.clone());
+ let mut waiting = tokio::spawn(async move { super::acquire(&s, &b, "m").await.is_some() });
+ let parked = timeout(SOON, &mut waiting).await.is_err();
+
+ // 这个接口每次都写 open_registration,原样送回去,部署设置那一行不变
+ let open = utopia_store::access::open_registration(&pool).await?;
+ let body = json!({
+ "open_registration": open,
+ "model_limit": {"base_url": base, "model": "m", "max_concurrent": 2},
+ });
+ let request = Request::builder()
+ .method("PUT")
+ .uri("/api/v1/admin/deployment")
+ .header(
+ "authorization",
+ format!("Bearer {}", crate::auth::issue_token(&state, admin)?),
+ )
+ .header("content-type", "application/json")
+ .body(Body::from(body.to_string()))?;
+ let status = crate::api::router(state.clone(), &Default::default())
+ .oneshot(request)
+ .await?
+ .status();
+ let admitted = timeout(SOON, &mut waiting).await.is_ok();
+ anyhow::ensure!(
+ status == StatusCode::OK && parked && admitted,
+ "status: {status}, parked: {parked}, admitted: {admitted}"
+ );
+ anyhow::Ok(())
+ }
+ .await;
+ utopia_store::model_limits::set(&pool, &base, "m", None).await?;
+ sqlx::query("DELETE FROM organizations WHERE id=$1")
+ .bind(org)
+ .execute(&pool)
+ .await?;
+ result
+}
+
+#[tokio::test]
+async fn lowering_the_limit_takes_effect_as_in_flight_calls_return() {
+ let gates = Arc::new(ModelGates::default());
+ let first = gates.admit("u", "m", 2).await.unwrap();
+ let second = gates.admit("u", "m", 2).await.unwrap();
+ let g = gates.clone();
+ let mut waiting = tokio::spawn(async move { g.admit("u", "m", 1).await });
+
+ // 两个在飞降到一个:还回来的第一张被收走,第二个调用还没完就仍满额
+ drop(first);
+ assert!(timeout(SOON, &mut waiting).await.is_err());
+ drop(second);
+ assert!(timeout(SOON, &mut waiting).await.is_ok());
+}
diff --git a/crates/utopia-store/src/model_limits.rs b/crates/utopia-store/src/model_limits.rs
index 2f30b1df..01d3b956 100644
--- a/crates/utopia-store/src/model_limits.rs
+++ b/crates/utopia-store/src/model_limits.rs
@@ -13,7 +13,8 @@ use utopia_core::AppResult;
/// 这个模型允许多少并发。没有专属配置就走部署缺省。
///
/// 每次 LLM 调用前查一次——相对于一次动辄二十几秒的调用,这个查询的成本可以
-/// 忽略,换来的是"管理员改完即时生效",不必做缓存失效。
+/// 忽略。闸门手里也存着一份:管理员改限额时 `ModelGates::refresh` 再查一遍、当场
+/// 调闸门(#1302),等在闸门上的任务不必等下一次调用来读。
pub async fn limit_for(pool: &PgPool, base_url: &str, model: &str) -> AppResult<usize> {
let row: Option<(i32,)> = sqlx::query_as(
"SELECT max_concurrent FROM model_concurrency WHERE base_url = $1 AND model = $2",