World's first open-source enterprise world model.
Rust agent-memorybitemporalgraphragknowledge-baseknowledge-graphllmontologypgvectorpostgresqlragrustself-hostedsemantic-searchtemporal-knowledge-graphworld-model
Exercise passage clues and adjudication inputs with scripted namesake cases (#1202)
Virtue03942cfb942d682
diff --git a/crates/utopia-server/src/main.rs b/crates/utopia-server/src/main.rsindex 6b1ede67..03e98a17 100644--- a/crates/utopia-server/src/main.rs+++ b/crates/utopia-server/src/main.rs@@ -19,6 +19,8 @@ mod live;mod llm_util;mod mapping_index;mod mappings;+#[cfg(test)]+mod namesake_scripted_cases;mod notion;mod object_storage;mod ontology_agent;diff --git a/crates/utopia-server/src/namesake_scripted_cases.rs b/crates/utopia-server/src/namesake_scripted_cases.rsnew file mode 100644index 00000000..8e245ab6--- /dev/null+++ b/crates/utopia-server/src/namesake_scripted_cases.rs@@ -0,0 +1,449 @@+//! Fixed, DB-backed namesake cases for #1136 and #1202; no provider or embedding calls.+//! Run with a migrated UTOPIA_DATABASE_URL and UTOPIA_TEST_REQUIRE_DB=1:+//! cargo test --locked -p utopia-server namesake_scripted_cases -- --nocapture+//! Normal cargo test --workspace runs this too; no CI flags or ignored tests.+//!+//! Resolver pairs have two existing candidates with different department facts.+//! The incoming passage names one department: it conflicts with the profile preference+//! in the failing case and agrees in the control. Gold is that candidate's exact ID,+//! so a resolver that always creates another entity fails the control.+//!+//! Adjudicator pairs test the real HTTP request, not a scripted model's judgement:+//! scalar employee numbers and entity-valued departments must reach their own Record+//! A/B sections. Fixed different replies only complete the production caller.+//! Both seed orders run independently. The named list requires today's known failures+//! to fail and every other check to pass; a fix removes its names from that list.+use crate::state::AppState;+use axum::{extract::State, http::StatusCode, response::IntoResponse, routing::post, Json, Router};+use serde_json::{json, Value};+use sha2::{Digest, Sha256};+use std::{+ collections::VecDeque,+ sync::{Arc, Mutex},+ time::Duration,+};+use utopia_store::resolution::ReviewStage;+use uuid::Uuid;++// Both profiles are viable: cosine 1.0 versus 0.8, outside the existing tie guard.+const DEPARTMENTS: [&str; 2] = ["Platform Engineering", "Finance"];+const EMPLOYEES: [&str; 2] = ["E17", "F42"];++#[derive(Clone, Copy)]+enum Check {+ Resolver { target: usize },+ Adjudicator { literal: bool },+}++struct Case {+ id: &'static str,+ check: Check,+}++const CASES: [Case; 4] = [+ Case {+ id: "resolver_passage_over_profile",+ check: Check::Resolver { target: 1 },+ },+ Case {+ id: "resolver_passage_agrees_with_profile",+ check: Check::Resolver { target: 0 },+ },+ Case {+ id: "adjudicator_literal_clues_reach_request",+ check: Check::Adjudicator { literal: true },+ },+ Case {+ id: "adjudicator_entity_clues_reach_request",+ check: Check::Adjudicator { literal: false },+ },+];++// #1136: a listed case MUST fail today; every other case MUST pass. A fix removes+// the corresponding names with its code change, so success cannot silently go stale.+const KNOWN_FAILURES: [(&str, &str); 4] = [+ ("resolver_passage_over_profile", "forward"),+ ("resolver_passage_over_profile", "reverse"),+ ("adjudicator_literal_clues_reach_request", "forward"),+ ("adjudicator_literal_clues_reach_request", "reverse"),+];++struct Step {+ prompt_has: String,+ text: String,+ stream: bool,+}++#[derive(Clone, Default)]+struct Model {+ steps: Arc<Mutex<VecDeque<Step>>>,+ requests: Arc<Mutex<Vec<Value>>>,+ errors: Arc<Mutex<Vec<String>>>,+}++async fn reply(State(m): State<Model>, Json(body): Json<Value>) -> axum::response::Response {+ m.requests.lock().unwrap().push(body.clone());+ let step = m.steps.lock().unwrap().pop_front();+ let error = match &step {+ None => Some("unexpected model request: script exhausted".to_string()),+ Some(s) if !body["messages"].to_string().contains(&s.prompt_has) => {+ Some(format!("unexpected prompt: missing {}", s.prompt_has))+ }+ Some(s) if (body["stream"] == true) != s.stream => {+ Some("unexpected chat transport".to_string())+ }+ _ => None,+ };+ if let Some(error) = error {+ // Handler panics alone can be lost in a detached task; the caller checks this ledger.+ m.errors.lock().unwrap().push(error.clone());+ return (StatusCode::BAD_REQUEST, error).into_response();+ }+ let step = step.unwrap();+ if !step.stream {+ return Json(+ json!({"choices":[{"message":{"role":"assistant","content":step.text},+ "finish_reason":"stop"}]}),+ )+ .into_response();+ }+ let frame = json!({"choices":[{"delta":{"content":step.text}}]});+ let done = json!({"choices":[{"delta":{},"finish_reason":"stop"}]});+ (+ [("content-type", "text/event-stream")],+ format!("data: {frame}\n\ndata: {done}\n\ndata: [DONE]\n\n"),+ )+ .into_response()+}++struct Fx {+ pool: sqlx::PgPool,+ state: AppState,+ org: Uuid,+ kb: Uuid,+ docs: [Uuid; 3],+ chunks: [Uuid; 3],+ quotes: [String; 3],+ departments: [Uuid; 2],+ model: Model,+ server: tokio::task::JoinHandle<()>,+ _dir: tempfile::TempDir,+}++impl Fx {+ async fn new(pool: &sqlx::PgPool, case: &Case, org: Uuid) -> anyhow::Result<Self> {+ let (ws, kb) = (Uuid::now_v7(), Uuid::now_v7());+ sqlx::query("INSERT INTO organizations(id,name) VALUES ($1,'scripted-namesake')")+ .bind(org)+ .execute(pool)+ .await?;+ sqlx::query("INSERT INTO workspaces(id,org_id,name) VALUES ($1,$2,'scripted-namesake')")+ .bind(ws)+ .bind(org)+ .execute(pool)+ .await?;+ sqlx::query(+ "INSERT INTO knowledge_bases(id,workspace_id,name) VALUES ($1,$2,'scripted-namesake')",+ )+ .bind(kb)+ .bind(ws)+ .execute(pool)+ .await?;+ let docs = std::array::from_fn(|_| Uuid::now_v7());+ let chunks = std::array::from_fn(|_| Uuid::now_v7());+ let departments = [Uuid::now_v7(), Uuid::now_v7()];+ for i in 0..2 {+ sqlx::query("INSERT INTO entities(id,kb_id,canonical_name) VALUES ($1,$2,$3)")+ .bind(departments[i])+ .bind(kb)+ .bind(DEPARTMENTS[i])+ .execute(pool)+ .await?;+ }+ let quotes = std::array::from_fn(|i| match (case.check, i) {+ (Check::Adjudicator { literal: true }, 0..=1) => format!(+ "Directory {}: Alex Lee has employee number {}.",+ i + 1,+ EMPLOYEES[i]+ ),+ (_, 0..=1) => format!(+ "Directory {}: Alex Lee works for {}.",+ i + 1,+ DEPARTMENTS[i]+ ),+ (Check::Resolver { target }, _) => {+ format!("Alex Lee from {} submitted a report.", DEPARTMENTS[target])+ }+ _ => "Unused incoming passage.".into(),+ });+ for i in 0..3 {+ sqlx::query("INSERT INTO documents(id,kb_id,filename,sha256) VALUES ($1,$2,$3,$4)")+ .bind(docs[i])+ .bind(kb)+ .bind(format!("directory-{i}.txt"))+ .bind(+ Sha256::digest(quotes[i].as_bytes())+ .iter()+ .map(|byte| format!("{byte:02x}"))+ .collect::<String>(),+ )+ .execute(pool)+ .await?;+ // Fixed context vectors exercise the real resolver without downloading an+ // embedding model or involving the independent name-vector recall channel.+ sqlx::query("INSERT INTO chunks(id,kb_id,document_id,seq,text,embedding) VALUES ($1,$2,$3,0,$4,'[1,0,0]'::vector)")+ .bind(chunks[i]).bind(kb).bind(docs[i]).bind("es[i]).execute(pool).await?;+ }+ let model = Model::default();+ let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?;+ let endpoint = format!("http://{}", listener.local_addr()?);+ let router = Router::new()+ .route("/chat/completions", post(reply))+ .with_state(model.clone());+ utopia_store::settings::upsert(+ pool,+ ws,+ Some(&endpoint),+ None,+ Some("scripted"),+ None,+ None,+ None,+ None,+ )+ .await?;+ let dir = tempfile::tempdir()?;+ let config = utopia_core::config::AppConfig {+ data_dir: dir.path().to_string_lossy().into_owned(),+ ..Default::default()+ };+ let search = Arc::new(utopia_search::SearchIndex::open(+ &dir.path().join("search"),+ )?);+ let state = AppState::new(pool.clone(), &config, search, "test-only".into());+ // Start only after fallible fixture initialization, so errors cannot leak a server.+ let server = tokio::spawn(async move { axum::serve(listener, router).await.unwrap() });+ Ok(Self {+ pool: pool.clone(),+ state,+ org,+ kb,+ docs,+ chunks,+ quotes,+ departments,+ model,+ server,+ _dir: dir,+ })+ }++ async fn seed(&self, i: usize, check: Check) -> anyhow::Result<Uuid> {+ let id = Uuid::now_v7();+ let profile = if i == 0 { "[1,0,0]" } else { "[0.8,0.6,0]" };+ sqlx::query("INSERT INTO entities(id,kb_id,canonical_name,profile_embedding,profile_n) VALUES ($1,$2,'Alex Lee',$3::vector,1)")+ .bind(id).bind(self.kb).bind(profile).execute(&self.pool).await?;+ let fact = Uuid::now_v7();+ if matches!(check, Check::Adjudicator { literal: true }) {+ sqlx::query("INSERT INTO facts(id,kb_id,subject_id,layer,phrase,object_value) VALUES ($1,$2,$3,'open','employee number',$4)")+ .bind(fact).bind(self.kb).bind(id).bind(json!({"value":EMPLOYEES[i]})).execute(&self.pool).await?;+ } else {+ sqlx::query("INSERT INTO facts(id,kb_id,subject_id,layer,phrase,object_id) VALUES ($1,$2,$3,'open','works for',$4)")+ .bind(fact).bind(self.kb).bind(id).bind(self.departments[i]).execute(&self.pool).await?;+ }+ sqlx::query("INSERT INTO fact_evidence(fact_id,chunk_id,quote,document_id,doc_version) VALUES ($1,$2,$3,$4,1)")+ .bind(fact).bind(self.chunks[i]).bind(&self.quotes[i]).bind(self.docs[i]).execute(&self.pool).await?;+ Ok(id)+ }++ async fn run(&self, case: &Case, reverse: bool) -> anyhow::Result<Value> {+ let order = if reverse { [1, 0] } else { [0, 1] };+ let mut holders = [Uuid::nil(); 2];+ for i in order {+ holders[i] = self.seed(i, case.check).await?;+ }+ let (passed, observation) = match case.check {+ Check::Resolver { target } => {+ self.model.steps.lock().unwrap().push_back(Step {+ prompt_has: self.quotes[2].clone(), stream: true,+ text: json!({"e":[["Alex Lee","person",1]],+ "s":[[self.quotes[2],"Alex Lee","submitted",null,"a report",null,null,null]],+ "n":[]}).to_string(),+ });+ crate::extraction::extract_document(&self.state, self.docs[2], Default::default())+ .await?;+ let status: String =+ sqlx::query_scalar("SELECT graph_status FROM documents WHERE id=$1")+ .bind(self.docs[2])+ .fetch_one(&self.pool)+ .await?;+ anyhow::ensure!(status == "done", "extraction did not complete: {status}");+ let owners: Vec<Uuid> = sqlx::query_scalar(+ "SELECT DISTINCT f.subject_id FROM facts f JOIN fact_evidence fe ON fe.fact_id=f.id+ WHERE f.kb_id=$1 AND fe.document_id=$2 AND f.layer='open' AND f.phrase='submitted'")+ .bind(self.kb).bind(self.docs[2]).fetch_all(&self.pool).await?;+ anyhow::ensure!(+ owners.len() == 1,+ "incoming source has {} owners",+ owners.len()+ );+ // Exact pre-existing ID: creating a new entity is not successful disambiguation.+ (+ owners[0] == holders[target],+ json!({"expected_owner":holders[target],+ "actual_owner":owners[0],"passage":self.quotes[2]}),+ )+ }+ Check::Adjudicator { literal } => {+ utopia_store::resolution::create_review(+ &self.pool,+ self.kb,+ holders[order[0]],+ holders[order[1]],+ 0.7,+ "scripted_same_name",+ ReviewStage::Adjudicating,+ )+ .await?;+ self.model.steps.lock().unwrap().push_back(Step {+ prompt_has: "Alex Lee".into(),+ stream: false,+ // The reply only completes the caller. We test its INPUT, not model judgement.+ text: json!({"verdicts":[{"i":0,"verdict":"different","confidence":0.99,+ "why":"scripted transport reply"}]})+ .to_string(),+ });+ crate::adjudication::adjudicate_entities(&self.state, self.kb).await?;+ let requests = self.model.requests.lock().unwrap();+ let user = requests+ .first()+ .and_then(|r| r["messages"].as_array())+ .and_then(|m| m.iter().find(|m| m["role"] == "user"))+ .and_then(|m| m["content"].as_str())+ .ok_or_else(|| anyhow::anyhow!("missing user request"))?;+ let pair = user+ .strip_prefix("Pair 0:\nRecord A: ")+ .ok_or_else(|| anyhow::anyhow!("missing first record"))?;+ let (left, right) = pair+ .split_once("\nRecord B: ")+ .ok_or_else(|| anyhow::anyhow!("missing second record"))?;+ let (phrase, clues) = if literal {+ ("employee number", EMPLOYEES)+ } else {+ ("works for", DEPARTMENTS)+ };+ let expected = order.map(|i| format!("{phrase} → {}", clues[i]));+ // Check each real HTTP side independently: overall contains would allow swaps.+ let visible = left.contains(&expected[0]) && right.contains(&expected[1]);+ let correctly_owned =+ !left.contains(clues[order[1]]) && !right.contains(clues[order[0]]);+ (+ visible && correctly_owned,+ json!({"expected_facts":expected,"request_sides":[left,right]}),+ )+ }+ };+ anyhow::ensure!(+ self.model.errors.lock().unwrap().is_empty(),+ "script errors: {:?}",+ self.model.errors.lock().unwrap()+ );+ anyhow::ensure!(+ self.model.steps.lock().unwrap().is_empty(),+ "unconsumed model script"+ );+ let model_calls = self.model.requests.lock().unwrap().len();+ anyhow::ensure!(+ model_calls == 1,+ "expected exactly one scripted request, got {model_calls}"+ );+ let pending_human: i64 = sqlx::query_scalar("SELECT count(*) FROM resolution_reviews WHERE kb_id=$1 AND stage='human' AND status='pending'")+ .bind(self.kb).fetch_one(&self.pool).await?;+ anyhow::ensure!(+ pending_human == 0,+ "unexpected pending Human reviews: {pending_human}"+ );+ Ok(+ json!({"id":case.id,"order":if reverse {"reverse"} else {"forward"},+ "passed":passed,"observation":observation,"pending_human":pending_human,"model_calls":model_calls}),+ )+ }++ async fn cleanup(&self) -> anyhow::Result<()> {+ self.server.abort();+ sqlx::query(+ "DELETE FROM jobs WHERE payload->>'kb_id'=$1 OR payload->>'document_id'=ANY($2)",+ )+ .bind(self.kb.to_string())+ .bind(self.docs.map(|d| d.to_string()).to_vec())+ .execute(&self.pool)+ .await?;+ sqlx::query("DELETE FROM organizations WHERE id=$1")+ .bind(self.org)+ .execute(&self.pool)+ .await?;+ Ok(())+ }+}++#[tokio::test]+async fn scripted_namesake_cases_have_only_the_named_known_failures() -> anyhow::Result<()> {+ let Some(url) = utopia_store::test_db::url() else {+ return Ok(());+ };+ let pool = sqlx::PgPool::connect(&url).await?;+ // CI migrates first; this fixture must not race a global migration.+ let mut mismatches = Vec::new();+ let mut seen = Vec::new();+ let mut failed = 0;+ for case in &CASES {+ for reverse in [false, true] {+ let order = if reverse { "reverse" } else { "forward" };+ let org = Uuid::now_v7();+ let f = match Fx::new(&pool, case, org).await {+ Ok(f) => f,+ Err(error) => {+ sqlx::query("DELETE FROM organizations WHERE id=$1")+ .bind(org)+ .execute(&pool)+ .await?;+ return Err(error);+ }+ };+ let run = tokio::time::timeout(Duration::from_secs(15), f.run(case, reverse)).await;+ f.cleanup().await?;+ let row = run.map_err(|_| anyhow::anyhow!("{} timed out", case.id))??;+ println!("NAMESAKE_CASE {row}");+ let passed = row["passed"].as_bool().unwrap();+ failed += usize::from(!passed);+ let name = (case.id, order);+ seen.push(name);+ if KNOWN_FAILURES.contains(&name) == passed {+ mismatches.push(format!(+ "{} / {order}: {}",+ case.id,+ if passed {+ "unexpected pass: remove this name with its fix"+ } else {+ "unlisted failure"+ }+ ));+ }+ }+ }+ anyhow::ensure!(+ KNOWN_FAILURES.iter().all(|name| seen.contains(name)),+ "unused known-failure name"+ );+ println!(+ "NAMESAKE_SUMMARY {}",+ json!({"cases":seen.len(),"failed_checks":failed,+ "known_failures":KNOWN_FAILURES,"mismatches":mismatches})+ );+ anyhow::ensure!(+ mismatches.is_empty(),+ "namesake expectation mismatch: {}",+ mismatches.join("; ")+ );+ Ok(())+}