World's first open-source enterprise world model.
Rust agent-memorybitemporalgraphragknowledge-baseknowledge-graphllmontologypgvectorpostgresqlragrustself-hostedsemantic-searchtemporal-knowledge-graphworld-model
//! Fixed, DB-backed namesake cases for #1136 and #1202; no provider or embedding calls.
//! Run with a migrated UTOPIA_DATABASE_URL and UTOPIA_TEST_REQUIRE_DB=1:
//! cargo test --locked -p utopia-server namesake_scripted_cases -- --nocapture
//! Normal cargo test --workspace runs this too; no CI flags or ignored tests.
//!
//! Resolver pairs have two existing candidates with different department facts.
//! The incoming passage names one department: it conflicts with the profile preference
//! in the failing case and agrees in the control. Gold is that candidate's exact ID,
//! so a resolver that always creates another entity fails the control.
//!
//! Adjudicator pairs test the real HTTP request, not a scripted model's judgement:
//! scalar employee numbers and entity-valued departments must reach their own Record
//! A/B sections. Fixed different replies only complete the production caller.
//! Both seed orders run independently. The named list requires today's known failures
//! to fail and every other check to pass; a fix removes its names from that list.
use crate::state::AppState;
use axum::{extract::State, http::StatusCode, response::IntoResponse, routing::post, Json, Router};
use serde_json::{json, Value};
use sha2::{Digest, Sha256};
use std::{
collections::VecDeque,
sync::{Arc, Mutex},
time::Duration,
};
use utopia_store::resolution::ReviewStage;
use uuid::Uuid;
// Both profiles are viable: cosine 1.0 versus 0.8, outside the existing tie guard.
const DEPARTMENTS: [&str; 2] = ["Platform Engineering", "Finance"];
const EMPLOYEES: [&str; 2] = ["E17", "F42"];
#[derive(Clone, Copy)]
enum Check {
Resolver { target: usize },
Adjudicator { literal: bool },
}
struct Case {
id: &'static str,
check: Check,
}
const CASES: [Case; 4] = [
Case {
id: "resolver_passage_over_profile",
check: Check::Resolver { target: 1 },
},
Case {
id: "resolver_passage_agrees_with_profile",
check: Check::Resolver { target: 0 },
},
Case {
id: "adjudicator_literal_clues_reach_request",
check: Check::Adjudicator { literal: true },
},
Case {
id: "adjudicator_entity_clues_reach_request",
check: Check::Adjudicator { literal: false },
},
];
// #1136: a listed case MUST fail today; every other case MUST pass. A fix removes
// the corresponding names with its code change, so success cannot silently go stale.
const KNOWN_FAILURES: [(&str, &str); 2] = [
("resolver_passage_over_profile", "forward"),
("resolver_passage_over_profile", "reverse"),
];
struct Step {
prompt_has: String,
text: String,
stream: bool,
}
#[derive(Clone, Default)]
struct Model {
steps: Arc<Mutex<VecDeque<Step>>>,
requests: Arc<Mutex<Vec<Value>>>,
errors: Arc<Mutex<Vec<String>>>,
}
async fn reply(State(m): State<Model>, Json(body): Json<Value>) -> axum::response::Response {
m.requests.lock().unwrap().push(body.clone());
let step = m.steps.lock().unwrap().pop_front();
let error = match &step {
None => Some("unexpected model request: script exhausted".to_string()),
Some(s) if !body["messages"].to_string().contains(&s.prompt_has) => {
Some(format!("unexpected prompt: missing {}", s.prompt_has))
}
Some(s) if (body["stream"] == true) != s.stream => {
Some("unexpected chat transport".to_string())
}
_ => None,
};
if let Some(error) = error {
// Handler panics alone can be lost in a detached task; the caller checks this ledger.
m.errors.lock().unwrap().push(error.clone());
return (StatusCode::BAD_REQUEST, error).into_response();
}
let step = step.unwrap();
if !step.stream {
return Json(
json!({"choices":[{"message":{"role":"assistant","content":step.text},
"finish_reason":"stop"}]}),
)
.into_response();
}
let frame = json!({"choices":[{"delta":{"content":step.text}}]});
let done = json!({"choices":[{"delta":{},"finish_reason":"stop"}]});
(
[("content-type", "text/event-stream")],
format!("data: {frame}\n\ndata: {done}\n\ndata: [DONE]\n\n"),
)
.into_response()
}
struct Fx {
pool: sqlx::PgPool,
state: AppState,
org: Uuid,
kb: Uuid,
docs: [Uuid; 3],
chunks: [Uuid; 3],
quotes: [String; 3],
departments: [Uuid; 2],
model: Model,
server: tokio::task::JoinHandle<()>,
_dir: tempfile::TempDir,
}
impl Fx {
async fn new(pool: &sqlx::PgPool, case: &Case, org: Uuid) -> anyhow::Result<Self> {
let (ws, kb) = (Uuid::now_v7(), Uuid::now_v7());
sqlx::query("INSERT INTO organizations(id,name) VALUES ($1,'scripted-namesake')")
.bind(org)
.execute(pool)
.await?;
sqlx::query("INSERT INTO workspaces(id,org_id,name) VALUES ($1,$2,'scripted-namesake')")
.bind(ws)
.bind(org)
.execute(pool)
.await?;
sqlx::query(
"INSERT INTO knowledge_bases(id,workspace_id,name) VALUES ($1,$2,'scripted-namesake')",
)
.bind(kb)
.bind(ws)
.execute(pool)
.await?;
let docs = std::array::from_fn(|_| Uuid::now_v7());
let chunks = std::array::from_fn(|_| Uuid::now_v7());
let departments = [Uuid::now_v7(), Uuid::now_v7()];
for i in 0..2 {
sqlx::query("INSERT INTO entities(id,kb_id,canonical_name) VALUES ($1,$2,$3)")
.bind(departments[i])
.bind(kb)
.bind(DEPARTMENTS[i])
.execute(pool)
.await?;
}
let quotes = std::array::from_fn(|i| match (case.check, i) {
(Check::Adjudicator { literal: true }, 0..=1) => format!(
"Directory {}: Alex Lee has employee number {}.",
i + 1,
EMPLOYEES[i]
),
(_, 0..=1) => format!(
"Directory {}: Alex Lee works for {}.",
i + 1,
DEPARTMENTS[i]
),
(Check::Resolver { target }, _) => {
format!("Alex Lee from {} submitted a report.", DEPARTMENTS[target])
}
_ => "Unused incoming passage.".into(),
});
for i in 0..3 {
sqlx::query("INSERT INTO documents(id,kb_id,filename,sha256) VALUES ($1,$2,$3,$4)")
.bind(docs[i])
.bind(kb)
.bind(format!("directory-{i}.txt"))
.bind(
Sha256::digest(quotes[i].as_bytes())
.iter()
.map(|byte| format!("{byte:02x}"))
.collect::<String>(),
)
.execute(pool)
.await?;
// Fixed context vectors exercise the real resolver without downloading an
// embedding model or involving the independent name-vector recall channel.
sqlx::query("INSERT INTO chunks(id,kb_id,document_id,seq,text,embedding) VALUES ($1,$2,$3,0,$4,'[1,0,0]'::vector)")
.bind(chunks[i]).bind(kb).bind(docs[i]).bind("es[i]).execute(pool).await?;
}
let model = Model::default();
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?;
let endpoint = format!("http://{}", listener.local_addr()?);
let router = Router::new()
.route("/chat/completions", post(reply))
.with_state(model.clone());
utopia_store::settings::upsert(
pool,
ws,
Some(&endpoint),
None,
Some("scripted"),
None,
None,
None,
None,
)
.await?;
let dir = tempfile::tempdir()?;
let config = utopia_core::config::AppConfig {
data_dir: dir.path().to_string_lossy().into_owned(),
..Default::default()
};
let search = Arc::new(utopia_search::SearchIndex::open(
&dir.path().join("search"),
)?);
let state = AppState::new(pool.clone(), &config, search, "test-only".into());
// Start only after fallible fixture initialization, so errors cannot leak a server.
let server = tokio::spawn(async move { axum::serve(listener, router).await.unwrap() });
Ok(Self {
pool: pool.clone(),
state,
org,
kb,
docs,
chunks,
quotes,
departments,
model,
server,
_dir: dir,
})
}
async fn seed(&self, i: usize, check: Check) -> anyhow::Result<Uuid> {
let id = Uuid::now_v7();
let profile = if i == 0 { "[1,0,0]" } else { "[0.8,0.6,0]" };
sqlx::query("INSERT INTO entities(id,kb_id,canonical_name,profile_embedding,profile_n) VALUES ($1,$2,'Alex Lee',$3::vector,1)")
.bind(id).bind(self.kb).bind(profile).execute(&self.pool).await?;
let fact = Uuid::now_v7();
if matches!(check, Check::Adjudicator { literal: true }) {
sqlx::query("INSERT INTO facts(id,kb_id,subject_id,layer,phrase,object_value) VALUES ($1,$2,$3,'open','employee number',$4)")
.bind(fact).bind(self.kb).bind(id).bind(json!({"value":EMPLOYEES[i]})).execute(&self.pool).await?;
} else {
sqlx::query("INSERT INTO facts(id,kb_id,subject_id,layer,phrase,object_id) VALUES ($1,$2,$3,'open','works for',$4)")
.bind(fact).bind(self.kb).bind(id).bind(self.departments[i]).execute(&self.pool).await?;
}
sqlx::query("INSERT INTO fact_evidence(fact_id,chunk_id,quote,document_id,doc_version) VALUES ($1,$2,$3,$4,1)")
.bind(fact).bind(self.chunks[i]).bind(&self.quotes[i]).bind(self.docs[i]).execute(&self.pool).await?;
Ok(id)
}
async fn run(&self, case: &Case, reverse: bool) -> anyhow::Result<Value> {
let order = if reverse { [1, 0] } else { [0, 1] };
let mut holders = [Uuid::nil(); 2];
for i in order {
holders[i] = self.seed(i, case.check).await?;
}
let (passed, observation) = match case.check {
Check::Resolver { target } => {
self.model.steps.lock().unwrap().push_back(Step {
prompt_has: self.quotes[2].clone(), stream: true,
text: json!({"e":[["Alex Lee","person",1]],
"s":[[self.quotes[2],"Alex Lee","submitted",null,"a report",null,null,null]],
"n":[]}).to_string(),
});
crate::extraction::extract_document(&self.state, self.docs[2], Default::default())
.await?;
let status: String =
sqlx::query_scalar("SELECT graph_status FROM documents WHERE id=$1")
.bind(self.docs[2])
.fetch_one(&self.pool)
.await?;
anyhow::ensure!(status == "done", "extraction did not complete: {status}");
let owners: Vec<Uuid> = sqlx::query_scalar(
"SELECT DISTINCT f.subject_id FROM facts f JOIN fact_evidence fe ON fe.fact_id=f.id
WHERE f.kb_id=$1 AND fe.document_id=$2 AND f.layer='open' AND f.phrase='submitted'")
.bind(self.kb).bind(self.docs[2]).fetch_all(&self.pool).await?;
anyhow::ensure!(
owners.len() == 1,
"incoming source has {} owners",
owners.len()
);
// Exact pre-existing ID: creating a new entity is not successful disambiguation.
(
owners[0] == holders[target],
json!({"expected_owner":holders[target],
"actual_owner":owners[0],"passage":self.quotes[2]}),
)
}
Check::Adjudicator { literal } => {
utopia_store::resolution::create_review(
&self.pool,
self.kb,
holders[order[0]],
holders[order[1]],
0.7,
"scripted_same_name",
ReviewStage::Adjudicating,
)
.await?;
self.model.steps.lock().unwrap().push_back(Step {
prompt_has: "Alex Lee".into(),
stream: false,
// The reply only completes the caller. We test its INPUT, not model judgement.
text: json!({"verdicts":[{"i":0,"verdict":"different","confidence":0.99,
"why":"scripted transport reply"}]})
.to_string(),
});
crate::adjudication::adjudicate_entities(&self.state, self.kb).await?;
let requests = self.model.requests.lock().unwrap();
let user = requests
.first()
.and_then(|r| r["messages"].as_array())
.and_then(|m| m.iter().find(|m| m["role"] == "user"))
.and_then(|m| m["content"].as_str())
.ok_or_else(|| anyhow::anyhow!("missing user request"))?;
let pair = user
.strip_prefix("Pair 0:\nRecord A: ")
.ok_or_else(|| anyhow::anyhow!("missing first record"))?;
let (left, right) = pair
.split_once("\nRecord B: ")
.ok_or_else(|| anyhow::anyhow!("missing second record"))?;
let (phrase, clues) = if literal {
("employee number", EMPLOYEES)
} else {
("works for", DEPARTMENTS)
};
let expected = order.map(|i| format!("{phrase} → {}", clues[i]));
// Check each real HTTP side independently: overall contains would allow swaps.
let visible = left.contains(&expected[0]) && right.contains(&expected[1]);
let correctly_owned =
!left.contains(clues[order[1]]) && !right.contains(clues[order[0]]);
(
visible && correctly_owned,
json!({"expected_facts":expected,"request_sides":[left,right]}),
)
}
};
anyhow::ensure!(
self.model.errors.lock().unwrap().is_empty(),
"script errors: {:?}",
self.model.errors.lock().unwrap()
);
anyhow::ensure!(
self.model.steps.lock().unwrap().is_empty(),
"unconsumed model script"
);
let model_calls = self.model.requests.lock().unwrap().len();
anyhow::ensure!(
model_calls == 1,
"expected exactly one scripted request, got {model_calls}"
);
let pending_human: i64 = sqlx::query_scalar("SELECT count(*) FROM resolution_reviews WHERE kb_id=$1 AND stage='human' AND status='pending'")
.bind(self.kb).fetch_one(&self.pool).await?;
anyhow::ensure!(
pending_human == 0,
"unexpected pending Human reviews: {pending_human}"
);
Ok(
json!({"id":case.id,"order":if reverse {"reverse"} else {"forward"},
"passed":passed,"observation":observation,"pending_human":pending_human,"model_calls":model_calls}),
)
}
async fn cleanup(&self) -> anyhow::Result<()> {
self.server.abort();
sqlx::query(
"DELETE FROM jobs WHERE payload->>'kb_id'=$1 OR payload->>'document_id'=ANY($2)",
)
.bind(self.kb.to_string())
.bind(self.docs.map(|d| d.to_string()).to_vec())
.execute(&self.pool)
.await?;
sqlx::query("DELETE FROM organizations WHERE id=$1")
.bind(self.org)
.execute(&self.pool)
.await?;
Ok(())
}
}
#[tokio::test]
async fn scripted_namesake_cases_have_only_the_named_known_failures() -> anyhow::Result<()> {
let Some(url) = utopia_store::test_db::url() else {
return Ok(());
};
let pool = sqlx::PgPool::connect(&url).await?;
// CI migrates first; this fixture must not race a global migration.
let mut mismatches = Vec::new();
let mut seen = Vec::new();
let mut failed = 0;
for case in &CASES {
for reverse in [false, true] {
let order = if reverse { "reverse" } else { "forward" };
let org = Uuid::now_v7();
let f = match Fx::new(&pool, case, org).await {
Ok(f) => f,
Err(error) => {
sqlx::query("DELETE FROM organizations WHERE id=$1")
.bind(org)
.execute(&pool)
.await?;
return Err(error);
}
};
let run = tokio::time::timeout(Duration::from_secs(15), f.run(case, reverse)).await;
f.cleanup().await?;
let row = run.map_err(|_| anyhow::anyhow!("{} timed out", case.id))??;
println!("NAMESAKE_CASE {row}");
let passed = row["passed"].as_bool().unwrap();
failed += usize::from(!passed);
let name = (case.id, order);
seen.push(name);
if KNOWN_FAILURES.contains(&name) == passed {
mismatches.push(format!(
"{} / {order}: {}",
case.id,
if passed {
"unexpected pass: remove this name with its fix"
} else {
"unlisted failure"
}
));
}
}
}
anyhow::ensure!(
KNOWN_FAILURES.iter().all(|name| seen.contains(name)),
"unused known-failure name"
);
println!(
"NAMESAKE_SUMMARY {}",
json!({"cases":seen.len(),"failed_checks":failed,
"known_failures":KNOWN_FAILURES,"mismatches":mismatches})
);
anyhow::ensure!(
mismatches.is_empty(),
"namesake expectation mismatch: {}",
mismatches.join("; ")
);
Ok(())
}