Skip to content

World's first open-source enterprise world model.

7.1 KB
Raw Blame
//! What the model is sent for one turn: the stored conversation, then the question once.
//!
//! The question is stored before the history is read, so the history already holds it, and
//! the runner sends the question again as its prompt. Each request the scripted endpoint
//! receives is checked for how often each question appears and what sits right before it.
use super::*;

/// How many user messages in one request say exactly `text`
fn user_turns(request: &serde_json::Value, text: &str) -> usize {
    request["messages"]
        .as_array()
        .into_iter()
        .flatten()
        .filter(|m| m["role"] == "user" && m["content"] == text)
        .count()
}

fn last_user(request: &serde_json::Value) -> String {
    request["messages"]
        .as_array()
        .into_iter()
        .flatten()
        .rev()
        .find(|m| m["role"] == "user")
        .and_then(|m| m["content"].as_str())
        .unwrap_or_default()
        .to_string()
}

/// Ask in an existing conversation and collect the whole SSE body
pub(super) async fn ask_in(f: &Fx, conversation_id: Uuid, message: &str) -> anyhow::Result<String> {
    let sse = chat(
        State(f.state.clone()),
        AuthUser(f.user.clone()),
        Path(f.kb),
        Json(ChatReq {
            conversation_id: Some(conversation_id),
            message: message.into(),
            retry_message_id: None,
        }),
    )
    .await
    .map_err(|_| anyhow::anyhow!("chat handler refused the request"))?;
    let body = axum::body::to_bytes(sse.into_response().into_body(), 4 * 1024 * 1024).await?;
    Ok(String::from_utf8_lossy(&body).into_owned())
}

#[tokio::test]
async fn the_question_reaches_the_model_once() -> anyhow::Result<()> {
    let gate = Reply::Tool("no_evidence_needed", r#"{"reason":"greeting"}"#);
    let Some(f) = fixture(Scripted::new(vec![
        gate,
        Reply::Text("First answer."),
        gate,
        Reply::Text("Second answer."),
    ]))
    .await?
    else {
        return Ok(());
    };
    let sse = f.ask("first question").await?;
    assert!(sse.contains("event: done"), "{sse}");
    let id: Uuid = sqlx::query_scalar("SELECT id FROM conversations WHERE kb_id = $1")
        .bind(f.kb)
        .fetch_one(&f.pool)
        .await?;
    let sse = ask_in(&f, id, "second question").await?;
    assert!(sse.contains("event: done"), "{sse}");

    let requests = f.requests();
    assert_eq!(requests.len(), 4, "a tool turn and an answer per question");
    for (i, request) in requests.iter().enumerate() {
        let current = if i < 2 {
            "first question"
        } else {
            "second question"
        };
        assert_eq!(
            user_turns(request, current),
            1,
            "request {i} must carry its question once:\n{request}"
        );
        assert_eq!(last_user(request), current, "request {i}:\n{request}");
        // The earlier question is replayed from the history, and only from there
        assert_eq!(
            user_turns(request, "first question"),
            1,
            "request {i}:\n{request}"
        );
    }
    f.cleanup().await
}

#[tokio::test]
async fn repeated_answers_keep_their_own_reasoning_when_history_is_replayed() -> anyhow::Result<()>
{
    const ANSWER: &str = "The same public answer.";
    let tool_thoughts = ["private-tool-1", "private-tool-2", "private-tool-3"];
    let answer_thoughts = ["private-answer-1", "private-answer-2", "private-answer-3"];
    let replies = tool_thoughts
        .into_iter()
        .zip(answer_thoughts)
        .flat_map(|(tool, answer)| {
            [
                Reply::ThoughtTool("no_evidence_needed", r#"{"reason":"greeting"}"#, tool),
                Reply::ThoughtText(ANSWER, answer),
            ]
        })
        .collect();
    let Some(f) = fixture(Scripted::new(replies)).await? else {
        return Ok(());
    };
    let id = utopia_store::conversations::create(&f.pool, f.kb, f.user.id, "Reasoning").await?;
    for question in ["first greeting", "second greeting", "third greeting"] {
        let sse = ask_in(&f, id, question).await?;
        assert!(sse.contains("event: done") && sse.contains(ANSWER), "{sse}");
        assert!(
            !sse.contains("private-"),
            "reasoning must not enter SSE: {sse}"
        );
        assert_eq!(f.stored_answer().await?.as_deref(), Some(ANSWER));
    }
    let requests = f.requests();
    assert_eq!(
        requests.len(),
        6,
        "each question has a tool turn then an answer"
    );
    let messages = requests[5]["messages"].as_array().expect("messages");
    let earlier_answers: Vec<_> = messages
        .iter()
        .filter(|m| m["role"] == "assistant" && m["content"] == ANSWER)
        .collect();
    assert_eq!(earlier_answers.len(), 2, "{messages:?}");
    for (message, reasoning) in earlier_answers.iter().zip(answer_thoughts) {
        assert_eq!(message["reasoning_content"], reasoning, "{message}");
    }
    let latest_tool = messages
        .iter()
        .rev()
        .find(|m| m["role"] == "assistant" && m["tool_calls"].is_array())
        .expect("the third tool turn is echoed for continuation");
    assert_eq!(latest_tool["reasoning_content"], tool_thoughts[2]);
    f.cleanup().await
}

#[tokio::test]
async fn earlier_entities_sit_right_before_the_question() -> anyhow::Result<()> {
    let Some(f) = fixture(Scripted::new(vec![
        Reply::Tool("no_evidence_needed", r#"{"reason":"follow-up"}"#),
        Reply::Text("Answered."),
    ]))
    .await?
    else {
        return Ok(());
    };
    // An earlier turn resolved an entity; the next turn sends its id as a system message
    let id = utopia_store::conversations::create(&f.pool, f.kb, f.user.id, "Acme").await?;
    utopia_store::conversations::append_message(
        &f.pool,
        id,
        "user",
        "Who founded Acme?",
        &utopia_store::conversations::TurnRecord::empty(),
    )
    .await?;
    utopia_store::conversations::append_message(
        &f.pool,
        id,
        "assistant",
        "Acme was founded by Ada.",
        &utopia_store::conversations::TurnRecord {
            stopped: false,
            steps: json!([]),
            sources: json!([]),
            resolved: json!([{ "id": Uuid::now_v7(), "name": "Acme", "type": "Organization" }]),
            tool_exchange: json!([]),
        },
    )
    .await?;
    let sse = ask_in(&f, id, "When was it founded?").await?;
    assert!(sse.contains("event: done"), "{sse}");

    let requests = f.requests();
    let first = &requests[0];
    let messages = first["messages"].as_array().expect("messages");
    let block = messages
        .iter()
        .position(|m| {
            m["role"] == "system"
                && m["content"]
                    .as_str()
                    .is_some_and(|c| c.starts_with("Entities already identified"))
        })
        .expect("the entities resolved earlier are sent");
    assert_eq!(messages[block + 1]["role"], "user", "{first}");
    assert_eq!(
        messages[block + 1]["content"],
        "When was it founded?",
        "{first}"
    );
    assert_eq!(user_turns(first, "When was it founded?"), 1, "{first}");
    assert_eq!(user_turns(first, "Who founded Acme?"), 1, "{first}");
    f.cleanup().await
}