Skip to content

Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors

Rust

fix(evals): replay each question's sessions under their own session id

AkitaOnRails7e08baa74bca
diff --git a/evals/src/retrieval/grep.rs b/evals/src/retrieval/grep.rs
index 21a8269e..6154f0ac 100644
--- a/evals/src/retrieval/grep.rs
+++ b/evals/src/retrieval/grep.rs
@@ -70,7 +70,7 @@ pub fn rank_question(q: &Question) -> Vec<uuid::Uuid> {
.filter(|((id, _), _)| seen.insert(id.as_str()))
.map(|((id, date), turns)| {
(
- stored_session_uuid(id),
+ stored_session_uuid(q, id),
session_text(date, turns).to_lowercase(),
)
})
@@ -160,7 +160,10 @@ mod tests {
let ranked = rank_question(&q);
assert_eq!(
ranked,
- [stored_session_uuid("s_a"), stored_session_uuid("s_b")]
+ [
+ stored_session_uuid(&q, "s_a"),
+ stored_session_uuid(&q, "s_b")
+ ]
);
}
}
diff --git a/evals/src/retrieval/ingest.rs b/evals/src/retrieval/ingest.rs
index 9bf55120..8faa1c29 100644
--- a/evals/src/retrieval/ingest.rs
+++ b/evals/src/retrieval/ingest.rs
@@ -111,14 +111,22 @@ pub fn project_for(q: &Question) -> String {
format!("lme-{}", q.question_id)
}
-/// The store hashes non-UUID session ids to UUIDv5(NAMESPACE_OID, raw)
-/// (`resolve_native_session_id`); the scorer recomputes the same mapping
-/// to attribute `sessions/<uuid>.md` page hits back to dataset sessions.
-pub fn stored_session_uuid(raw: &str) -> uuid::Uuid {
- match uuid::Uuid::parse_str(raw) {
- Ok(u) => u,
- Err(_) => uuid::Uuid::new_v5(&uuid::Uuid::NAMESPACE_OID, raw.as_bytes()),
- }
+/// The stored session id for dataset session `raw` in `q`'s haystack.
+///
+/// LongMemEval reuses session ids across questions: 4,715 of the 23,566
+/// scored haystack sessions (8 of them evidence) also sit in an earlier
+/// question's haystack. Replayed under its raw id, such a session already
+/// exists in the other question's project when the second replay arrives;
+/// the store files its turns under the new project (follow-cwd routing)
+/// but drops its `session-end` as naming a foreign scope, so the second
+/// project never gets the session page. A real harness mints a fresh id
+/// per session, so the replay namespaces the id by project and sends the
+/// resulting UUID, which the store keeps verbatim.
+pub fn stored_session_uuid(q: &Question, raw: &str) -> uuid::Uuid {
+ uuid::Uuid::new_v5(
+ &uuid::Uuid::NAMESPACE_OID,
+ format!("{}\n{raw}", project_for(q)).as_bytes(),
+ )
}
/// Truncate on a char boundary so the marker stays valid UTF-8.
@@ -212,7 +220,7 @@ fn build_items(q: &Question, prefix: DatePrefix) -> Vec<BatchItem> {
};
for (idx, session) in q.haystack_sessions.iter().enumerate() {
- let sid = &q.haystack_session_ids[idx];
+ let sid = &stored_session_uuid(q, &q.haystack_session_ids[idx]).to_string();
let date = &q.haystack_dates[idx];
let mut start = json!({ "hook_event_name": "SessionStart", "source": "startup" });
if prefix == DatePrefix::SessionStart {
@@ -363,7 +371,7 @@ pub async fn write_session_pages(
let request = json!({
"workspace": EVAL_WORKSPACE,
"project": project,
- "path": format!("sessions/{}.md", stored_session_uuid(sid)),
+ "path": format!("sessions/{}.md", stored_session_uuid(q, sid)),
"title": session_page_title(date),
"body": body,
// Hook-synthesised session pages are episodic too.
@@ -455,15 +463,23 @@ mod tests {
}
#[test]
- fn non_uuid_session_ids_map_to_the_stores_v5_hash() {
- // Mirrors resolve_native_session_id in the hooks router.
- let mapped = stored_session_uuid("sharegpt_abc_0");
- assert_eq!(
- mapped,
- uuid::Uuid::new_v5(&uuid::Uuid::NAMESPACE_OID, b"sharegpt_abc_0")
- );
- let real = uuid::Uuid::now_v7();
- assert_eq!(stored_session_uuid(&real.to_string()), real);
+ fn replayed_session_ids_are_per_question_uuids() {
+ let first = q();
+ let mut second = q();
+ second.question_id = "qY".into();
+ let a = stored_session_uuid(&first, "sharegpt_abc_0");
+ let b = stored_session_uuid(&second, "sharegpt_abc_0");
+ // A session shared by two haystacks must not share a stored id.
+ assert_ne!(a, b);
+ assert_eq!(a, stored_session_uuid(&first, "sharegpt_abc_0"));
+ // The replay sends the UUID itself, which the store keeps verbatim.
+ for item in build_items(&first, DatePrefix::Turn) {
+ assert_eq!(
+ item.body["session_id"].as_str(),
+ Some(a.to_string().as_str())
+ );
+ assert!(item.url.contains(&format!("session_id={a}")));
+ }
}
#[test]
diff --git a/evals/src/retrieval/mod.rs b/evals/src/retrieval/mod.rs
index b0b6f74c..52bdbee1 100644
--- a/evals/src/retrieval/mod.rs
+++ b/evals/src/retrieval/mod.rs
@@ -684,7 +684,7 @@ async fn run_config(run: &RunContext<'_>, knobs: &Knobs) -> Result<report::Repor
let evidence: HashSet<uuid::Uuid> = q
.answer_session_ids
.iter()
- .map(|s| stored_session_uuid(s))
+ .map(|s| stored_session_uuid(&q, s))
.collect();
let mut score = score::score_question(
&q.question_id,
@@ -833,7 +833,7 @@ fn run_grep(
let evidence: HashSet<uuid::Uuid> = q
.answer_session_ids
.iter()
- .map(|s| stored_session_uuid(s))
+ .map(|s| stored_session_uuid(q, s))
.collect();
score::score_question(
&q.question_id,
diff --git a/evals/src/retrieval/reachability.rs b/evals/src/retrieval/reachability.rs
index 1f059e3d..def87dfe 100644
--- a/evals/src/retrieval/reachability.rs
+++ b/evals/src/retrieval/reachability.rs
@@ -414,7 +414,7 @@ pub async fn measure_question(
.iter()
.position(|s| *s == t.session_id)
.context("evidence session not in haystack")?;
- let uuid = stored_session_uuid(&t.session_id);
+ let uuid = stored_session_uuid(q, &t.session_id);
let mut s = Surfaces::default();
if let Some((title, body)) = read_page(client, base_url, &project, uuid).await? {
s.page_present = true;