Skip to content

armgit / zed

Public Imported from GitHub
0

Code at the speed of thought – Zed is a high-performance, multiplayer code editor from the creators of Atom and Tree-sitter.

agent: Accumulate and persist cumulative token usage for threads (#58378)

Anant Goel27191913e908
diff --git a/crates/agent/src/tests/mod.rs b/crates/agent/src/tests/mod.rs
index f82c4c7dddf..758a4f51cb4 100644
--- a/crates/agent/src/tests/mod.rs
+++ b/crates/agent/src/tests/mod.rs
@@ -3206,6 +3206,146 @@ async fn test_latest_token_usage_counts_cached_input_tokens(cx: &mut TestAppCont
});
}
+#[gpui::test]
+async fn test_cumulative_token_usage(cx: &mut TestAppContext) {
+ let ThreadTest {
+ model,
+ thread,
+ project_context,
+ ..
+ } = setup(cx, TestModel::Fake).await;
+ let fake_model = model.as_fake();
+
+ thread
+ .update(cx, |thread, cx| {
+ thread.add_tool(EchoTool);
+ thread.send(UserMessageId::new(), ["Use the echo tool"], cx)
+ })
+ .unwrap();
+ cx.run_until_parked();
+
+ // The first request emits two cumulative snapshots; only the final values
+ // must be counted, exactly once.
+ fake_model.send_last_completion_stream_event(LanguageModelCompletionEvent::UsageUpdate(
+ TokenUsage {
+ input_tokens: 100,
+ output_tokens: 10,
+ ..Default::default()
+ },
+ ));
+ fake_model.send_last_completion_stream_event(LanguageModelCompletionEvent::UsageUpdate(
+ TokenUsage {
+ input_tokens: 100,
+ output_tokens: 50,
+ ..Default::default()
+ },
+ ));
+ fake_model.send_last_completion_stream_event(LanguageModelCompletionEvent::ToolUse(
+ LanguageModelToolUse {
+ id: "tool_1".into(),
+ name: EchoTool::NAME.into(),
+ raw_input: json!({"text": "hello"}).to_string(),
+ input: json!({"text": "hello"}),
+ is_input_complete: true,
+ thought_signature: None,
+ },
+ ));
+ fake_model.end_last_completion_stream();
+ cx.run_until_parked();
+
+ // The second request (after the tool call) is counted in addition to the first.
+ fake_model.send_last_completion_stream_text_chunk("Done");
+ fake_model.send_last_completion_stream_event(LanguageModelCompletionEvent::UsageUpdate(
+ TokenUsage {
+ input_tokens: 200,
+ output_tokens: 30,
+ ..Default::default()
+ },
+ ));
+ fake_model.end_last_completion_stream();
+ cx.run_until_parked();
+
+ let expected = TokenUsage {
+ input_tokens: 300,
+ output_tokens: 80,
+ ..Default::default()
+ };
+ thread.read_with(cx, |thread, _| {
+ assert_eq!(thread.cumulative_token_usage(), expected);
+ });
+
+ let db_thread = thread.read_with(cx, |thread, cx| thread.to_db(cx)).await;
+ assert_eq!(db_thread.cumulative_token_usage, expected);
+
+ cx.update(|cx| {
+ LanguageModelRegistry::test(cx);
+ });
+ let restored = cx.update(|cx| {
+ let thread = thread.read(cx);
+ let project = thread.project.clone();
+ let context_server_registry = thread.context_server_registry.clone();
+ let templates = thread.templates.clone();
+ cx.new(|cx| {
+ Thread::from_db(
+ acp::SessionId::new("restored"),
+ db_thread,
+ project,
+ project_context.clone(),
+ context_server_registry,
+ templates,
+ cx,
+ )
+ })
+ });
+ restored.read_with(cx, |thread, _| {
+ assert_eq!(thread.cumulative_token_usage(), expected);
+ });
+}
+
+#[gpui::test]
+async fn test_cumulative_token_usage_keeps_accounted_usage_monotonic(cx: &mut TestAppContext) {
+ let ThreadTest { model, thread, .. } = setup(cx, TestModel::Fake).await;
+ let fake_model = model.as_fake();
+
+ thread
+ .update(cx, |thread, cx| {
+ thread.send(UserMessageId::new(), ["hello"], cx)
+ })
+ .unwrap();
+ cx.run_until_parked();
+
+ fake_model.send_last_completion_stream_event(LanguageModelCompletionEvent::UsageUpdate(
+ TokenUsage {
+ input_tokens: 100,
+ output_tokens: 10,
+ ..Default::default()
+ },
+ ));
+ fake_model.send_last_completion_stream_event(LanguageModelCompletionEvent::UsageUpdate(
+ TokenUsage::default(),
+ ));
+ fake_model.send_last_completion_stream_event(LanguageModelCompletionEvent::UsageUpdate(
+ TokenUsage {
+ input_tokens: 100,
+ output_tokens: 50,
+ ..Default::default()
+ },
+ ));
+ fake_model.end_last_completion_stream();
+ cx.run_until_parked();
+
+ thread.read_with(cx, |thread, _| {
+ assert_eq!(
+ thread.cumulative_token_usage(),
+ TokenUsage {
+ input_tokens: 100,
+ output_tokens: 50,
+ ..Default::default()
+ }
+ );
+ });
+}
+
#[gpui::test]
async fn test_truncate_second_message(cx: &mut TestAppContext) {
let ThreadTest { model, thread, .. } = setup(cx, TestModel::Fake).await;
diff --git a/crates/agent/src/thread.rs b/crates/agent/src/thread.rs
index d742750b226..68b6bf03abe 100644
--- a/crates/agent/src/thread.rs
+++ b/crates/agent/src/thread.rs
@@ -1111,8 +1111,11 @@ pub struct Thread {
pending_message: Option<AgentMessage>,
pub(crate) tools: BTreeMap<SharedString, Arc<dyn AnyAgentTool>>,
request_token_usage: HashMap<UserMessageId, language_model::TokenUsage>,
- #[allow(unused)]
cumulative_token_usage: TokenUsage,
+ /// The per-field maximum usage snapshot already added to
+ /// `cumulative_token_usage` for the in-flight completion request. Reset at
+ /// the start of each request.
+ current_request_token_usage: TokenUsage,
#[allow(unused)]
initial_project_snapshot: Shared<Task<Option<Arc<ProjectSnapshot>>>>,
pub(crate) context_server_registry: Entity<ContextServerRegistry>,
@@ -1245,6 +1248,7 @@ impl Thread {
tools: BTreeMap::default(),
request_token_usage: HashMap::default(),
cumulative_token_usage: TokenUsage::default(),
+ current_request_token_usage: TokenUsage::default(),
initial_project_snapshot: {
let project_snapshot = Self::project_snapshot(project.clone(), cx);
cx.foreground_executor()
@@ -1611,6 +1615,7 @@ impl Thread {
tools: BTreeMap::default(),
request_token_usage: db_thread.request_token_usage.clone(),
cumulative_token_usage: db_thread.cumulative_token_usage,
+ current_request_token_usage: TokenUsage::default(),
initial_project_snapshot: Task::ready(db_thread.initial_project_snapshot).shared(),
context_server_registry,
profile_id,
@@ -1991,7 +1996,43 @@ impl Thread {
self.has_queued_message
}
+ fn accumulate_token_usage(&mut self, update: language_model::TokenUsage) {
+ let previous_accounted_usage = self.current_request_token_usage;
+ let current_accounted_usage = TokenUsage {
+ input_tokens: previous_accounted_usage
+ .input_tokens
+ .max(update.input_tokens),
+ output_tokens: previous_accounted_usage
+ .output_tokens
+ .max(update.output_tokens),
+ cache_creation_input_tokens: previous_accounted_usage
+ .cache_creation_input_tokens
+ .max(update.cache_creation_input_tokens),
+ cache_read_input_tokens: previous_accounted_usage
+ .cache_read_input_tokens
+ .max(update.cache_read_input_tokens),
+ };
+ self.current_request_token_usage = current_accounted_usage;
+ self.cumulative_token_usage = self.cumulative_token_usage
+ + TokenUsage {
+ input_tokens: current_accounted_usage
+ .input_tokens
+ .saturating_sub(previous_accounted_usage.input_tokens),
+ output_tokens: current_accounted_usage
+ .output_tokens
+ .saturating_sub(previous_accounted_usage.output_tokens),
+ cache_creation_input_tokens: current_accounted_usage
+ .cache_creation_input_tokens
+ .saturating_sub(previous_accounted_usage.cache_creation_input_tokens),
+ cache_read_input_tokens: current_accounted_usage
+ .cache_read_input_tokens
+ .saturating_sub(previous_accounted_usage.cache_read_input_tokens),
+ };
+ }
+
fn update_token_usage(&mut self, update: language_model::TokenUsage, cx: &mut Context<Self>) {
+ self.accumulate_token_usage(update);
+
let Some(last_user_message) = self.last_user_message() else {
return;
};
@@ -2032,6 +2073,10 @@ impl Thread {
Some(*tokens)
}
+ pub fn cumulative_token_usage(&self) -> language_model::TokenUsage {
+ self.cumulative_token_usage
+ }
+
pub fn latest_token_usage(&self) -> Option<acp_thread::TokenUsage> {
let usage = self.latest_request_token_usage()?;
let model = self.model.clone()?;
@@ -2301,6 +2346,7 @@ impl Thread {
.ok_or_else(|| anyhow!(NoModelConfiguredError))?;
this.refresh_turn_tools(cx);
let request = this.build_completion_request(intent, cx)?;
+ this.current_request_token_usage = TokenUsage::default();
anyhow::Ok((model, request))
})??;
@@ -2532,6 +2578,7 @@ impl Thread {
};
let model = this.model.clone()?;
let request = this.build_compaction_request(insertion_ix, &model, cx);
+ this.current_request_token_usage = TokenUsage::default();
Some((model, request, insertion_ix))
})?
else {
@@ -2575,10 +2622,14 @@ impl Thread {
summary.push_str(&text);
event_stream.send_context_compaction_update(compaction_id.clone(), &text);
}
+ LanguageModelCompletionEvent::UsageUpdate(usage) => {
+ this.update(cx, |this, _cx| {
+ this.accumulate_token_usage(usage);
+ })?;
+ }
LanguageModelCompletionEvent::Stop(_)
| LanguageModelCompletionEvent::Started
| LanguageModelCompletionEvent::Queued { .. }
- | LanguageModelCompletionEvent::UsageUpdate(_)
| LanguageModelCompletionEvent::Thinking { .. }
| LanguageModelCompletionEvent::RedactedThinking { .. }
| LanguageModelCompletionEvent::ReasoningDetails(_)
@@ -5646,6 +5697,108 @@ mod tests {
});
}
+ #[gpui::test]
+ async fn test_compaction_usage_counts_toward_cumulative_usage(cx: &mut TestAppContext) {
+ let (thread, _event_stream) = setup_thread_for_test(cx).await;
+ let model = Arc::new(FakeLanguageModel::default());
+ let old_user_message_id = UserMessageId::new();
+ let new_user_message_id = UserMessageId::new();
+ let prior_usage = TokenUsage {
+ input_tokens: 960_000,
+ output_tokens: 25,
+ ..Default::default()
+ };
+ let compaction_usage = TokenUsage {
+ input_tokens: 40,
+ output_tokens: 9,
+ cache_creation_input_tokens: 2,
+ cache_read_input_tokens: 3,
+ };
+ let final_usage = TokenUsage {
+ input_tokens: 500,
+ output_tokens: 50,
+ ..Default::default()
+ };
+
+ cx.update(|cx| {
+ cx.update_flags(true, vec!["handoff".to_string()]);
+ thread.update(cx, |thread, cx| {
+ thread.set_model(model.clone(), cx);
+ thread
+ .messages
+ .push(user_text_message(old_user_message_id.clone(), "old user"));
+ thread.messages.push(agent_text_message("old assistant"));
+ thread
+ .request_token_usage
+ .insert(old_user_message_id.clone(), prior_usage);
+ thread.cumulative_token_usage = prior_usage;
+ thread.current_request_token_usage = prior_usage;
+ });
+ });
+
+ let _events = cx
+ .update(|cx| {
+ thread.update(cx, |thread, cx| {
+ thread.send(new_user_message_id.clone(), vec!["new prompt"], cx)
+ })
+ })
+ .unwrap();
+ cx.run_until_parked();
+
+ let compaction_request = model.pending_completions().pop().unwrap();
+ assert_eq!(
+ compaction_request.intent,
+ Some(CompletionIntent::ThreadContextSummarization)
+ );
+
+ model.send_completion_stream_event(
+ &compaction_request,
+ LanguageModelCompletionEvent::UsageUpdate(TokenUsage {
+ input_tokens: 40,
+ output_tokens: 4,
+ ..Default::default()
+ }),
+ );
+ model.send_completion_stream_event(
+ &compaction_request,
+ LanguageModelCompletionEvent::UsageUpdate(compaction_usage),
+ );
+ model.send_completion_stream_text_chunk(&compaction_request, "compacted old context");
+ model.end_completion_stream(&compaction_request);
+ cx.run_until_parked();
+
+ let expected_after_compaction = prior_usage + compaction_usage;
+ thread.read_with(cx, |thread, _cx| {
+ assert_eq!(thread.cumulative_token_usage(), expected_after_compaction);
+ assert!(
+ !thread
+ .request_token_usage
+ .contains_key(&new_user_message_id)
+ );
+ });
+
+ let final_request = model.pending_completions().pop().unwrap();
+ assert_eq!(final_request.intent, Some(CompletionIntent::UserPrompt));
+
+ model.send_completion_stream_event(
+ &final_request,
+ LanguageModelCompletionEvent::UsageUpdate(final_usage),
+ );
+ model.end_completion_stream(&final_request);
+ cx.run_until_parked();
+
+ thread.read_with(cx, |thread, _cx| {
+ assert_eq!(
+ thread.cumulative_token_usage(),
+ expected_after_compaction + final_usage
+ );
+ assert_eq!(
+ thread.request_token_usage.get(&new_user_message_id),
+ Some(&final_usage)
+ );
+ });
+ }
+
#[gpui::test]
async fn test_replay_emits_context_compaction(cx: &mut TestAppContext) {
let (thread, _event_stream) = setup_thread_for_test(cx).await;