Code at the speed of thought – Zed is a high-performance, multiplayer code editor from the creators of Atom and Tree-sitter.
ep: Send settled data to cloud (#56572)
Ben Kunkle4742e75bc9d8
diff --git a/crates/cloud_api_types/src/cloud_api_types.rs b/crates/cloud_api_types/src/cloud_api_types.rsindex 03c7c668a0f..8966d02d0de 100644--- a/crates/cloud_api_types/src/cloud_api_types.rs+++ b/crates/cloud_api_types/src/cloud_api_types.rs@@ -117,3 +117,47 @@ pub struct SubmitEditPredictionFeedbackBody {pub expected_output: Option<String>,pub feedback: String,}++#[derive(Debug, PartialEq, Serialize, Deserialize)]+pub struct SubmitEditPredictionSettledBody {+ pub request_id: String,+ #[serde(skip_serializing_if = "Option::is_none")]+ pub settled_editable_region: Option<String>,+ pub ts_error_count_before_prediction: usize,+ pub ts_error_count_after_prediction: usize,+ pub can_collect_data: bool,+ pub is_in_open_source_repo: bool,+ #[serde(flatten)]+ pub kept_chars: EditPredictionSettledKeptChars,+ pub example: Option<serde_json::Value>,+ pub model_version: Option<String>,+ #[serde(rename = "e2e_latency")]+ pub e2e_latency_ms: u128,+}++#[derive(Debug, PartialEq, Serialize, Deserialize)]+pub struct SubmitEditPredictionSettledResponse {}++#[derive(Debug, PartialEq, Serialize, Deserialize)]+pub struct EditPredictionSettledKeptChars {+ #[serde(rename = "edit_bytes_candidate_new")]+ pub candidate_new: usize,+ #[serde(rename = "edit_bytes_reference_new")]+ pub reference_new: usize,+ #[serde(rename = "edit_bytes_candidate_deleted")]+ pub candidate_deleted: usize,+ #[serde(rename = "edit_bytes_reference_deleted")]+ pub reference_deleted: usize,+ #[serde(rename = "edit_bytes_kept")]+ pub kept: usize,+ #[serde(rename = "edit_bytes_correctly_deleted")]+ pub correctly_deleted: usize,+ #[serde(rename = "edit_bytes_discarded")]+ pub discarded: usize,+ #[serde(rename = "edit_bytes_context")]+ pub context: usize,+ #[serde(rename = "edit_bytes_kept_rate")]+ pub kept_rate: f64,+ #[serde(rename = "edit_bytes_recall_rate")]+ pub recall_rate: f64,+}diff --git a/crates/edit_prediction/src/edit_prediction.rs b/crates/edit_prediction/src/edit_prediction.rsindex b57c0227691..37964e484a5 100644--- a/crates/edit_prediction/src/edit_prediction.rs+++ b/crates/edit_prediction/src/edit_prediction.rs@@ -1,7 +1,10 @@use anyhow::Result;use client::{Client, EditPredictionUsage, UserStore, global_llm_token};use cloud_api_client::LlmApiToken;-use cloud_api_types::{OrganizationId, SubmitEditPredictionFeedbackBody};+use cloud_api_types::{+ EditPredictionSettledKeptChars, OrganizationId, SubmitEditPredictionFeedbackBody,+ SubmitEditPredictionSettledBody,+};use cloud_llm_client::predict_edits_v3::{PREDICT_EDITS_MODE_HEADER_NAME, PREDICT_EDITS_REQUEST_ID_HEADER_NAME,PREDICT_EDITS_TRIGGER_HEADER_NAME, PredictEditsMode, PredictEditsV3Request,@@ -115,7 +118,7 @@ const COLLABORATOR_EDIT_LOCALITY_CONTEXT_TOKENS: usize = 512;const LAST_CHANGE_GROUPING_TIME: Duration = Duration::from_secs(1);const ZED_PREDICT_DATA_COLLECTION_CHOICE: &str = "zed_predict_data_collection_choice";const REJECT_REQUEST_DEBOUNCE: Duration = Duration::from_secs(15);-const EDIT_PREDICTION_SETTLED_EVENT: &str = "Edit Prediction Settled";+const EDIT_PREDICTION_SETTLED_TTL: Duration = Duration::from_secs(60 * 5);const EDIT_PREDICTION_SETTLED_QUIESCENCE: Duration = Duration::from_secs(10);@@ -496,7 +499,11 @@ struct PendingSettledPrediction {predicted_editable_region: String,ts_error_count_before_prediction: usize,ts_error_count_after_prediction: usize,+ organization_id: Option<OrganizationId>,+ can_collect_data: bool,+ is_in_open_source_repo: bool,example: Option<ExampleSpec>,+ model_version: Option<String>,enqueued_at: Instant,last_edit_at: Instant,e2e_latency: std::time::Duration,@@ -783,8 +790,21 @@ impl EditPredictionStore {.detach();let (settled_predictions_tx, settled_predictions_rx) = mpsc::unbounded();- cx.spawn(async move |this, cx| {- Self::run_settled_predictions_worker(this, settled_predictions_rx, cx).await;+ cx.spawn({+ let client = client.clone();+ let llm_token = llm_token.clone();+ let app_version = AppVersion::global(cx);+ async move |this, cx| {+ Self::run_settled_predictions_worker(+ this,+ settled_predictions_rx,+ client,+ llm_token,+ app_version,+ cx,+ )+ .await;+ }}).detach();@@ -1593,6 +1613,9 @@ impl EditPredictionStore {async fn run_settled_predictions_worker(this: WeakEntity<Self>,mut rx: UnboundedReceiver<Instant>,+ client: Arc<Client>,+ llm_token: LlmApiToken,+ app_version: Version,cx: &mut AsyncApp,) {let mut next_wake_time: Option<Instant> = None;@@ -1666,7 +1689,11 @@ impl EditPredictionStore {predicted_editable_region,ts_error_count_before_prediction,ts_error_count_after_prediction,+ organization_id,+ can_collect_data,+ is_in_open_source_repo,example,+ model_version,e2e_latency,..} = pending_prediction;@@ -1692,25 +1719,68 @@ impl EditPredictionStore {});}- telemetry::event!(- EDIT_PREDICTION_SETTLED_EVENT,- request_id = request_id.0.clone(),- settled_editable_region,- ts_error_count_before_prediction,- ts_error_count_after_prediction,- edit_bytes_candidate_new = kept_rate_result.candidate_new_chars,- edit_bytes_reference_new = kept_rate_result.reference_new_chars,- edit_bytes_candidate_deleted = kept_rate_result.candidate_deleted_chars,- edit_bytes_reference_deleted = kept_rate_result.reference_deleted_chars,- edit_bytes_kept = kept_rate_result.kept_chars,- edit_bytes_correctly_deleted = kept_rate_result.correctly_deleted_chars,- edit_bytes_discarded = kept_rate_result.discarded_chars,- edit_bytes_context = kept_rate_result.context_chars,- edit_bytes_kept_rate = kept_rate_result.kept_rate,- edit_bytes_recall_rate = kept_rate_result.recall_rate,- example,- e2e_latency = e2e_latency.as_millis(),- );+ cx.background_spawn({+ let client = client.clone();+ let llm_token = llm_token.clone();+ let app_version = app_version.clone();+ async move {+ let result: anyhow::Result<()> = async {+ let settled_editable_region =+ can_collect_data.then_some(settled_editable_region);+ let example = if can_collect_data {+ example.map(serde_json::to_value).transpose()?+ } else {+ None+ };++ let body = SubmitEditPredictionSettledBody {+ request_id: request_id.0.to_string(),+ settled_editable_region,+ ts_error_count_before_prediction,+ ts_error_count_after_prediction,+ can_collect_data,+ is_in_open_source_repo,+ kept_chars: EditPredictionSettledKeptChars {+ candidate_new: kept_rate_result.candidate_new_chars,+ reference_new: kept_rate_result.reference_new_chars,+ candidate_deleted: kept_rate_result.candidate_deleted_chars,+ reference_deleted: kept_rate_result.reference_deleted_chars,+ kept: kept_rate_result.kept_chars,+ correctly_deleted: kept_rate_result.correctly_deleted_chars,+ discarded: kept_rate_result.discarded_chars,+ context: kept_rate_result.context_chars,+ kept_rate: kept_rate_result.kept_rate,+ recall_rate: kept_rate_result.recall_rate,+ },+ example,+ model_version,+ e2e_latency_ms: e2e_latency.as_millis(),+ };+ let url = client+ .http_client()+ .build_zed_llm_url("/predict_edits/settled", &[])?;+ Self::send_api_request::<serde_json::Value>(+ |builder| {+ Ok(builder+ .uri(url.as_ref())+ .body(serde_json::to_string(&body)?.into())?)+ },+ client,+ llm_token,+ organization_id,+ app_version,+ )+ .await?;+ Ok(())+ }+ .await;++ if let Err(error) = result {+ log::error!("failed to submit edit prediction settled: {error:?}");+ }+ }+ })+ .detach();}next_wake_time = oldest_edited_at.map(|time| time + EDIT_PREDICTION_SETTLED_QUIESCENCE);@@ -1726,10 +1796,24 @@ impl EditPredictionStore {editable_offset_range: Range<usize>,edit_preview: &EditPreview,example: Option<ExampleSpec>,+ model_version: Option<String>,e2e_latency: std::time::Duration,cx: &mut Context<Self>,) {let this = &mut *self;+ let is_in_open_source_repo = edited_buffer_snapshot+ .file()+ .map_or(false, |file| this.is_file_open_source(project, file, cx));+ let can_collect_data = !cfg!(test)+ && is_in_open_source_repo+ && this.is_data_collection_enabled(cx)+ && matches!(this.edit_prediction_model, EditPredictionModel::Zeta);++ let organization_id = this+ .user_store+ .read(cx)+ .current_organization()+ .map(|organization| organization.id.clone());let project_state = this.get_or_init_project(project, cx);let Some(registered_buffer) = project_state.registered_buffers@@ -1770,7 +1854,11 @@ impl EditPredictionStore {predicted_editable_region,ts_error_count_before_prediction,ts_error_count_after_prediction,+ organization_id,+ can_collect_data,+ is_in_open_source_repo,example,+ model_version,e2e_latency,enqueued_at: now,last_edit_at: now,diff --git a/crates/edit_prediction/src/edit_prediction_tests.rs b/crates/edit_prediction/src/edit_prediction_tests.rsindex 7f0b690974e..21af57374e4 100644--- a/crates/edit_prediction/src/edit_prediction_tests.rs+++ b/crates/edit_prediction/src/edit_prediction_tests.rs@@ -5,7 +5,8 @@ use clock::FakeSystemClock;use clock::ReplicaId;use cloud_api_types::{CreateLlmTokenResponse, LlmToken, Organization, OrganizationConfiguration,- OrganizationEditPredictionConfiguration, OrganizationId,+ OrganizationEditPredictionConfiguration, OrganizationId, SubmitEditPredictionSettledBody,+ SubmitEditPredictionSettledResponse,};use cloud_llm_client::{EditPredictionRejectReason, EditPredictionRejection, RejectEditPredictionsBody,@@ -2503,6 +2504,7 @@ struct RequestChannels {oneshot::Sender<PredictEditsV3Response>,)>,reject: mpsc::UnboundedReceiver<(RejectEditPredictionsBody, oneshot::Sender<()>)>,+ settled: mpsc::UnboundedReceiver<SubmitEditPredictionSettledBody>,}fn init_test_with_fake_client(@@ -2534,6 +2536,7 @@ fn init_test_with_fake_client_and_legacy_data_collection(let (predict_req_tx, predict_req_rx) = mpsc::unbounded();let (reject_req_tx, reject_req_rx) = mpsc::unbounded();+ let (settled_req_tx, settled_req_rx) = mpsc::unbounded();let http_client = FakeHttpClient::create({move |req| {@@ -2541,6 +2544,7 @@ fn init_test_with_fake_client_and_legacy_data_collection(let mut body = req.into_body();let predict_req_tx = predict_req_tx.clone();let reject_req_tx = reject_req_tx.clone();+ let settled_req_tx = settled_req_tx.clone();async move {let resp = match uri.as_str() {"/client/llm_tokens" => serde_json::to_string(&json!({@@ -2566,6 +2570,13 @@ fn init_test_with_fake_client_and_legacy_data_collection(reject_req_tx.unbounded_send((req, res_tx)).unwrap();serde_json::to_string(&res_rx.await?).unwrap()}+ "/predict_edits/settled" => {+ let mut buf = Vec::new();+ body.read_to_end(&mut buf).await.ok();+ let req = serde_json::from_slice(&buf).unwrap();+ settled_req_tx.unbounded_send(req).unwrap();+ serde_json::to_string(&SubmitEditPredictionSettledResponse {}).unwrap()+ }_ => {panic!("Unexpected path: {}", uri)}@@ -2589,6 +2600,7 @@ fn init_test_with_fake_client_and_legacy_data_collection(RequestChannels {predict: predict_req_rx,reject: reject_req_rx,+ settled: settled_req_rx,},)})@@ -3416,6 +3428,7 @@ async fn test_edit_prediction_settled(cx: &mut TestAppContext) {editable_region_a.clone(),&edit_preview_a,None,+ None,Duration::from_secs(0),cx,);@@ -3482,6 +3495,7 @@ async fn test_edit_prediction_settled(cx: &mut TestAppContext) {editable_region_b.clone(),&edit_preview_b,None,+ None,Duration::from_secs(0),cx,);@@ -3531,6 +3545,84 @@ async fn test_edit_prediction_settled(cx: &mut TestAppContext) {}}+#[gpui::test]+async fn test_edit_prediction_settled_omits_body_when_data_collection_is_disabled(+ cx: &mut TestAppContext,+) {+ let (ep_store, mut requests) = init_test_with_fake_client(cx);+ let fs = FakeFs::new(cx.executor());+ fs.insert_tree(+ "/root",+ json!({+ "foo.md": "sensitive source\n"+ }),+ )+ .await;+ let project = Project::test(fs, vec![path!("/root").as_ref()], cx).await;+ let buffer = project+ .update(cx, |project, cx| {+ let path = project.find_project_path(path!("root/foo.md"), cx).unwrap();+ project.open_buffer(path, cx)+ })+ .await+ .unwrap();++ ep_store.update(cx, |ep_store, cx| {+ ep_store.register_buffer(&buffer, &project, cx);+ });++ let snapshot = buffer.read_with(cx, |buffer, _cx| buffer.snapshot());+ let edits: Arc<[(Range<Anchor>, Arc<str>)]> =+ cx.update(|cx| to_completion_edits([(0..9, "replacement".into())], &buffer, cx).into());+ let edit_preview = buffer+ .read_with(cx, |buffer, cx| buffer.preview_edits(edits, cx))+ .await;++ ep_store.update(cx, |ep_store, cx| {+ ep_store.enqueue_settled_prediction(+ EditPredictionId("prediction-private".into()),+ &project,+ &buffer,+ &snapshot,+ 0..snapshot.len(),+ &edit_preview,+ Some(ExampleSpec {+ name: "test example".to_string(),+ repository_url: "https://example.com/repo".to_string(),+ revision: "rev".to_string(),+ tags: Vec::new(),+ reasoning: None,+ uncommitted_diff: String::new(),+ cursor_path: Path::new("foo.md").into(),+ cursor_position: "0".to_string(),+ edit_history: "sensitive edit history".to_string(),+ expected_patches: vec!["sensitive patch".to_string()],+ rejected_patch: None,+ telemetry: None,+ human_feedback: Vec::new(),+ rating: None,+ }),+ Some("test-model".to_string()),+ Duration::from_millis(42),+ cx,+ );+ });++ cx.run_until_parked();+ cx.executor()+ .advance_clock(EDIT_PREDICTION_SETTLED_QUIESCENCE);+ cx.run_until_parked();++ let settled_request = requests+ .settled+ .next()+ .await+ .expect("settled request should be sent");+ assert!(!settled_request.can_collect_data);+ assert_eq!(settled_request.settled_editable_region, None);+ assert_eq!(settled_request.example, None);+}+#[gpui::test]fn test_buffer_path_with_id_fallback_for_untitled_buffers(cx: &mut TestAppContext) {let buffer_1 = cx.new(|cx| Buffer::local("one", cx));diff --git a/crates/edit_prediction/src/zeta.rs b/crates/edit_prediction/src/zeta.rsindex f8eee2f079e..efa7552504e 100644--- a/crates/edit_prediction/src/zeta.rs+++ b/crates/edit_prediction/src/zeta.rs@@ -411,6 +411,7 @@ pub fn request_prediction_with_zeta(let edited_buffer_snapshot = edited_buffer_snapshot.clone();let editable_range_in_buffer = editable_range_in_buffer.clone();let edit_preview = prediction.edit_preview.clone();+ let model_version = prediction.model_version.clone();let example_task = capture_data.and_then(|stored_events| {cx.update(|cx| {crate::capture_example(@@ -440,6 +441,7 @@ pub fn request_prediction_with_zeta(editable_range_in_buffer,&edit_preview,example_spec,+ model_version,request_duration,cx,);