Skip to content

armgit / zed

Public Imported from GitHub
0

Code at the speed of thought – Zed is a high-performance, multiplayer code editor from the creators of Atom and Tree-sitter.

ep: Send settled data to cloud (#56572)

Ben Kunkle4742e75bc9d8
diff --git a/crates/cloud_api_types/src/cloud_api_types.rs b/crates/cloud_api_types/src/cloud_api_types.rs
index 03c7c668a0f..8966d02d0de 100644
--- a/crates/cloud_api_types/src/cloud_api_types.rs
+++ b/crates/cloud_api_types/src/cloud_api_types.rs
@@ -117,3 +117,47 @@ pub struct SubmitEditPredictionFeedbackBody {
pub expected_output: Option<String>,
pub feedback: String,
}
+
+#[derive(Debug, PartialEq, Serialize, Deserialize)]
+pub struct SubmitEditPredictionSettledBody {
+ pub request_id: String,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub settled_editable_region: Option<String>,
+ pub ts_error_count_before_prediction: usize,
+ pub ts_error_count_after_prediction: usize,
+ pub can_collect_data: bool,
+ pub is_in_open_source_repo: bool,
+ #[serde(flatten)]
+ pub kept_chars: EditPredictionSettledKeptChars,
+ pub example: Option<serde_json::Value>,
+ pub model_version: Option<String>,
+ #[serde(rename = "e2e_latency")]
+ pub e2e_latency_ms: u128,
+}
+
+#[derive(Debug, PartialEq, Serialize, Deserialize)]
+pub struct SubmitEditPredictionSettledResponse {}
+
+#[derive(Debug, PartialEq, Serialize, Deserialize)]
+pub struct EditPredictionSettledKeptChars {
+ #[serde(rename = "edit_bytes_candidate_new")]
+ pub candidate_new: usize,
+ #[serde(rename = "edit_bytes_reference_new")]
+ pub reference_new: usize,
+ #[serde(rename = "edit_bytes_candidate_deleted")]
+ pub candidate_deleted: usize,
+ #[serde(rename = "edit_bytes_reference_deleted")]
+ pub reference_deleted: usize,
+ #[serde(rename = "edit_bytes_kept")]
+ pub kept: usize,
+ #[serde(rename = "edit_bytes_correctly_deleted")]
+ pub correctly_deleted: usize,
+ #[serde(rename = "edit_bytes_discarded")]
+ pub discarded: usize,
+ #[serde(rename = "edit_bytes_context")]
+ pub context: usize,
+ #[serde(rename = "edit_bytes_kept_rate")]
+ pub kept_rate: f64,
+ #[serde(rename = "edit_bytes_recall_rate")]
+ pub recall_rate: f64,
+}
diff --git a/crates/edit_prediction/src/edit_prediction.rs b/crates/edit_prediction/src/edit_prediction.rs
index b57c0227691..37964e484a5 100644
--- a/crates/edit_prediction/src/edit_prediction.rs
+++ b/crates/edit_prediction/src/edit_prediction.rs
@@ -1,7 +1,10 @@
use anyhow::Result;
use client::{Client, EditPredictionUsage, UserStore, global_llm_token};
use cloud_api_client::LlmApiToken;
-use cloud_api_types::{OrganizationId, SubmitEditPredictionFeedbackBody};
+use cloud_api_types::{
+ EditPredictionSettledKeptChars, OrganizationId, SubmitEditPredictionFeedbackBody,
+ SubmitEditPredictionSettledBody,
+};
use cloud_llm_client::predict_edits_v3::{
PREDICT_EDITS_MODE_HEADER_NAME, PREDICT_EDITS_REQUEST_ID_HEADER_NAME,
PREDICT_EDITS_TRIGGER_HEADER_NAME, PredictEditsMode, PredictEditsV3Request,
@@ -115,7 +118,7 @@ const COLLABORATOR_EDIT_LOCALITY_CONTEXT_TOKENS: usize = 512;
const LAST_CHANGE_GROUPING_TIME: Duration = Duration::from_secs(1);
const ZED_PREDICT_DATA_COLLECTION_CHOICE: &str = "zed_predict_data_collection_choice";
const REJECT_REQUEST_DEBOUNCE: Duration = Duration::from_secs(15);
-const EDIT_PREDICTION_SETTLED_EVENT: &str = "Edit Prediction Settled";
+
const EDIT_PREDICTION_SETTLED_TTL: Duration = Duration::from_secs(60 * 5);
const EDIT_PREDICTION_SETTLED_QUIESCENCE: Duration = Duration::from_secs(10);
@@ -496,7 +499,11 @@ struct PendingSettledPrediction {
predicted_editable_region: String,
ts_error_count_before_prediction: usize,
ts_error_count_after_prediction: usize,
+ organization_id: Option<OrganizationId>,
+ can_collect_data: bool,
+ is_in_open_source_repo: bool,
example: Option<ExampleSpec>,
+ model_version: Option<String>,
enqueued_at: Instant,
last_edit_at: Instant,
e2e_latency: std::time::Duration,
@@ -783,8 +790,21 @@ impl EditPredictionStore {
.detach();
let (settled_predictions_tx, settled_predictions_rx) = mpsc::unbounded();
- cx.spawn(async move |this, cx| {
- Self::run_settled_predictions_worker(this, settled_predictions_rx, cx).await;
+ cx.spawn({
+ let client = client.clone();
+ let llm_token = llm_token.clone();
+ let app_version = AppVersion::global(cx);
+ async move |this, cx| {
+ Self::run_settled_predictions_worker(
+ this,
+ settled_predictions_rx,
+ client,
+ llm_token,
+ app_version,
+ cx,
+ )
+ .await;
+ }
})
.detach();
@@ -1593,6 +1613,9 @@ impl EditPredictionStore {
async fn run_settled_predictions_worker(
this: WeakEntity<Self>,
mut rx: UnboundedReceiver<Instant>,
+ client: Arc<Client>,
+ llm_token: LlmApiToken,
+ app_version: Version,
cx: &mut AsyncApp,
) {
let mut next_wake_time: Option<Instant> = None;
@@ -1666,7 +1689,11 @@ impl EditPredictionStore {
predicted_editable_region,
ts_error_count_before_prediction,
ts_error_count_after_prediction,
+ organization_id,
+ can_collect_data,
+ is_in_open_source_repo,
example,
+ model_version,
e2e_latency,
..
} = pending_prediction;
@@ -1692,25 +1719,68 @@ impl EditPredictionStore {
});
}
- telemetry::event!(
- EDIT_PREDICTION_SETTLED_EVENT,
- request_id = request_id.0.clone(),
- settled_editable_region,
- ts_error_count_before_prediction,
- ts_error_count_after_prediction,
- edit_bytes_candidate_new = kept_rate_result.candidate_new_chars,
- edit_bytes_reference_new = kept_rate_result.reference_new_chars,
- edit_bytes_candidate_deleted = kept_rate_result.candidate_deleted_chars,
- edit_bytes_reference_deleted = kept_rate_result.reference_deleted_chars,
- edit_bytes_kept = kept_rate_result.kept_chars,
- edit_bytes_correctly_deleted = kept_rate_result.correctly_deleted_chars,
- edit_bytes_discarded = kept_rate_result.discarded_chars,
- edit_bytes_context = kept_rate_result.context_chars,
- edit_bytes_kept_rate = kept_rate_result.kept_rate,
- edit_bytes_recall_rate = kept_rate_result.recall_rate,
- example,
- e2e_latency = e2e_latency.as_millis(),
- );
+ cx.background_spawn({
+ let client = client.clone();
+ let llm_token = llm_token.clone();
+ let app_version = app_version.clone();
+ async move {
+ let result: anyhow::Result<()> = async {
+ let settled_editable_region =
+ can_collect_data.then_some(settled_editable_region);
+ let example = if can_collect_data {
+ example.map(serde_json::to_value).transpose()?
+ } else {
+ None
+ };
+
+ let body = SubmitEditPredictionSettledBody {
+ request_id: request_id.0.to_string(),
+ settled_editable_region,
+ ts_error_count_before_prediction,
+ ts_error_count_after_prediction,
+ can_collect_data,
+ is_in_open_source_repo,
+ kept_chars: EditPredictionSettledKeptChars {
+ candidate_new: kept_rate_result.candidate_new_chars,
+ reference_new: kept_rate_result.reference_new_chars,
+ candidate_deleted: kept_rate_result.candidate_deleted_chars,
+ reference_deleted: kept_rate_result.reference_deleted_chars,
+ kept: kept_rate_result.kept_chars,
+ correctly_deleted: kept_rate_result.correctly_deleted_chars,
+ discarded: kept_rate_result.discarded_chars,
+ context: kept_rate_result.context_chars,
+ kept_rate: kept_rate_result.kept_rate,
+ recall_rate: kept_rate_result.recall_rate,
+ },
+ example,
+ model_version,
+ e2e_latency_ms: e2e_latency.as_millis(),
+ };
+ let url = client
+ .http_client()
+ .build_zed_llm_url("/predict_edits/settled", &[])?;
+ Self::send_api_request::<serde_json::Value>(
+ |builder| {
+ Ok(builder
+ .uri(url.as_ref())
+ .body(serde_json::to_string(&body)?.into())?)
+ },
+ client,
+ llm_token,
+ organization_id,
+ app_version,
+ )
+ .await?;
+ Ok(())
+ }
+ .await;
+
+ if let Err(error) = result {
+ log::error!("failed to submit edit prediction settled: {error:?}");
+ }
+ }
+ })
+ .detach();
}
next_wake_time = oldest_edited_at.map(|time| time + EDIT_PREDICTION_SETTLED_QUIESCENCE);
@@ -1726,10 +1796,24 @@ impl EditPredictionStore {
editable_offset_range: Range<usize>,
edit_preview: &EditPreview,
example: Option<ExampleSpec>,
+ model_version: Option<String>,
e2e_latency: std::time::Duration,
cx: &mut Context<Self>,
) {
let this = &mut *self;
+ let is_in_open_source_repo = edited_buffer_snapshot
+ .file()
+ .map_or(false, |file| this.is_file_open_source(project, file, cx));
+ let can_collect_data = !cfg!(test)
+ && is_in_open_source_repo
+ && this.is_data_collection_enabled(cx)
+ && matches!(this.edit_prediction_model, EditPredictionModel::Zeta);
+
+ let organization_id = this
+ .user_store
+ .read(cx)
+ .current_organization()
+ .map(|organization| organization.id.clone());
let project_state = this.get_or_init_project(project, cx);
let Some(registered_buffer) = project_state
.registered_buffers
@@ -1770,7 +1854,11 @@ impl EditPredictionStore {
predicted_editable_region,
ts_error_count_before_prediction,
ts_error_count_after_prediction,
+ organization_id,
+ can_collect_data,
+ is_in_open_source_repo,
example,
+ model_version,
e2e_latency,
enqueued_at: now,
last_edit_at: now,
diff --git a/crates/edit_prediction/src/edit_prediction_tests.rs b/crates/edit_prediction/src/edit_prediction_tests.rs
index 7f0b690974e..21af57374e4 100644
--- a/crates/edit_prediction/src/edit_prediction_tests.rs
+++ b/crates/edit_prediction/src/edit_prediction_tests.rs
@@ -5,7 +5,8 @@ use clock::FakeSystemClock;
use clock::ReplicaId;
use cloud_api_types::{
CreateLlmTokenResponse, LlmToken, Organization, OrganizationConfiguration,
- OrganizationEditPredictionConfiguration, OrganizationId,
+ OrganizationEditPredictionConfiguration, OrganizationId, SubmitEditPredictionSettledBody,
+ SubmitEditPredictionSettledResponse,
};
use cloud_llm_client::{
EditPredictionRejectReason, EditPredictionRejection, RejectEditPredictionsBody,
@@ -2503,6 +2504,7 @@ struct RequestChannels {
oneshot::Sender<PredictEditsV3Response>,
)>,
reject: mpsc::UnboundedReceiver<(RejectEditPredictionsBody, oneshot::Sender<()>)>,
+ settled: mpsc::UnboundedReceiver<SubmitEditPredictionSettledBody>,
}
fn init_test_with_fake_client(
@@ -2534,6 +2536,7 @@ fn init_test_with_fake_client_and_legacy_data_collection(
let (predict_req_tx, predict_req_rx) = mpsc::unbounded();
let (reject_req_tx, reject_req_rx) = mpsc::unbounded();
+ let (settled_req_tx, settled_req_rx) = mpsc::unbounded();
let http_client = FakeHttpClient::create({
move |req| {
@@ -2541,6 +2544,7 @@ fn init_test_with_fake_client_and_legacy_data_collection(
let mut body = req.into_body();
let predict_req_tx = predict_req_tx.clone();
let reject_req_tx = reject_req_tx.clone();
+ let settled_req_tx = settled_req_tx.clone();
async move {
let resp = match uri.as_str() {
"/client/llm_tokens" => serde_json::to_string(&json!({
@@ -2566,6 +2570,13 @@ fn init_test_with_fake_client_and_legacy_data_collection(
reject_req_tx.unbounded_send((req, res_tx)).unwrap();
serde_json::to_string(&res_rx.await?).unwrap()
}
+ "/predict_edits/settled" => {
+ let mut buf = Vec::new();
+ body.read_to_end(&mut buf).await.ok();
+ let req = serde_json::from_slice(&buf).unwrap();
+ settled_req_tx.unbounded_send(req).unwrap();
+ serde_json::to_string(&SubmitEditPredictionSettledResponse {}).unwrap()
+ }
_ => {
panic!("Unexpected path: {}", uri)
}
@@ -2589,6 +2600,7 @@ fn init_test_with_fake_client_and_legacy_data_collection(
RequestChannels {
predict: predict_req_rx,
reject: reject_req_rx,
+ settled: settled_req_rx,
},
)
})
@@ -3416,6 +3428,7 @@ async fn test_edit_prediction_settled(cx: &mut TestAppContext) {
editable_region_a.clone(),
&edit_preview_a,
None,
+ None,
Duration::from_secs(0),
cx,
);
@@ -3482,6 +3495,7 @@ async fn test_edit_prediction_settled(cx: &mut TestAppContext) {
editable_region_b.clone(),
&edit_preview_b,
None,
+ None,
Duration::from_secs(0),
cx,
);
@@ -3531,6 +3545,84 @@ async fn test_edit_prediction_settled(cx: &mut TestAppContext) {
}
}
+#[gpui::test]
+async fn test_edit_prediction_settled_omits_body_when_data_collection_is_disabled(
+ cx: &mut TestAppContext,
+) {
+ let (ep_store, mut requests) = init_test_with_fake_client(cx);
+ let fs = FakeFs::new(cx.executor());
+ fs.insert_tree(
+ "/root",
+ json!({
+ "foo.md": "sensitive source\n"
+ }),
+ )
+ .await;
+ let project = Project::test(fs, vec![path!("/root").as_ref()], cx).await;
+ let buffer = project
+ .update(cx, |project, cx| {
+ let path = project.find_project_path(path!("root/foo.md"), cx).unwrap();
+ project.open_buffer(path, cx)
+ })
+ .await
+ .unwrap();
+
+ ep_store.update(cx, |ep_store, cx| {
+ ep_store.register_buffer(&buffer, &project, cx);
+ });
+
+ let snapshot = buffer.read_with(cx, |buffer, _cx| buffer.snapshot());
+ let edits: Arc<[(Range<Anchor>, Arc<str>)]> =
+ cx.update(|cx| to_completion_edits([(0..9, "replacement".into())], &buffer, cx).into());
+ let edit_preview = buffer
+ .read_with(cx, |buffer, cx| buffer.preview_edits(edits, cx))
+ .await;
+
+ ep_store.update(cx, |ep_store, cx| {
+ ep_store.enqueue_settled_prediction(
+ EditPredictionId("prediction-private".into()),
+ &project,
+ &buffer,
+ &snapshot,
+ 0..snapshot.len(),
+ &edit_preview,
+ Some(ExampleSpec {
+ name: "test example".to_string(),
+ repository_url: "https://example.com/repo".to_string(),
+ revision: "rev".to_string(),
+ tags: Vec::new(),
+ reasoning: None,
+ uncommitted_diff: String::new(),
+ cursor_path: Path::new("foo.md").into(),
+ cursor_position: "0".to_string(),
+ edit_history: "sensitive edit history".to_string(),
+ expected_patches: vec!["sensitive patch".to_string()],
+ rejected_patch: None,
+ telemetry: None,
+ human_feedback: Vec::new(),
+ rating: None,
+ }),
+ Some("test-model".to_string()),
+ Duration::from_millis(42),
+ cx,
+ );
+ });
+
+ cx.run_until_parked();
+ cx.executor()
+ .advance_clock(EDIT_PREDICTION_SETTLED_QUIESCENCE);
+ cx.run_until_parked();
+
+ let settled_request = requests
+ .settled
+ .next()
+ .await
+ .expect("settled request should be sent");
+ assert!(!settled_request.can_collect_data);
+ assert_eq!(settled_request.settled_editable_region, None);
+ assert_eq!(settled_request.example, None);
+}
+
#[gpui::test]
fn test_buffer_path_with_id_fallback_for_untitled_buffers(cx: &mut TestAppContext) {
let buffer_1 = cx.new(|cx| Buffer::local("one", cx));
diff --git a/crates/edit_prediction/src/zeta.rs b/crates/edit_prediction/src/zeta.rs
index f8eee2f079e..efa7552504e 100644
--- a/crates/edit_prediction/src/zeta.rs
+++ b/crates/edit_prediction/src/zeta.rs
@@ -411,6 +411,7 @@ pub fn request_prediction_with_zeta(
let edited_buffer_snapshot = edited_buffer_snapshot.clone();
let editable_range_in_buffer = editable_range_in_buffer.clone();
let edit_preview = prediction.edit_preview.clone();
+ let model_version = prediction.model_version.clone();
let example_task = capture_data.and_then(|stored_events| {
cx.update(|cx| {
crate::capture_example(
@@ -440,6 +441,7 @@ pub fn request_prediction_with_zeta(
editable_range_in_buffer,
&edit_preview,
example_spec,
+ model_version,
request_duration,
cx,
);