Code at the speed of thought – Zed is a high-performance, multiplayer code editor from the creators of Atom and Tree-sitter.
ep: Minor changes to evaluate context retrieval (#58572)
Oleksiy Syvokond1271beee788
diff --git a/crates/edit_prediction_cli/src/format_prompt.rs b/crates/edit_prediction_cli/src/format_prompt.rsindex 933560b476b..8007499315a 100644--- a/crates/edit_prediction_cli/src/format_prompt.rs+++ b/crates/edit_prediction_cli/src/format_prompt.rs@@ -34,7 +34,7 @@ pub async fn run_format_prompt(example,app_state.clone(),example_progress,- ContextRetrievalType::Lsp,+ vec![ContextRetrievalType::Lsp],false,cx.clone(),)diff --git a/crates/edit_prediction_cli/src/main.rs b/crates/edit_prediction_cli/src/main.rsindex c079c9291ab..78f8324432d 100644--- a/crates/edit_prediction_cli/src/main.rs+++ b/crates/edit_prediction_cli/src/main.rs@@ -129,14 +129,25 @@ pub enum FailedHandling {#[derive(Args, Debug, Clone)]struct ContextArgs {- /// Which context collector to run.- #[arg(long = "type", value_enum, default_value_t = ContextRetrievalType::Lsp)]- context_type: ContextRetrievalType,+ /// Which context collectors to run.+ /// May be repeated or comma-delimited, e.g. `--type=all,oracle-file`.+ #[arg(long = "type", value_enum, value_delimiter = ',')]+ context_types: Vec<ContextRetrievalType>,/// Recompute context even if the example already has related files.#[arg(long, short = 'f', default_value_t = false)]force: bool,}+impl ContextArgs {+ fn context_types(&self) -> Vec<ContextRetrievalType> {+ if self.context_types.is_empty() {+ vec![ContextRetrievalType::Lsp]+ } else {+ self.context_types.clone()+ }+ }+}+const INPUTS_HELP: &str = r#"Inputs can be file paths or special specifiers:@@ -252,7 +263,13 @@ impl Display for Command {Command::Read(_) => write!(f, "read"),Command::LoadProject => write!(f, "load-project"),Command::Context(args) => {- write!(f, "context --type={}", args.context_type)?;+ write!(f, "context --type=")?;+ for (index, context_type) in args.context_types().iter().enumerate() {+ if index > 0 {+ write!(f, ",")?;+ }+ write!(f, "{}", context_type)?;+ }if args.force {write!(f, " --force")?;}@@ -1306,7 +1323,7 @@ fn main() {example,app_state.clone(),&example_progress,- args.context_type,+ args.context_types(),args.force,cx.clone(),)@@ -1534,7 +1551,12 @@ fn main() {context_source_filter.as_deref(),);if let Some(summary_path) = &args.summary_json {- score::write_summary_json(&examples, summary_path)?;+ score::write_summary_json(+ &examples,+ summary_path,+ Some(args.related_context_limit * 3),+ context_source_filter.as_deref(),+ )?;}}Command::Repair(args) => {diff --git a/crates/edit_prediction_cli/src/predict.rs b/crates/edit_prediction_cli/src/predict.rsindex c21066aa812..5ba7efb5bfa 100644--- a/crates/edit_prediction_cli/src/predict.rs+++ b/crates/edit_prediction_cli/src/predict.rs@@ -72,7 +72,7 @@ pub async fn run_prediction(example,app_state.clone(),example_progress,- ContextRetrievalType::Lsp,+ vec![ContextRetrievalType::Lsp],false,cx.clone(),)@@ -123,7 +123,7 @@ pub async fn run_prediction(example,app_state.clone(),example_progress,- ContextRetrievalType::Lsp,+ vec![ContextRetrievalType::Lsp],false,cx.clone(),)diff --git a/crates/edit_prediction_cli/src/retrieve_context.rs b/crates/edit_prediction_cli/src/retrieve_context.rsindex 78953f8787a..93ce5ecb2c3 100644--- a/crates/edit_prediction_cli/src/retrieve_context.rs+++ b/crates/edit_prediction_cli/src/retrieve_context.rs@@ -70,23 +70,6 @@ impl ContextRetrievalType {ContextRetrievalType::None => Vec::new(),}}-- fn includes_lsp(self) -> bool {- self.context_sources().contains(&ContextSource::Lsp)- }-- fn editable_context_sources(self) -> Option<Vec<ContextSource>> {- let context_sources = self- .context_sources()- .into_iter()- .filter(|context_source| *context_source != ContextSource::Lsp)- .collect::<Vec<_>>();- if context_sources.is_empty() {- None- } else {- Some(context_sources)- }- }}pub fn context_sources_for_types(context_types: &[ContextRetrievalType]) -> Vec<ContextSource> {@@ -109,7 +92,6 @@ fn editable_context_sources() -> Vec<ContextSource> {ContextSource::EditHistoryFile,ContextSource::GitLog,ContextSource::Bm25,- ContextSource::OracleFile,]}@@ -117,7 +99,7 @@ pub async fn run_context_retrieval(example: &mut Example,app_state: Arc<EpAppState>,example_progress: &ExampleProgress,- context_type: ContextRetrievalType,+ context_types: Vec<ContextRetrievalType>,force: bool,mut cx: AsyncApp,) -> anyhow::Result<()> {@@ -143,8 +125,9 @@ pub async fn run_context_retrieval(.context("EditPredictionStore not initialized")?;let mut context_files = Vec::new();+ let context_sources = context_sources_for_types(&context_types);- if context_type.includes_lsp() {+ if context_sources.contains(&ContextSource::Lsp) {let _lsp_handle = project.update(&mut cx, |project, cx| {project.register_buffer_with_language_servers(&state.buffer, cx)});@@ -170,8 +153,12 @@ pub async fn run_context_retrieval(.extend(ep_store.update(&mut cx, |store, cx| store.context_for_project(&project, cx)));}- if let Some(context_sources) = context_type.editable_context_sources() {- let oracle_paths = if context_sources.contains(&ContextSource::OracleFile) {+ let editable_context_sources = context_sources+ .into_iter()+ .filter(|context_source| *context_source != ContextSource::Lsp)+ .collect::<Vec<_>>();+ if !editable_context_sources.is_empty() {+ let oracle_paths = if editable_context_sources.contains(&ContextSource::OracleFile) {let oracle_paths = oracle_paths_from_expected_patches(example);refresh_paths(&project, &oracle_paths, &mut cx).await?;oracle_paths@@ -186,7 +173,7 @@ pub async fn run_context_retrieval(state.buffer.clone(),state.cursor_position,oracle_paths,- context_sources,+ editable_context_sources,cx,)})diff --git a/crates/edit_prediction_cli/src/score.rs b/crates/edit_prediction_cli/src/score.rsindex 2a860df0fdb..841d15541e4 100644--- a/crates/edit_prediction_cli/src/score.rs+++ b/crates/edit_prediction_cli/src/score.rs@@ -891,8 +891,14 @@ fn truncate_name(name: &str, max_len: usize) -> String {pub type SummaryJson = edit_prediction_metrics::SummaryJson;-pub fn compute_summary(examples: &[Example]) -> SummaryJson {+pub fn compute_summary(+ examples: &[Example],+ retrieved_context_byte_limit: Option<usize>,+ context_source_filter: Option<&[ContextSource]>,+) -> SummaryJson {edit_prediction_metrics::compute_summary(examples.iter().flat_map(|example| {+ let retrieved_context_bytes =+ retrieved_context_bytes(example, retrieved_context_byte_limit, context_source_filter);example.score.iter()@@ -906,14 +912,30 @@ pub fn compute_summary(examples: &[Example]) -> SummaryJson {reverts_edits: qa.reverts_edits,confidence: qa.confidence,});-- edit_prediction_metrics::PredictionSummaryInput { score, qa }+ let retrieved_context_bytes = (score_idx == 0)+ .then_some(retrieved_context_bytes)+ .flatten();++ edit_prediction_metrics::PredictionSummaryInput {+ score,+ qa,+ retrieved_context_bytes,+ }})}))}-pub fn write_summary_json(examples: &[Example], path: &Path) -> anyhow::Result<()> {- let summary = compute_summary(examples);+pub fn write_summary_json(+ examples: &[Example],+ path: &Path,+ retrieved_context_byte_limit: Option<usize>,+ context_source_filter: Option<&[ContextSource]>,+) -> anyhow::Result<()> {+ let summary = compute_summary(+ examples,+ retrieved_context_byte_limit,+ context_source_filter,+ );let file = File::create(path).with_context(|| format!("Failed to create summary JSON file: {}", path.display()))?;let writer = BufWriter::new(file);@@ -922,3 +944,100 @@ pub fn write_summary_json(examples: &[Example], path: &Path) -> anyhow::Result<(eprintln!("Wrote summary JSON to: {}", path.display());Ok(())}++#[cfg(test)]+mod tests {+ use super::*;+ use edit_prediction::example_spec::ExampleSpec;+ use edit_prediction_metrics::PredictionScore;+ use std::path::Path;+ use zeta_prompt::{ExcerptRanges, RelatedExcerpt, ZetaPromptInput};++ #[test]+ fn summary_includes_limited_filtered_retrieved_context_bytes_once_per_example() {+ let examples = vec![+ example_with_related_files(+ Some(vec![RelatedFile {+ path: Path::new("project/src/lib.rs").into(),+ max_row: 10,+ excerpts: vec![+ related_excerpt("abcd", 0..1, 0, ContextSource::CurrentFile),+ related_excerpt("ignored by source filter", 1..2, 1, ContextSource::Lsp),+ related_excerpt("efghij", 2..3, 2, ContextSource::CurrentFile),+ ],+ in_open_source_repo: false,+ }]),+ 2,+ ),+ example_with_related_files(None, 1),+ ];++ let summary = compute_summary(&examples, Some(10), Some(&[ContextSource::CurrentFile]));++ assert_eq!(summary.total_examples, 3);+ assert_eq!(summary.avg_retrieved_context_bytes, Some(10.0));+ assert_eq!(summary.total_retrieved_context_bytes, Some(10));+ assert_eq!(summary.retrieved_context_examples, Some(1));+ }++ fn example_with_related_files(+ related_files: Option<Vec<RelatedFile>>,+ score_count: usize,+ ) -> Example {+ Example {+ spec: ExampleSpec {+ name: "example".to_string(),+ repository_url: "https://github.com/zed-industries/zed.git".to_string(),+ revision: "revision".to_string(),+ tags: Vec::new(),+ reasoning: None,+ uncommitted_diff: String::new(),+ recently_opened_files: Vec::new(),+ recently_viewed_files: Vec::new(),+ uncommitted_diff_contains_edit_history: false,+ cursor_path: Path::new("project/src/main.rs").into(),+ cursor_position: String::new(),+ edit_history: String::new(),+ expected_patches: Vec::new(),+ rejected_patch: None,+ telemetry: None,+ human_feedback: Vec::new(),+ rating: None,+ },+ prompt_inputs: Some(ZetaPromptInput {+ cursor_path: Path::new("project/src/main.rs").into(),+ cursor_excerpt: "".into(),+ cursor_offset_in_excerpt: 0,+ excerpt_start_row: None,+ events: Vec::new(),+ related_files,+ active_buffer_diagnostics: Vec::new(),+ excerpt_ranges: ExcerptRanges::default(),+ syntax_ranges: None,+ in_open_source_repo: false,+ can_collect_data: false,+ repo_url: None,+ }),+ prompt: None,+ predictions: Vec::new(),+ score: vec![PredictionScore::zero(); score_count],+ qa: Vec::new(),+ zed_version: None,+ state: None,+ }+ }++ fn related_excerpt(+ text: &str,+ row_range: std::ops::Range<u32>,+ order: usize,+ context_source: ContextSource,+ ) -> RelatedExcerpt {+ RelatedExcerpt {+ row_range,+ text: text.into(),+ order,+ context_source,+ }+ }+}diff --git a/crates/edit_prediction_metrics/src/summary.rs b/crates/edit_prediction_metrics/src/summary.rsindex 9c4028a300b..f927a027d76 100644--- a/crates/edit_prediction_metrics/src/summary.rs+++ b/crates/edit_prediction_metrics/src/summary.rs@@ -13,6 +13,7 @@ pub struct QaSummaryData {pub struct PredictionSummaryInput<'a> {pub score: &'a PredictionScore,pub qa: Option<QaSummaryData>,+ pub retrieved_context_bytes: Option<usize>,}#[derive(Clone, Debug, Serialize)]@@ -80,6 +81,12 @@ pub struct SummaryJson {pub editable_context_files_fp: Option<usize>,#[serde(skip_serializing_if = "Option::is_none")]pub editable_context_files_fn: Option<usize>,+ #[serde(skip_serializing_if = "Option::is_none")]+ pub avg_retrieved_context_bytes: Option<f64>,+ #[serde(skip_serializing_if = "Option::is_none")]+ pub total_retrieved_context_bytes: Option<usize>,+ #[serde(skip_serializing_if = "Option::is_none")]+ pub retrieved_context_examples: Option<usize>,}pub fn compute_summary<'a>(@@ -128,6 +135,8 @@ pub fn compute_summary<'a>(let mut editable_context_files_tp: usize = 0;let mut editable_context_files_fp: usize = 0;let mut editable_context_files_fn: usize = 0;+ let mut retrieved_context_bytes_total: usize = 0;+ let mut retrieved_context_bytes_count: usize = 0;for prediction in predictions {let score = prediction.score;@@ -186,6 +195,11 @@ pub fn compute_summary<'a>(recall_rate_sum += recall_rate;recall_rate_count += 1;}+ if let Some(retrieved_context_bytes) = prediction.retrieved_context_bytes {+ retrieved_context_bytes_total += retrieved_context_bytes;+ retrieved_context_bytes_count += 1;+ }+if let Some(coverage) = &score.editable_context_coverage {editable_context_lines_precision_sum += coverage.lines_precision;editable_context_lines_recall_sum += coverage.lines_recall;@@ -364,6 +378,21 @@ pub fn compute_summary<'a>(} else {None};+ let avg_retrieved_context_bytes = if retrieved_context_bytes_count > 0 {+ Some(retrieved_context_bytes_total as f64 / retrieved_context_bytes_count as f64)+ } else {+ None+ };+ let total_retrieved_context_bytes = if retrieved_context_bytes_count > 0 {+ Some(retrieved_context_bytes_total)+ } else {+ None+ };+ let retrieved_context_examples = if retrieved_context_bytes_count > 0 {+ Some(retrieved_context_bytes_count)+ } else {+ None+ };SummaryJson {total_examples: total_scores,@@ -414,5 +443,8 @@ pub fn compute_summary<'a>(editable_context_files_tp,editable_context_files_fp,editable_context_files_fn,+ avg_retrieved_context_bytes,+ total_retrieved_context_bytes,+ retrieved_context_examples,}}