Skip to content

armgit / zed

Public Imported from GitHub
0

Code at the speed of thought – Zed is a high-performance, multiplayer code editor from the creators of Atom and Tree-sitter.

ep: Minor changes to evaluate context retrieval (#58572)

Oleksiy Syvokond1271beee788
diff --git a/crates/edit_prediction_cli/src/format_prompt.rs b/crates/edit_prediction_cli/src/format_prompt.rs
index 933560b476b..8007499315a 100644
--- a/crates/edit_prediction_cli/src/format_prompt.rs
+++ b/crates/edit_prediction_cli/src/format_prompt.rs
@@ -34,7 +34,7 @@ pub async fn run_format_prompt(
example,
app_state.clone(),
example_progress,
- ContextRetrievalType::Lsp,
+ vec![ContextRetrievalType::Lsp],
false,
cx.clone(),
)
diff --git a/crates/edit_prediction_cli/src/main.rs b/crates/edit_prediction_cli/src/main.rs
index c079c9291ab..78f8324432d 100644
--- a/crates/edit_prediction_cli/src/main.rs
+++ b/crates/edit_prediction_cli/src/main.rs
@@ -129,14 +129,25 @@ pub enum FailedHandling {
#[derive(Args, Debug, Clone)]
struct ContextArgs {
- /// Which context collector to run.
- #[arg(long = "type", value_enum, default_value_t = ContextRetrievalType::Lsp)]
- context_type: ContextRetrievalType,
+ /// Which context collectors to run.
+ /// May be repeated or comma-delimited, e.g. `--type=all,oracle-file`.
+ #[arg(long = "type", value_enum, value_delimiter = ',')]
+ context_types: Vec<ContextRetrievalType>,
/// Recompute context even if the example already has related files.
#[arg(long, short = 'f', default_value_t = false)]
force: bool,
}
+impl ContextArgs {
+ fn context_types(&self) -> Vec<ContextRetrievalType> {
+ if self.context_types.is_empty() {
+ vec![ContextRetrievalType::Lsp]
+ } else {
+ self.context_types.clone()
+ }
+ }
+}
+
const INPUTS_HELP: &str = r#"
Inputs can be file paths or special specifiers:
@@ -252,7 +263,13 @@ impl Display for Command {
Command::Read(_) => write!(f, "read"),
Command::LoadProject => write!(f, "load-project"),
Command::Context(args) => {
- write!(f, "context --type={}", args.context_type)?;
+ write!(f, "context --type=")?;
+ for (index, context_type) in args.context_types().iter().enumerate() {
+ if index > 0 {
+ write!(f, ",")?;
+ }
+ write!(f, "{}", context_type)?;
+ }
if args.force {
write!(f, " --force")?;
}
@@ -1306,7 +1323,7 @@ fn main() {
example,
app_state.clone(),
&example_progress,
- args.context_type,
+ args.context_types(),
args.force,
cx.clone(),
)
@@ -1534,7 +1551,12 @@ fn main() {
context_source_filter.as_deref(),
);
if let Some(summary_path) = &args.summary_json {
- score::write_summary_json(&examples, summary_path)?;
+ score::write_summary_json(
+ &examples,
+ summary_path,
+ Some(args.related_context_limit * 3),
+ context_source_filter.as_deref(),
+ )?;
}
}
Command::Repair(args) => {
diff --git a/crates/edit_prediction_cli/src/predict.rs b/crates/edit_prediction_cli/src/predict.rs
index c21066aa812..5ba7efb5bfa 100644
--- a/crates/edit_prediction_cli/src/predict.rs
+++ b/crates/edit_prediction_cli/src/predict.rs
@@ -72,7 +72,7 @@ pub async fn run_prediction(
example,
app_state.clone(),
example_progress,
- ContextRetrievalType::Lsp,
+ vec![ContextRetrievalType::Lsp],
false,
cx.clone(),
)
@@ -123,7 +123,7 @@ pub async fn run_prediction(
example,
app_state.clone(),
example_progress,
- ContextRetrievalType::Lsp,
+ vec![ContextRetrievalType::Lsp],
false,
cx.clone(),
)
diff --git a/crates/edit_prediction_cli/src/retrieve_context.rs b/crates/edit_prediction_cli/src/retrieve_context.rs
index 78953f8787a..93ce5ecb2c3 100644
--- a/crates/edit_prediction_cli/src/retrieve_context.rs
+++ b/crates/edit_prediction_cli/src/retrieve_context.rs
@@ -70,23 +70,6 @@ impl ContextRetrievalType {
ContextRetrievalType::None => Vec::new(),
}
}
-
- fn includes_lsp(self) -> bool {
- self.context_sources().contains(&ContextSource::Lsp)
- }
-
- fn editable_context_sources(self) -> Option<Vec<ContextSource>> {
- let context_sources = self
- .context_sources()
- .into_iter()
- .filter(|context_source| *context_source != ContextSource::Lsp)
- .collect::<Vec<_>>();
- if context_sources.is_empty() {
- None
- } else {
- Some(context_sources)
- }
- }
}
pub fn context_sources_for_types(context_types: &[ContextRetrievalType]) -> Vec<ContextSource> {
@@ -109,7 +92,6 @@ fn editable_context_sources() -> Vec<ContextSource> {
ContextSource::EditHistoryFile,
ContextSource::GitLog,
ContextSource::Bm25,
- ContextSource::OracleFile,
]
}
@@ -117,7 +99,7 @@ pub async fn run_context_retrieval(
example: &mut Example,
app_state: Arc<EpAppState>,
example_progress: &ExampleProgress,
- context_type: ContextRetrievalType,
+ context_types: Vec<ContextRetrievalType>,
force: bool,
mut cx: AsyncApp,
) -> anyhow::Result<()> {
@@ -143,8 +125,9 @@ pub async fn run_context_retrieval(
.context("EditPredictionStore not initialized")?;
let mut context_files = Vec::new();
+ let context_sources = context_sources_for_types(&context_types);
- if context_type.includes_lsp() {
+ if context_sources.contains(&ContextSource::Lsp) {
let _lsp_handle = project.update(&mut cx, |project, cx| {
project.register_buffer_with_language_servers(&state.buffer, cx)
});
@@ -170,8 +153,12 @@ pub async fn run_context_retrieval(
.extend(ep_store.update(&mut cx, |store, cx| store.context_for_project(&project, cx)));
}
- if let Some(context_sources) = context_type.editable_context_sources() {
- let oracle_paths = if context_sources.contains(&ContextSource::OracleFile) {
+ let editable_context_sources = context_sources
+ .into_iter()
+ .filter(|context_source| *context_source != ContextSource::Lsp)
+ .collect::<Vec<_>>();
+ if !editable_context_sources.is_empty() {
+ let oracle_paths = if editable_context_sources.contains(&ContextSource::OracleFile) {
let oracle_paths = oracle_paths_from_expected_patches(example);
refresh_paths(&project, &oracle_paths, &mut cx).await?;
oracle_paths
@@ -186,7 +173,7 @@ pub async fn run_context_retrieval(
state.buffer.clone(),
state.cursor_position,
oracle_paths,
- context_sources,
+ editable_context_sources,
cx,
)
})
diff --git a/crates/edit_prediction_cli/src/score.rs b/crates/edit_prediction_cli/src/score.rs
index 2a860df0fdb..841d15541e4 100644
--- a/crates/edit_prediction_cli/src/score.rs
+++ b/crates/edit_prediction_cli/src/score.rs
@@ -891,8 +891,14 @@ fn truncate_name(name: &str, max_len: usize) -> String {
pub type SummaryJson = edit_prediction_metrics::SummaryJson;
-pub fn compute_summary(examples: &[Example]) -> SummaryJson {
+pub fn compute_summary(
+ examples: &[Example],
+ retrieved_context_byte_limit: Option<usize>,
+ context_source_filter: Option<&[ContextSource]>,
+) -> SummaryJson {
edit_prediction_metrics::compute_summary(examples.iter().flat_map(|example| {
+ let retrieved_context_bytes =
+ retrieved_context_bytes(example, retrieved_context_byte_limit, context_source_filter);
example
.score
.iter()
@@ -906,14 +912,30 @@ pub fn compute_summary(examples: &[Example]) -> SummaryJson {
reverts_edits: qa.reverts_edits,
confidence: qa.confidence,
});
-
- edit_prediction_metrics::PredictionSummaryInput { score, qa }
+ let retrieved_context_bytes = (score_idx == 0)
+ .then_some(retrieved_context_bytes)
+ .flatten();
+
+ edit_prediction_metrics::PredictionSummaryInput {
+ score,
+ qa,
+ retrieved_context_bytes,
+ }
})
}))
}
-pub fn write_summary_json(examples: &[Example], path: &Path) -> anyhow::Result<()> {
- let summary = compute_summary(examples);
+pub fn write_summary_json(
+ examples: &[Example],
+ path: &Path,
+ retrieved_context_byte_limit: Option<usize>,
+ context_source_filter: Option<&[ContextSource]>,
+) -> anyhow::Result<()> {
+ let summary = compute_summary(
+ examples,
+ retrieved_context_byte_limit,
+ context_source_filter,
+ );
let file = File::create(path)
.with_context(|| format!("Failed to create summary JSON file: {}", path.display()))?;
let writer = BufWriter::new(file);
@@ -922,3 +944,100 @@ pub fn write_summary_json(examples: &[Example], path: &Path) -> anyhow::Result<(
eprintln!("Wrote summary JSON to: {}", path.display());
Ok(())
}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use edit_prediction::example_spec::ExampleSpec;
+ use edit_prediction_metrics::PredictionScore;
+ use std::path::Path;
+ use zeta_prompt::{ExcerptRanges, RelatedExcerpt, ZetaPromptInput};
+
+ #[test]
+ fn summary_includes_limited_filtered_retrieved_context_bytes_once_per_example() {
+ let examples = vec![
+ example_with_related_files(
+ Some(vec![RelatedFile {
+ path: Path::new("project/src/lib.rs").into(),
+ max_row: 10,
+ excerpts: vec![
+ related_excerpt("abcd", 0..1, 0, ContextSource::CurrentFile),
+ related_excerpt("ignored by source filter", 1..2, 1, ContextSource::Lsp),
+ related_excerpt("efghij", 2..3, 2, ContextSource::CurrentFile),
+ ],
+ in_open_source_repo: false,
+ }]),
+ 2,
+ ),
+ example_with_related_files(None, 1),
+ ];
+
+ let summary = compute_summary(&examples, Some(10), Some(&[ContextSource::CurrentFile]));
+
+ assert_eq!(summary.total_examples, 3);
+ assert_eq!(summary.avg_retrieved_context_bytes, Some(10.0));
+ assert_eq!(summary.total_retrieved_context_bytes, Some(10));
+ assert_eq!(summary.retrieved_context_examples, Some(1));
+ }
+
+ fn example_with_related_files(
+ related_files: Option<Vec<RelatedFile>>,
+ score_count: usize,
+ ) -> Example {
+ Example {
+ spec: ExampleSpec {
+ name: "example".to_string(),
+ repository_url: "https://github.com/zed-industries/zed.git".to_string(),
+ revision: "revision".to_string(),
+ tags: Vec::new(),
+ reasoning: None,
+ uncommitted_diff: String::new(),
+ recently_opened_files: Vec::new(),
+ recently_viewed_files: Vec::new(),
+ uncommitted_diff_contains_edit_history: false,
+ cursor_path: Path::new("project/src/main.rs").into(),
+ cursor_position: String::new(),
+ edit_history: String::new(),
+ expected_patches: Vec::new(),
+ rejected_patch: None,
+ telemetry: None,
+ human_feedback: Vec::new(),
+ rating: None,
+ },
+ prompt_inputs: Some(ZetaPromptInput {
+ cursor_path: Path::new("project/src/main.rs").into(),
+ cursor_excerpt: "".into(),
+ cursor_offset_in_excerpt: 0,
+ excerpt_start_row: None,
+ events: Vec::new(),
+ related_files,
+ active_buffer_diagnostics: Vec::new(),
+ excerpt_ranges: ExcerptRanges::default(),
+ syntax_ranges: None,
+ in_open_source_repo: false,
+ can_collect_data: false,
+ repo_url: None,
+ }),
+ prompt: None,
+ predictions: Vec::new(),
+ score: vec![PredictionScore::zero(); score_count],
+ qa: Vec::new(),
+ zed_version: None,
+ state: None,
+ }
+ }
+
+ fn related_excerpt(
+ text: &str,
+ row_range: std::ops::Range<u32>,
+ order: usize,
+ context_source: ContextSource,
+ ) -> RelatedExcerpt {
+ RelatedExcerpt {
+ row_range,
+ text: text.into(),
+ order,
+ context_source,
+ }
+ }
+}
diff --git a/crates/edit_prediction_metrics/src/summary.rs b/crates/edit_prediction_metrics/src/summary.rs
index 9c4028a300b..f927a027d76 100644
--- a/crates/edit_prediction_metrics/src/summary.rs
+++ b/crates/edit_prediction_metrics/src/summary.rs
@@ -13,6 +13,7 @@ pub struct QaSummaryData {
pub struct PredictionSummaryInput<'a> {
pub score: &'a PredictionScore,
pub qa: Option<QaSummaryData>,
+ pub retrieved_context_bytes: Option<usize>,
}
#[derive(Clone, Debug, Serialize)]
@@ -80,6 +81,12 @@ pub struct SummaryJson {
pub editable_context_files_fp: Option<usize>,
#[serde(skip_serializing_if = "Option::is_none")]
pub editable_context_files_fn: Option<usize>,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub avg_retrieved_context_bytes: Option<f64>,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub total_retrieved_context_bytes: Option<usize>,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub retrieved_context_examples: Option<usize>,
}
pub fn compute_summary<'a>(
@@ -128,6 +135,8 @@ pub fn compute_summary<'a>(
let mut editable_context_files_tp: usize = 0;
let mut editable_context_files_fp: usize = 0;
let mut editable_context_files_fn: usize = 0;
+ let mut retrieved_context_bytes_total: usize = 0;
+ let mut retrieved_context_bytes_count: usize = 0;
for prediction in predictions {
let score = prediction.score;
@@ -186,6 +195,11 @@ pub fn compute_summary<'a>(
recall_rate_sum += recall_rate;
recall_rate_count += 1;
}
+ if let Some(retrieved_context_bytes) = prediction.retrieved_context_bytes {
+ retrieved_context_bytes_total += retrieved_context_bytes;
+ retrieved_context_bytes_count += 1;
+ }
+
if let Some(coverage) = &score.editable_context_coverage {
editable_context_lines_precision_sum += coverage.lines_precision;
editable_context_lines_recall_sum += coverage.lines_recall;
@@ -364,6 +378,21 @@ pub fn compute_summary<'a>(
} else {
None
};
+ let avg_retrieved_context_bytes = if retrieved_context_bytes_count > 0 {
+ Some(retrieved_context_bytes_total as f64 / retrieved_context_bytes_count as f64)
+ } else {
+ None
+ };
+ let total_retrieved_context_bytes = if retrieved_context_bytes_count > 0 {
+ Some(retrieved_context_bytes_total)
+ } else {
+ None
+ };
+ let retrieved_context_examples = if retrieved_context_bytes_count > 0 {
+ Some(retrieved_context_bytes_count)
+ } else {
+ None
+ };
SummaryJson {
total_examples: total_scores,
@@ -414,5 +443,8 @@ pub fn compute_summary<'a>(
editable_context_files_tp,
editable_context_files_fp,
editable_context_files_fn,
+ avg_retrieved_context_bytes,
+ total_retrieved_context_bytes,
+ retrieved_context_examples,
}
}