Code at the speed of thought – Zed is a high-performance, multiplayer code editor from the creators of Atom and Tree-sitter.
ep: Editable context collectors and metrics (#58400)
Oleksiy Syvokonade2b219f788
diff --git a/Cargo.lock b/Cargo.lockindex b07750b7ba8..830cfca9688 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -5524,6 +5524,7 @@ dependencies = ["debug_adapter_extension","dirs","edit_prediction",+ "edit_prediction_context","edit_prediction_metrics","extension","flate2",diff --git a/crates/edit_prediction/src/edit_prediction.rs b/crates/edit_prediction/src/edit_prediction.rsindex ab88dbf2fa6..5e48063cd6d 100644--- a/crates/edit_prediction/src/edit_prediction.rs+++ b/crates/edit_prediction/src/edit_prediction.rs@@ -55,6 +55,8 @@ use std::env;use std::rc::Rc;use text::{AnchorRangeExt, Edit};use workspace::{AppState, Workspace};+#[cfg(feature = "cli-support")]+use zeta_prompt::ContextSource;use zeta_prompt::{ZetaFormat, ZetaPromptInput};use std::mem;@@ -3047,6 +3049,51 @@ impl EditPredictionStore {});}+ #[cfg(feature = "cli-support")]+ pub fn collect_editable_context(+ &mut self,+ project: Entity<Project>,+ buffer: Entity<language::Buffer>,+ cursor_position: language::Anchor,+ oracle_paths: Vec<Arc<Path>>,+ context_sources: Vec<ContextSource>,+ cx: &mut Context<Self>,+ ) -> Task<anyhow::Result<Vec<RelatedFile>>> {+ use edit_prediction_context::{EditHistoryContextEntry, collect_editable_context};++ let buffers_by_id = project.read(cx).opened_buffers(cx).into_iter().fold(+ HashMap::default(),+ |mut buffers_by_id, buffer| {+ buffers_by_id.insert(buffer.read(cx).remote_id(), buffer.clone());+ buffers_by_id+ },+ );+ let edit_history = self+ .edit_history_for_project(&project, cx)+ .into_iter()+ .filter_map(|event| {+ let buffer = buffers_by_id.get(&event.old_snapshot.remote_id())?.clone();+ Some(EditHistoryContextEntry {+ buffer,+ edited_range: event.total_edit_range,+ })+ })+ .collect();++ cx.spawn(async move |_, cx| {+ collect_editable_context(+ project,+ buffer,+ cursor_position,+ edit_history,+ oracle_paths,+ context_sources,+ cx,+ )+ .await+ })+ }+#[cfg(feature = "cli-support")]pub fn set_context_for_buffer(&mut self,diff --git a/crates/edit_prediction_cli/Cargo.toml b/crates/edit_prediction_cli/Cargo.tomlindex 81e91fa262d..733deb895c0 100644--- a/crates/edit_prediction_cli/Cargo.toml+++ b/crates/edit_prediction_cli/Cargo.toml@@ -61,6 +61,7 @@ terminal_view.workspace = trueutil.workspace = truewatch.workspace = trueedit_prediction = { workspace = true, features = ["cli-support"] }+edit_prediction_context.workspace = trueedit_prediction_metrics = { workspace = true, features = ["tree-sitter"] }telemetry_events.workspace = truewasmtime.workspace = true@@ -90,4 +91,3 @@ pretty_assertions.workspace = trueproject = { workspace = true, features = ["test-support"] }tempfile.workspace = trueworkspace = { workspace = true, features = ["test-support"] }-diff --git a/crates/edit_prediction_cli/src/format_prompt.rs b/crates/edit_prediction_cli/src/format_prompt.rsindex e0354a78aca..933560b476b 100644--- a/crates/edit_prediction_cli/src/format_prompt.rs+++ b/crates/edit_prediction_cli/src/format_prompt.rs@@ -3,7 +3,7 @@ use crate::{example::{ActualCursor, Example, ExamplePrompt},headless::EpAppState,progress::{ExampleProgress, Step},- retrieve_context::run_context_retrieval,+ retrieve_context::{ContextRetrievalType, run_context_retrieval},};use anyhow::{Context as _, Result, anyhow};use gpui::AsyncApp;@@ -30,7 +30,15 @@ pub async fn run_format_prompt(example_progress: &ExampleProgress,cx: AsyncApp,) -> Result<()> {- run_context_retrieval(example, app_state.clone(), example_progress, cx.clone()).await?;+ run_context_retrieval(+ example,+ app_state.clone(),+ example_progress,+ ContextRetrievalType::Lsp,+ false,+ cx.clone(),+ )+ .await?;let step_progress = example_progress.start(Step::FormatPrompt);diff --git a/crates/edit_prediction_cli/src/main.rs b/crates/edit_prediction_cli/src/main.rsindex 41e7b397745..69167ce22b4 100644--- a/crates/edit_prediction_cli/src/main.rs+++ b/crates/edit_prediction_cli/src/main.rs@@ -59,7 +59,7 @@ use crate::paths::{FAILED_EXAMPLES_DIR, RUN_DIR};use crate::predict::run_prediction;use crate::progress::Progress;use crate::pull_examples::{fetch_settled_examples_after, parse_settled_after_input};-use crate::retrieve_context::run_context_retrieval;+use crate::retrieve_context::{ContextRetrievalType, run_context_retrieval};use crate::score::run_scoring;use crate::split_commit::SplitCommitArgs;use crate::split_dataset::SplitArgs;@@ -100,7 +100,7 @@ struct EpArgs {output: Option<PathBuf>,#[arg(long, short, global = true)]in_place: bool,- #[arg(long, short, global = true)]+ #[arg(long, global = true)]failfast: bool,/// How to handle failed examples in output: keep them or skip them./// Failed examples are always logged to the run's failed directory.@@ -125,6 +125,16 @@ pub enum FailedHandling {SkipNoFiles,}+#[derive(Args, Debug, Clone)]+struct ContextArgs {+ /// Which context collector to run.+ #[arg(long = "type", value_enum, default_value_t = ContextRetrievalType::Lsp)]+ context_type: ContextRetrievalType,+ /// Recompute context even if the example already has related files.+ #[arg(long, short = 'f', default_value_t = false)]+ force: bool,+}+const INPUTS_HELP: &str = r#"Inputs can be file paths or special specifiers:@@ -197,7 +207,7 @@ enum Command {/// Create git worktrees for each example and load file contentsLoadProject,/// Retrieve context for input examples.- Context,+ Context(ContextArgs),/// Generate a prompt string for a specific modelFormatPrompt(FormatPromptArgs),/// Runs edit prediction@@ -239,7 +249,13 @@ impl Display for Command {match self {Command::Read(_) => write!(f, "read"),Command::LoadProject => write!(f, "load-project"),- Command::Context => write!(f, "context"),+ Command::Context(args) => {+ write!(f, "context --type={}", args.context_type)?;+ if args.force {+ write!(f, " --force")?;+ }+ Ok(())+ }Command::FormatPrompt(args) => {write!(f, "format-prompt --provider={}", args.provider)}@@ -253,10 +269,16 @@ impl Display for Command {None => write!(f, "score"),},Command::Distill => write!(f, "distill"),- Command::Eval(args) => match &args.predict.provider {- Some(provider) => write!(f, "eval --provider={}", provider),- None => write!(f, "eval"),- },+ Command::Eval(args) => {+ write!(f, "eval")?;+ if args.context_only {+ write!(f, " --context-only")?;+ }+ if let Some(provider) = &args.predict.provider {+ write!(f, " --provider={}", provider)?;+ }+ Ok(())+ }Command::Synthesize(args) => {write!(f, "synthesize --repos {}", args.repos.join(" "))}@@ -309,6 +331,9 @@ struct PredictArgs {struct EvalArgs {#[clap(flatten)]predict: PredictArgs,+ /// Only compute editable context coverage from expected patches and retrieved context.+ #[clap(long)]+ context_only: bool,/// Path to write summary scores as JSON#[clap(long)]summary_json: Option<PathBuf>,@@ -1133,7 +1158,9 @@ fn main() {predict::sync_batches(args.provider.as_ref()).await?;}Command::Eval(args) => {- predict::sync_batches(args.predict.provider.as_ref()).await?;+ if !args.context_only {+ predict::sync_batches(args.predict.provider.as_ref()).await?;+ }}Command::Qa(args) => {qa::sync_batches(args).await?;@@ -1222,11 +1249,13 @@ fn main() {).await?;}- Command::Context => {+ Command::Context(args) => {run_context_retrieval(example,app_state.clone(),&example_progress,+ args.context_type,+ args.force,cx.clone(),).await?;@@ -1264,18 +1293,30 @@ fn main() {app_state.clone(),&example_progress,cx.clone(),+ false,+ None,).await?;}Command::Eval(args) => {- run_scoring(- example,- &args.predict,- app_state.clone(),- &example_progress,- cx.clone(),- )- .await?;+ if args.context_only {+ score::run_context_coverage_scoring(+ example,+ &example_progress,+ Some(score::EVAL_RETRIEVED_CONTEXT_BYTE_LIMIT),+ )?;+ } else {+ run_scoring(+ example,+ &args.predict,+ app_state.clone(),+ &example_progress,+ cx.clone(),+ true,+ Some(score::EVAL_RETRIEVED_CONTEXT_BYTE_LIMIT),+ )+ .await?;+ }}Command::Qa(args) => {qa::run_qa(example, args, &example_progress).await?;@@ -1394,15 +1435,17 @@ fn main() {}}Command::Eval(args) => {- predict::sync_batches(args.predict.provider.as_ref()).await?;- if args.predict.wait {- predict::wait_for_batches(args.predict.provider.as_ref()).await?;- let mut examples =- std::mem::take(&mut *finished_examples.lock().unwrap());- predict::reprocess_after_batch_wait(&mut examples, &args.predict)- .await?;- rewrite_output(&examples, write_path, is_markdown)?;- *finished_examples.lock().unwrap() = examples;+ if !args.context_only {+ predict::sync_batches(args.predict.provider.as_ref()).await?;+ if args.predict.wait {+ predict::wait_for_batches(args.predict.provider.as_ref()).await?;+ let mut examples =+ std::mem::take(&mut *finished_examples.lock().unwrap());+ predict::reprocess_after_batch_wait(&mut examples, &args.predict)+ .await?;+ rewrite_output(&examples, write_path, is_markdown)?;+ *finished_examples.lock().unwrap() = examples;+ }}}Command::Qa(args) => {@@ -1425,7 +1468,12 @@ fn main() {match &command {Command::Eval(args) => {let examples = finished_examples.lock().unwrap();- score::print_report(&examples, args.verbose);+ score::print_report(+ &examples,+ args.verbose,+ args.context_only,+ Some(score::EVAL_RETRIEVED_CONTEXT_BYTE_LIMIT),+ );if let Some(summary_path) = &args.summary_json {score::write_summary_json(&examples, summary_path)?;}diff --git a/crates/edit_prediction_cli/src/predict.rs b/crates/edit_prediction_cli/src/predict.rsindex a78d4d12d70..c21066aa812 100644--- a/crates/edit_prediction_cli/src/predict.rs+++ b/crates/edit_prediction_cli/src/predict.rs@@ -9,7 +9,7 @@ use crate::{parse_output::parse_prediction_output,paths::{LATEST_EXAMPLE_RUN_DIR, RUN_DIR},progress::{ExampleProgress, InfoStyle, Progress, Step, StepProgress},- retrieve_context::run_context_retrieval,+ retrieve_context::{ContextRetrievalType, run_context_retrieval},};use anyhow::Context as _;use cloud_llm_client::predict_edits_v3::{RawCompletionRequest, RawCompletionResponse};@@ -68,7 +68,15 @@ pub async fn run_prediction(if let PredictionProvider::Teacher(backend, _)| PredictionProvider::TeacherNonBatching(backend, _) = provider{- run_context_retrieval(example, app_state.clone(), example_progress, cx.clone()).await?;+ run_context_retrieval(+ example,+ app_state.clone(),+ example_progress,+ ContextRetrievalType::Lsp,+ false,+ cx.clone(),+ )+ .await?;run_format_prompt(example,&FormatPromptArgs { provider },@@ -111,7 +119,15 @@ pub async fn run_prediction(}run_load_project(example, app_state.clone(), example_progress, cx.clone()).await?;- run_context_retrieval(example, app_state.clone(), example_progress, cx.clone()).await?;+ run_context_retrieval(+ example,+ app_state.clone(),+ example_progress,+ ContextRetrievalType::Lsp,+ false,+ cx.clone(),+ )+ .await?;let step_progress = example_progress.start(Step::Predict);diff --git a/crates/edit_prediction_cli/src/retrieve_context.rs b/crates/edit_prediction_cli/src/retrieve_context.rsindex f02509ceb06..91d9ab0761b 100644--- a/crates/edit_prediction_cli/src/retrieve_context.rs+++ b/crates/edit_prediction_cli/src/retrieve_context.rs@@ -5,25 +5,88 @@ use crate::{progress::{ExampleProgress, InfoStyle, Step, StepProgress},};use anyhow::Context as _;+use clap::ValueEnum;use collections::HashSet;-use edit_prediction::{DebugEvent, EditPredictionStore};+use edit_prediction::{DebugEvent, EditPredictionStore, udiff::refresh_worktree_entries};use futures::{FutureExt as _, StreamExt as _, channel::mpsc};use gpui::{AsyncApp, Entity};use language::Buffer;use project::Project;-use std::sync::Arc;-use std::time::Duration;+use std::{+ path::{Path, PathBuf},+ sync::Arc,+ time::Duration,+};+use zeta_prompt::{ContextSource, udiff::DiffLine};++#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, ValueEnum)]+pub enum ContextRetrievalType {+ Lsp,+ Editable,+ CurrentFile,+ EditHistory,+ EditHistoryFile,+ GitLog,+ OracleFile,+ #[default]+ All,+ None,+}++impl std::fmt::Display for ContextRetrievalType {+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ match self {+ ContextRetrievalType::Lsp => write!(f, "lsp"),+ ContextRetrievalType::Editable => write!(f, "editable"),+ ContextRetrievalType::CurrentFile => write!(f, "current-file"),+ ContextRetrievalType::EditHistory => write!(f, "edit-history"),+ ContextRetrievalType::EditHistoryFile => write!(f, "edit-history-file"),+ ContextRetrievalType::GitLog => write!(f, "git-log"),+ ContextRetrievalType::OracleFile => write!(f, "oracle-file"),+ ContextRetrievalType::All => write!(f, "all"),+ ContextRetrievalType::None => write!(f, "none"),+ }+ }+}++impl ContextRetrievalType {+ fn includes_lsp(self) -> bool {+ matches!(self, ContextRetrievalType::Lsp | ContextRetrievalType::All)+ }++ fn editable_context_sources(self) -> Option<Vec<ContextSource>> {+ match self {+ ContextRetrievalType::Editable | ContextRetrievalType::All => Some(vec![+ ContextSource::CursorExcerpt,+ ContextSource::CurrentFile,+ ContextSource::EditHistory,+ ContextSource::EditHistoryFile,+ ContextSource::GitLog,+ ContextSource::OracleFile,+ ]),+ ContextRetrievalType::CurrentFile => Some(vec![ContextSource::CurrentFile]),+ ContextRetrievalType::EditHistory => Some(vec![ContextSource::EditHistory]),+ ContextRetrievalType::EditHistoryFile => Some(vec![ContextSource::EditHistoryFile]),+ ContextRetrievalType::GitLog => Some(vec![ContextSource::GitLog]),+ ContextRetrievalType::OracleFile => Some(vec![ContextSource::OracleFile]),+ ContextRetrievalType::Lsp | ContextRetrievalType::None => None,+ }+ }+}pub async fn run_context_retrieval(example: &mut Example,app_state: Arc<EpAppState>,example_progress: &ExampleProgress,+ context_type: ContextRetrievalType,+ force: bool,mut cx: AsyncApp,) -> anyhow::Result<()> {- if example- .prompt_inputs- .as_ref()- .is_some_and(|inputs| inputs.related_files.is_some())+ if (!force+ && example+ .prompt_inputs+ .as_ref()+ .is_some_and(|inputs| inputs.related_files.is_some()))|| example.spec.repository_url.is_empty(){return Ok(());@@ -36,32 +99,61 @@ pub async fn run_context_retrieval(let state = example.state.as_ref().unwrap();let project = state.project.clone();- let _lsp_handle = project.update(&mut cx, |project, cx| {- project.register_buffer_with_language_servers(&state.buffer, cx)- });- wait_for_language_servers_to_start(&project, &state.buffer, &step_progress, &mut cx).await?;-let ep_store = cx.update(|cx| EditPredictionStore::try_global(cx)).context("EditPredictionStore not initialized")?;- let mut events = ep_store.update(&mut cx, |store, cx| {- store.register_buffer(&state.buffer, &project, cx);- store.refresh_context(&project, &state.buffer, state.cursor_position, cx);- store.debug_info(&project, cx)- });+ let mut context_files = Vec::new();++ if context_type.includes_lsp() {+ let _lsp_handle = project.update(&mut cx, |project, cx| {+ project.register_buffer_with_language_servers(&state.buffer, cx)+ });+ wait_for_language_servers_to_start(&project, &state.buffer, &step_progress, &mut cx)+ .await?;++ let mut events = ep_store.update(&mut cx, |store, cx| {+ store.register_buffer(&state.buffer, &project, cx);+ store.refresh_context(&project, &state.buffer, state.cursor_position, cx);+ store.debug_info(&project, cx)+ });- while let Some(event) = events.next().await {- match event {- DebugEvent::ContextRetrievalFinished(_) => {- break;+ while let Some(event) = events.next().await {+ match event {+ DebugEvent::ContextRetrievalFinished(_) => {+ break;+ }+ _ => {}}- _ => {}}++ context_files+ .extend(ep_store.update(&mut cx, |store, cx| store.context_for_project(&project, cx)));}- let context_files =- ep_store.update(&mut cx, |store, cx| store.context_for_project(&project, cx));+ if let Some(context_sources) = context_type.editable_context_sources() {+ let oracle_paths = if context_sources.contains(&ContextSource::OracleFile) {+ let oracle_paths = oracle_paths_from_expected_patches(example);+ refresh_paths(&project, &oracle_paths, &mut cx).await?;+ oracle_paths+ } else {+ Vec::new()+ };++ let editable_context = ep_store+ .update(&mut cx, |store, cx| {+ store.collect_editable_context(+ project.clone(),+ state.buffer.clone(),+ state.cursor_position,+ oracle_paths,+ context_sources,+ cx,+ )+ })+ .await?;+ merge_context_files(&mut context_files, editable_context);+ }let excerpt_count: usize = context_files.iter().map(|f| f.excerpts.len()).sum();step_progress.set_info(format!("{} excerpts", excerpt_count), InfoStyle::Normal);@@ -72,6 +164,73 @@ pub async fn run_context_retrieval(Ok(())}+fn merge_context_files(+ context_files: &mut Vec<zeta_prompt::RelatedFile>,+ new_files: Vec<zeta_prompt::RelatedFile>,+) {+ for mut new_file in new_files {+ if let Some(existing_file) = context_files+ .iter_mut()+ .find(|existing_file| existing_file.path == new_file.path)+ {+ existing_file.max_row = existing_file.max_row.max(new_file.max_row);+ existing_file.excerpts.append(&mut new_file.excerpts);+ existing_file+ .excerpts+ .sort_by_key(|excerpt| (excerpt.order, excerpt.row_range.start));+ existing_file.in_open_source_repo =+ existing_file.in_open_source_repo && new_file.in_open_source_repo;+ } else {+ context_files.push(new_file);+ }+ }+}++fn oracle_paths_from_expected_patches(example: &Example) -> Vec<Arc<Path>> {+ let mut seen_paths = HashSet::default();+ let mut paths = Vec::new();++ for patch in &example.spec.expected_patches {+ for path in paths_from_diff(patch) {+ if seen_paths.insert(path.clone()) {+ paths.push(path.into());+ }+ }+ }++ paths+}++fn paths_from_diff(diff: &str) -> Vec<PathBuf> {+ diff.lines()+ .filter_map(|line| match DiffLine::parse(line) {+ DiffLine::OldPath { path } | DiffLine::NewPath { path }+ if path.as_ref() != "/dev/null" =>+ {+ Some(Path::new(path.as_ref()).to_path_buf())+ }+ _ => None,+ })+ .collect()+}++async fn refresh_paths(+ project: &Entity<Project>,+ paths: &[Arc<Path>],+ cx: &mut AsyncApp,+) -> anyhow::Result<()> {+ if paths.is_empty() {+ return Ok(());+ }++ let Some(worktree) = project.read_with(cx, |project, cx| project.visible_worktrees(cx).next())+ else {+ return Ok(());+ };++ refresh_worktree_entries(&worktree, paths.iter().map(|path| path.as_ref()), cx).await+}+async fn wait_for_language_servers_to_start(project: &Entity<Project>,buffer: &Entity<Buffer>,diff --git a/crates/edit_prediction_cli/src/score.rs b/crates/edit_prediction_cli/src/score.rsindex cbaeb338650..e8227a0d7c4 100644--- a/crates/edit_prediction_cli/src/score.rs+++ b/crates/edit_prediction_cli/src/score.rs@@ -8,8 +8,9 @@ use crate::{progress::{ExampleProgress, Step},};use anyhow::Context as _;+use edit_prediction_context::limit_retrieved_context_to_bytes;use edit_prediction_metrics::{- ActualPredictionCursor, PredictionReversalContext, PredictionScoringInput,+ ActualPredictionCursor, Excerpt, PredictionReversalContext, PredictionScoringInput,};use gpui::{AppContext as _, AsyncApp};use std::fs::File;@@ -17,14 +18,20 @@ use std::io::BufWriter;use std::path::Path;use std::sync::Arc;+pub const EVAL_RETRIEVED_CONTEXT_BYTE_LIMIT: usize = 4000;+pub async fn run_scoring(example: &mut Example,args: &PredictArgs,app_state: Arc<EpAppState>,example_progress: &ExampleProgress,cx: AsyncApp,+ allow_missing_predictions: bool,+ retrieved_context_byte_limit: Option<usize>,) -> anyhow::Result<()> {- run_prediction(example, args, app_state, example_progress, cx.clone()).await?;+ if !(allow_missing_predictions && args.provider.is_none() && example.predictions.is_empty()) {+ run_prediction(example, args, app_state, example_progress, cx.clone()).await?;+ }let progress = example_progress.start(Step::Score);@@ -70,8 +77,32 @@ pub async fn run_scoring(})?;let cursor_path = example_for_scoring.spec.cursor_path.as_ref();+ let context = context_excerpts(+ &example_for_scoring,+ prompt_inputs,+ retrieved_context_byte_limit,+ );let mut scores = vec![];+ if allow_missing_predictions && example_for_scoring.predictions.is_empty() {+ scores.push(edit_prediction_metrics::score_prediction(+ PredictionScoringInput {+ original_text,+ expected_patches: &prepared_expected_patches,+ actual_patch: None,+ actual_cursor: None,+ reversal_context: Some(PredictionReversalContext {+ edit_history: &prompt_inputs.events,+ excerpt_start_row: prompt_inputs.excerpt_start_row,+ cursor_path,+ }),+ cumulative_logprob: None,+ avg_logprob: None,+ context: Some(&context),+ },+ ));+ }+for prediction in &example_for_scoring.predictions {let actual_patch = prediction.actual_patch.clone().or_else(|| {parse_prediction_output(@@ -103,6 +134,7 @@ pub async fn run_scoring(}),cumulative_logprob: prediction.cumulative_logprob,avg_logprob: prediction.avg_logprob,+ context: Some(&context),},));}@@ -113,11 +145,120 @@ pub async fn run_scoring(Ok(())}-pub fn print_report(examples: &[Example], verbose: bool) {+pub fn run_context_coverage_scoring(+ example: &mut Example,+ example_progress: &ExampleProgress,+ retrieved_context_byte_limit: Option<usize>,+) -> anyhow::Result<()> {+ let progress = example_progress.start(Step::Score);++ progress.set_substatus("computing context coverage");+ let prompt_inputs = example+ .prompt_inputs+ .as_ref()+ .context("prompt_inputs is required for context coverage scoring")?;+ let context = context_excerpts(example, prompt_inputs, retrieved_context_byte_limit);++ let editable_context_coverage = example+ .spec+ .expected_patches_with_cursor_positions()+ .iter()+ .map(|(expected_patch, _)| {+ edit_prediction_metrics::editable_context_coverage(expected_patch, &context)+ })+ .max_by(|left, right| {+ left.lines_f1+ .total_cmp(&right.lines_f1)+ .then_with(|| left.files_f1.total_cmp(&right.files_f1))+ });++ let mut score = edit_prediction_metrics::PredictionScore::zero();+ score.editable_context_coverage = editable_context_coverage;+ example.score = vec![score];++ Ok(())+}++fn context_excerpts(+ _example: &Example,+ prompt_inputs: &zeta_prompt::ZetaPromptInput,+ retrieved_context_byte_limit: Option<usize>,+) -> Vec<Excerpt> {+ let mut context = Vec::new();++ if let Some(excerpt_start_row) = prompt_inputs.excerpt_start_row {+ let row_count = prompt_inputs.cursor_excerpt.lines().count() as u32;++ context.push(Excerpt {+ path: prompt_inputs.cursor_path.to_string_lossy().to_string(),+ row_range: excerpt_start_row..excerpt_start_row.saturating_add(row_count),+ content: prompt_inputs.cursor_excerpt.to_string(),+ });+ }++ if let Some(related_files) = &prompt_inputs.related_files {+ let related_files = if let Some(max_bytes) = retrieved_context_byte_limit {+ limit_retrieved_context_to_bytes(related_files, max_bytes)+ } else {+ related_files.clone()+ };+ for related_file in &related_files {+ for excerpt in &related_file.excerpts {+ // First component is a project name which is not present in expected patch, skip it+ let path = related_file+ .path+ .iter()+ .skip(1)+ .collect::<std::path::PathBuf>()+ .to_string_lossy()+ .to_string();+ context.push(Excerpt {+ path,+ row_range: excerpt.row_range.clone(),+ content: excerpt.text.to_string(),+ });+ }+ }+ }++ context+}++fn retrieved_context_bytes(+ example: &Example,+ retrieved_context_byte_limit: Option<usize>,+) -> Option<usize> {+ let related_files = example.prompt_inputs.as_ref()?.related_files.as_ref()?;+ let related_files = if let Some(max_bytes) = retrieved_context_byte_limit {+ limit_retrieved_context_to_bytes(related_files, max_bytes)+ } else {+ related_files.clone()+ };+ Some(+ related_files+ .iter()+ .flat_map(|file| file.excerpts.iter())+ .map(|excerpt| excerpt.text.len())+ .sum::<usize>(),+ )+}++pub fn print_report(+ examples: &[Example],+ verbose: bool,+ context_only: bool,+ retrieved_context_byte_limit: Option<usize>,+) {const MAX_EXAMPLES_DEFAULT: usize = 20;use crate::metrics::ClassificationMetrics;const LINE_WIDTH: usize = 101;++ if context_only {+ print_context_coverage_report(examples, verbose, retrieved_context_byte_limit);+ return;+ }+let separator = "─".repeat(LINE_WIDTH);println!("{}", separator);@@ -154,14 +295,30 @@ pub fn print_report(examples: &[Example], verbose: bool) {let mut discarded_chars_total: usize = 0;let mut recall_rate_sum: f64 = 0.0;let mut recall_rate_count: usize = 0;+ let mut editable_context_coverage_count: usize = 0;+ let mut editable_context_lines_precision_sum = 0.0;+ let mut editable_context_lines_recall_sum = 0.0;+ let mut editable_context_lines_f1_sum = 0.0;+ let mut editable_context_files_precision_sum = 0.0;+ let mut editable_context_files_recall_sum = 0.0;+ let mut editable_context_files_f1_sum = 0.0;+ let mut total_editable_context_lines = ClassificationMetrics::default();+ let mut total_editable_context_files = ClassificationMetrics::default();let mut patch_inserted_tokens: Vec<usize> = Vec::new();let mut patch_deleted_tokens: Vec<usize> = Vec::new();let mut predictions_with_patch: usize = 0;+ let mut retrieved_context_bytes_sum = 0.0;+ let mut retrieved_context_bytes_count = 0;let mut printed_lines: usize = 0;let mut skipped_lines: usize = 0;for example in examples {+ if let Some(bytes) = retrieved_context_bytes(example, retrieved_context_byte_limit) {+ retrieved_context_bytes_sum += bytes as f64;+ retrieved_context_bytes_count += 1;+ }+for (score_idx, score) in example.score.iter().enumerate() {let exact_lines = score.exact_lines_counts();@@ -264,6 +421,25 @@ pub fn print_report(examples: &[Example], verbose: bool) {recall_rate_sum += rr;recall_rate_count += 1;}+ if let Some(coverage) = &score.editable_context_coverage {+ editable_context_coverage_count += 1;+ editable_context_lines_precision_sum += coverage.lines_precision;+ editable_context_lines_recall_sum += coverage.lines_recall;+ editable_context_lines_f1_sum += coverage.lines_f1;+ editable_context_files_precision_sum += coverage.files_precision;+ editable_context_files_recall_sum += coverage.files_recall;+ editable_context_files_f1_sum += coverage.files_f1;+ total_editable_context_lines.accumulate(&ClassificationMetrics {+ true_positives: coverage.lines_tp,+ false_positives: coverage.lines_fp,+ false_negatives: coverage.lines_fn,+ });+ total_editable_context_files.accumulate(&ClassificationMetrics {+ true_positives: coverage.files_tp,+ false_positives: coverage.files_fp,+ false_negatives: coverage.files_fn,+ });+ }// Accumulate token change metrics (only for predictions that produced a patch)let has_patch = example@@ -414,6 +590,36 @@ pub fn print_report(examples: &[Example], verbose: bool) {recall_rate_count);}+ if retrieved_context_bytes_count > 0 {+ println!(+ "Retrieved context size: {:.0} bytes avg ({} examples)",+ retrieved_context_bytes_sum / retrieved_context_bytes_count as f64,+ retrieved_context_bytes_count+ );+ }+ if editable_context_coverage_count > 0 {+ let count = editable_context_coverage_count as f64;+ println!(+ "Editable context lines: P={:.1}%, R={:.1}%, F1={:.1}% avg ({} evaluated, TP={}, FP={}, FN={})",+ editable_context_lines_precision_sum / count * 100.0,+ editable_context_lines_recall_sum / count * 100.0,+ editable_context_lines_f1_sum / count * 100.0,+ editable_context_coverage_count,+ total_editable_context_lines.true_positives,+ total_editable_context_lines.false_positives,+ total_editable_context_lines.false_negatives+ );+ println!(+ "Editable context files: P={:.1}%, R={:.1}%, F1={:.1}% avg ({} evaluated, TP={}, FP={}, FN={})",+ editable_context_files_precision_sum / count * 100.0,+ editable_context_files_recall_sum / count * 100.0,+ editable_context_files_f1_sum / count * 100.0,+ editable_context_coverage_count,+ total_editable_context_files.true_positives,+ total_editable_context_files.false_positives,+ total_editable_context_files.false_negatives+ );+ }// Print token change percentile summary (only for predictions with a patch)if !patch_inserted_tokens.is_empty() {@@ -470,6 +676,148 @@ pub fn print_report(examples: &[Example], verbose: bool) {println!("\n");}+fn print_context_coverage_report(+ examples: &[Example],+ verbose: bool,+ retrieved_context_byte_limit: Option<usize>,+) {+ const MAX_EXAMPLES_DEFAULT: usize = 20;+ const LINE_WIDTH: usize = 120;++ use crate::metrics::ClassificationMetrics;++ let separator = "─".repeat(LINE_WIDTH);+ println!("{}", separator);+ println!(+ "{:<40} {:>6} {:>6} {:>6} {:>5} {:>5} {:>5} {:>6} {:>6} {:>6} {:>5} {:>5} {:>5}",+ "Example",+ "LineP",+ "LineR",+ "LineF1",+ "LTP",+ "LFP",+ "LFN",+ "FileP",+ "FileR",+ "FileF1",+ "FTP",+ "FFP",+ "FFN"+ );+ println!("{}", separator);++ let mut total_lines = ClassificationMetrics::default();+ let mut total_files = ClassificationMetrics::default();+ let mut line_precision_sum = 0.0;+ let mut line_recall_sum = 0.0;+ let mut line_f1_sum = 0.0;+ let mut file_precision_sum = 0.0;+ let mut file_recall_sum = 0.0;+ let mut file_f1_sum = 0.0;+ let mut total_scores = 0;+ let mut retrieved_context_bytes_sum = 0.0;+ let mut retrieved_context_bytes_count = 0;+ let mut printed_lines = 0;+ let mut skipped_lines = 0;++ for example in examples {+ if let Some(bytes) = retrieved_context_bytes(example, retrieved_context_byte_limit) {+ retrieved_context_bytes_sum += bytes as f64;+ retrieved_context_bytes_count += 1;+ }++ for score in &example.score {+ let Some(coverage) = &score.editable_context_coverage else {+ continue;+ };++ if verbose || printed_lines < MAX_EXAMPLES_DEFAULT {+ println!(+ "{:<40} {:>5.1}% {:>5.1}% {:>5.1}% {:>5} {:>5} {:>5} {:>5.1}% {:>5.1}% {:>5.1}% {:>5} {:>5} {:>5}",+ truncate_name(&example.spec.name, 40),+ coverage.lines_precision * 100.0,+ coverage.lines_recall * 100.0,+ coverage.lines_f1 * 100.0,+ coverage.lines_tp,+ coverage.lines_fp,+ coverage.lines_fn,+ coverage.files_precision * 100.0,+ coverage.files_recall * 100.0,+ coverage.files_f1 * 100.0,+ coverage.files_tp,+ coverage.files_fp,+ coverage.files_fn+ );+ printed_lines += 1;+ } else {+ skipped_lines += 1;+ }++ total_scores += 1;+ line_precision_sum += coverage.lines_precision;+ line_recall_sum += coverage.lines_recall;+ line_f1_sum += coverage.lines_f1;+ file_precision_sum += coverage.files_precision;+ file_recall_sum += coverage.files_recall;+ file_f1_sum += coverage.files_f1;+ total_lines.accumulate(&ClassificationMetrics {+ true_positives: coverage.lines_tp,+ false_positives: coverage.lines_fp,+ false_negatives: coverage.lines_fn,+ });+ total_files.accumulate(&ClassificationMetrics {+ true_positives: coverage.files_tp,+ false_positives: coverage.files_fp,+ false_negatives: coverage.files_fn,+ });+ }+ }++ if skipped_lines > 0 {+ println!(+ "{:<40} (use --verbose to see all {} examples)",+ format!("... and {} more", skipped_lines),+ printed_lines + skipped_lines+ );+ }++ println!("{}", separator);++ if total_scores > 0 {+ let count = total_scores as f64;+ println!(+ "{:<40} {:>5.1}% {:>5.1}% {:>5.1}% {:>5} {:>5} {:>5} {:>5.1}% {:>5.1}% {:>5.1}% {:>5} {:>5} {:>5}",+ "TOTAL / AVERAGE",+ line_precision_sum / count * 100.0,+ line_recall_sum / count * 100.0,+ line_f1_sum / count * 100.0,+ total_lines.true_positives,+ total_lines.false_positives,+ total_lines.false_negatives,+ file_precision_sum / count * 100.0,+ file_recall_sum / count * 100.0,+ file_f1_sum / count * 100.0,+ total_files.true_positives,+ total_files.false_positives,+ total_files.false_negatives+ );+ println!("{}", separator);+ println!(+ "Evaluated editable context coverage for {} examples",+ total_scores+ );+ if retrieved_context_bytes_count > 0 {+ println!(+ "Retrieved context size: {:.0} bytes avg ({} examples)",+ retrieved_context_bytes_sum / retrieved_context_bytes_count as f64,+ retrieved_context_bytes_count+ );+ }+ }++ println!("\n");+}+fn percentile(sorted_values: &[usize], p: usize) -> usize {if sorted_values.is_empty() {return 0;diff --git a/crates/edit_prediction_context/src/edit_prediction_context.rs b/crates/edit_prediction_context/src/edit_prediction_context.rsindex 5671df6092e..97a1729ec64 100644--- a/crates/edit_prediction_context/src/edit_prediction_context.rs+++ b/crates/edit_prediction_context/src/edit_prediction_context.rs@@ -22,10 +22,16 @@ use util::{RangeExt as _, ResultExt};mod assemble_excerpts;#[cfg(test)]mod edit_prediction_context_tests;+mod editable_context;#[cfg(test)]mod fake_definition_lsp;+mod git_log_context;-pub use zeta_prompt::{RelatedExcerpt, RelatedFile};+pub use editable_context::{+ EditHistoryContextEntry, collect_editable_context, limit_retrieved_context_to_bytes,+};++pub use zeta_prompt::{ContextSource, RelatedExcerpt, RelatedFile};const IDENTIFIER_LINE_COUNT: u32 = 3;@@ -573,6 +579,7 @@ impl RelatedBuffer {row_range: start.row..end.row,text: buffer.text_for_range(start..end).collect::<String>().into(),order,+ context_source: ContextSource::Lsp,}}).collect::<Vec<_>>();diff --git a/crates/edit_prediction_context/src/edit_prediction_context_tests.rs b/crates/edit_prediction_context/src/edit_prediction_context_tests.rsindex 32dc37b9532..5b8e9c6aecb 100644--- a/crates/edit_prediction_context/src/edit_prediction_context_tests.rs+++ b/crates/edit_prediction_context/src/edit_prediction_context_tests.rs@@ -308,6 +308,7 @@ async fn test_assemble_excerpts(cx: &mut TestAppContext) {row_range,text: buffer.text_for_range(start..end).collect::<String>().into(),order,+ context_source: ContextSource::Lsp,}}).collect();diff --git a/crates/edit_prediction_context/src/editable_context.rs b/crates/edit_prediction_context/src/editable_context.rsnew file mode 100644index 00000000000..c1beaaf634d--- /dev/null+++ b/crates/edit_prediction_context/src/editable_context.rs@@ -0,0 +1,616 @@+use collections::{HashMap, HashSet};+use gpui::{App, AppContext as _, AsyncApp, Entity, EntityId};+use language::{Buffer, BufferSnapshot, Point, ToPoint as _};+use project::{Project, ProjectPath};+use std::{+ ops::Range,+ path::{Path, PathBuf},+ sync::Arc,+};+use text::Anchor;+use util::{paths::PathStyle, rel_path::RelPath};+use zeta_prompt::{ContextSource, RelatedExcerpt, RelatedFile};++use crate::git_log_context::build_git_log_index;++/// This module contains collectors for editable context:+/// excerpts or full files that are likely to be edited.+const CURSOR_CONTEXT_LINE_COUNT: u32 = 20;+const EDIT_HISTORY_CONTEXT_LINE_COUNT: u32 = 20;+const GIT_LOG_CONTEXT_LINE_COUNT: u32 = 10000;+const GIT_LOG_CONTEXT_FILE_COUNT: usize = 10;++type RangesByBuffer = HashMap<EntityId, (Entity<Buffer>, Vec<EditableContextRange>)>;++#[derive(Clone)]+pub struct EditHistoryContextEntry {+ pub buffer: Entity<Buffer>,+ pub edited_range: Range<Anchor>,+}++struct EditableContextRange {+ range: Range<Anchor>,+ order: usize,+ context_source: ContextSource,+}++struct ResolvedEditableContextRange {+ range: Range<Point>,+ order: usize,+ context_source: ContextSource,+}++pub async fn collect_editable_context(+ project: Entity<Project>,+ active_buffer: Entity<Buffer>,+ cursor_position: Anchor,+ edit_history: Vec<EditHistoryContextEntry>,+ oracle_paths: Vec<Arc<Path>>,+ context_sources: Vec<ContextSource>,+ cx: &mut AsyncApp,+) -> anyhow::Result<Vec<RelatedFile>> {+ let mut ranges_by_buffer = RangesByBuffer::default();++ if context_sources.contains(&ContextSource::CursorExcerpt) {+ collect_cursor_excerpt_context(+ &mut ranges_by_buffer,+ active_buffer.clone(),+ cursor_position,+ cx,+ );+ }+ if context_sources.contains(&ContextSource::CurrentFile) {+ collect_current_file_context(&mut ranges_by_buffer, active_buffer.clone(), cx);+ }+ if context_sources.contains(&ContextSource::EditHistory) {+ collect_edit_history_context(&mut ranges_by_buffer, &edit_history, cx);+ }+ if context_sources.contains(&ContextSource::EditHistoryFile) {+ collect_edit_history_file_context(&mut ranges_by_buffer, &edit_history, cx);+ }+ if context_sources.contains(&ContextSource::GitLog) {+ collect_git_log_context(&mut ranges_by_buffer, project.clone(), active_buffer, cx).await;+ }+ if context_sources.contains(&ContextSource::OracleFile) {+ collect_oracle_file_context(&mut ranges_by_buffer, project.clone(), oracle_paths, cx).await;+ }++ Ok(cx.update(|cx| {+ let project = project.read(cx);+ let mut related_files = ranges_by_buffer+ .into_values()+ .filter_map(|(buffer, ranges)| related_file_for_ranges(&project, &buffer, ranges, cx))+ .collect::<Vec<_>>();+ related_files.sort_by_key(|file| {+ file.excerpts+ .iter()+ .map(|excerpt| excerpt.order)+ .min()+ .unwrap_or(usize::MAX)+ });+ related_files+ }))+}++pub fn limit_retrieved_context_to_bytes(+ related_files: &[RelatedFile],+ max_bytes: usize,+) -> Vec<RelatedFile> {+ struct ExcerptCandidate {+ file_index: usize,+ excerpt_index: usize,+ order: usize,+ }++ let mut candidates = related_files+ .iter()+ .enumerate()+ .flat_map(|(file_index, file)| {+ file.excerpts+ .iter()+ .enumerate()+ .map(move |(excerpt_index, excerpt)| ExcerptCandidate {+ file_index,+ excerpt_index,+ order: excerpt.order,+ })+ })+ .collect::<Vec<_>>();+ candidates.sort_by_key(|candidate| {+ (+ candidate.order,+ candidate.file_index,+ candidate.excerpt_index,+ )+ });++ let mut selected_excerpts = related_files+ .iter()+ .map(|file| vec![false; file.excerpts.len()])+ .collect::<Vec<_>>();+ let mut covered_ranges_by_file = vec![Vec::<Range<u32>>::new(); related_files.len()];+ let mut selected_bytes: usize = 0;++ for candidate in candidates {+ let file = &related_files[candidate.file_index];+ let excerpt = &file.excerpts[candidate.excerpt_index];+ let added_bytes =+ uncovered_excerpt_bytes(excerpt, &covered_ranges_by_file[candidate.file_index]);+ if added_bytes == 0 || selected_bytes.saturating_add(added_bytes) > max_bytes {+ continue;+ }++ selected_bytes += added_bytes;+ selected_excerpts[candidate.file_index][candidate.excerpt_index] = true;+ push_covered_range(+ &mut covered_ranges_by_file[candidate.file_index],+ excerpt.row_range.clone(),+ );+ }++ related_files+ .iter()+ .enumerate()+ .filter_map(|(file_index, file)| {+ let excerpts = file+ .excerpts+ .iter()+ .enumerate()+ .filter_map(|(excerpt_index, excerpt)| {+ selected_excerpts[file_index][excerpt_index].then(|| excerpt.clone())+ })+ .collect::<Vec<_>>();+ if excerpts.is_empty() {+ return None;+ }++ Some(RelatedFile {+ path: file.path.clone(),+ max_row: file.max_row,+ excerpts,+ in_open_source_repo: file.in_open_source_repo,+ })+ })+ .collect()+}++fn uncovered_excerpt_bytes(excerpt: &RelatedExcerpt, covered_ranges: &[Range<u32>]) -> usize {+ let mut bytes = 0;++ for (row, line) in (excerpt.row_range.start..).zip(excerpt.text.split_inclusive('\n')) {+ if row >= excerpt.row_range.end {+ break;+ }+ if !covered_ranges+ .iter()+ .any(|covered_range| covered_range.contains(&row))+ {+ bytes += line.len();+ }+ }++ bytes+}++fn push_covered_range(covered_ranges: &mut Vec<Range<u32>>, range: Range<u32>) {+ covered_ranges.push(range);+ covered_ranges.sort_by_key(|range| (range.start, range.end));++ let mut merged_ranges: Vec<Range<u32>> = Vec::new();+ for range in covered_ranges.drain(..) {+ if let Some(last_range) = merged_ranges.last_mut()+ && range.start <= last_range.end+ {+ last_range.end = last_range.end.max(range.end);+ continue;+ }++ merged_ranges.push(range);+ }++ *covered_ranges = merged_ranges;+}++fn collect_cursor_excerpt_context(+ ranges_by_buffer: &mut RangesByBuffer,+ active_buffer: Entity<Buffer>,+ cursor_position: Anchor,+ cx: &mut AsyncApp,+) {+ let cursor_range = active_buffer.read_with(cx, |buffer, _cx| {+ let snapshot = buffer.snapshot();+ expanded_anchor_range(+ &snapshot,+ cursor_position..cursor_position,+ CURSOR_CONTEXT_LINE_COUNT,+ )+ });++ push_context_range(+ ranges_by_buffer,+ active_buffer,+ cursor_range,+ 0,+ ContextSource::CursorExcerpt,+ );+}++fn collect_current_file_context(+ ranges_by_buffer: &mut RangesByBuffer,+ active_buffer: Entity<Buffer>,+ cx: &mut AsyncApp,+) {+ collect_full_buffer_context(+ ranges_by_buffer,+ active_buffer,+ 0,+ ContextSource::CurrentFile,+ cx,+ );+}++fn collect_edit_history_context(+ ranges_by_buffer: &mut RangesByBuffer,+ edit_history: &[EditHistoryContextEntry],+ cx: &mut AsyncApp,+) {+ for (index, entry) in edit_history.iter().enumerate() {+ let edit_history_range = entry.buffer.read_with(cx, |buffer, _cx| {+ expanded_anchor_range(+ &buffer.snapshot(),+ entry.edited_range.clone(),+ EDIT_HISTORY_CONTEXT_LINE_COUNT,+ )+ });++ push_context_range(+ ranges_by_buffer,+ entry.buffer.clone(),+ edit_history_range,+ index + 1,+ ContextSource::EditHistory,+ );+ }+}++fn collect_edit_history_file_context(+ ranges_by_buffer: &mut RangesByBuffer,+ edit_history: &[EditHistoryContextEntry],+ cx: &mut AsyncApp,+) {+ let next_order = next_context_order(ranges_by_buffer);+ let mut seen_buffers = HashSet::default();+ let mut index = 0;++ for entry in edit_history {+ if !seen_buffers.insert(entry.buffer.entity_id()) {+ continue;+ }++ collect_full_buffer_context(+ ranges_by_buffer,+ entry.buffer.clone(),+ next_order + index,+ ContextSource::EditHistoryFile,+ cx,+ );+ index += 1;+ }+}++async fn collect_oracle_file_context(+ ranges_by_buffer: &mut RangesByBuffer,+ project: Entity<Project>,+ oracle_paths: Vec<Arc<Path>>,+ cx: &mut AsyncApp,+) {+ let next_order = next_context_order(ranges_by_buffer);+ let mut seen_buffers = HashSet::default();+ let mut index = 0;++ for path in oracle_paths {+ let buffer = match open_buffer_for_path(&project, &path, cx).await {+ Ok(Some(buffer)) => buffer,+ Ok(None) => {+ log::debug!("failed to find oracle file path: {}", path.display());+ continue;+ }+ Err(error) => {+ log::debug!(+ "failed to open oracle file path {}: {error:#}",+ path.display()+ );+ continue;+ }+ };++ if !seen_buffers.insert(buffer.entity_id()) {+ continue;+ }++ collect_full_buffer_context(+ ranges_by_buffer,+ buffer,+ next_order + index,+ ContextSource::OracleFile,+ cx,+ );+ index += 1;+ }+}++async fn open_buffer_for_path(+ project: &Entity<Project>,+ path: &Path,+ cx: &mut AsyncApp,+) -> anyhow::Result<Option<Entity<Buffer>>> {+ let path = path.to_path_buf();+ let path_without_prefix: PathBuf = path.components().skip(1).collect();+ let project_path = project.update(cx, |project, cx| {+ project.find_project_path(&path, cx).or_else(|| {+ if path_without_prefix.as_os_str().is_empty() {+ None+ } else {+ project.find_project_path(&path_without_prefix, cx)+ }+ })+ });++ let Some(project_path) = project_path else {+ return Ok(None);+ };++ project+ .update(cx, |project, cx| project.open_buffer(project_path, cx))+ .await+ .map(Some)+}++fn collect_full_buffer_context(+ ranges_by_buffer: &mut RangesByBuffer,+ buffer: Entity<Buffer>,+ order: usize,+ context_source: ContextSource,+ cx: &mut AsyncApp,+) {+ let range = buffer.read_with(cx, |buffer, _cx| full_file_anchor_range(&buffer.snapshot()));+ push_context_range(ranges_by_buffer, buffer, range, order, context_source);+}++fn full_file_anchor_range(snapshot: &BufferSnapshot) -> Range<Anchor> {+ let start = snapshot.anchor_before(Point::new(0, 0));+ let max_point = snapshot.max_point();+ let end = snapshot.anchor_after(max_point);+ start..end+}++fn next_context_order(ranges_by_buffer: &RangesByBuffer) -> usize {+ ranges_by_buffer+ .values()+ .flat_map(|(_, ranges)| ranges.iter().map(|range| range.order))+ .max()+ .map_or(0, |order| order + 1)+}++async fn collect_git_log_context(+ ranges_by_buffer: &mut RangesByBuffer,+ project: Entity<Project>,+ active_buffer: Entity<Buffer>,+ cx: &mut AsyncApp,+) {+ let Some((worktree_id, active_path, worktree_abs_path)) = cx.update(|cx| {+ let buffer = active_buffer.read(cx);+ let file = buffer.file()?;+ let project = project.read(cx);+ if !project.is_local() {+ return None;+ }+ let worktree = project.worktree_for_id(file.worktree_id(cx), cx)?;+ let worktree = worktree.read(cx);+ if !worktree.is_local() {+ return None;+ }+ Some((+ file.worktree_id(cx),+ file.path().clone(),+ worktree.abs_path(),+ ))+ }) else {+ return;+ };++ let index_result = cx+ .background_spawn(async move { build_git_log_index(&worktree_abs_path).await })+ .await;+ let index = match index_result {+ Ok(index) => index,+ Err(error) => {+ log::debug!("failed to build git log context index: {error:#}");+ return;+ }+ };++ let next_order = next_context_order(ranges_by_buffer);++ for (index, related_path) in index+ .get_related(active_path.as_std_path(), GIT_LOG_CONTEXT_FILE_COUNT)+ .into_iter()+ .enumerate()+ {+ let Ok(related_path) = RelPath::new(&related_path, PathStyle::Posix) else {+ continue;+ };+ let project_path = ProjectPath {+ worktree_id,+ path: related_path.into_owned().into(),+ };+ let buffer = match project+ .update(cx, |project, cx| project.open_buffer(project_path, cx))+ .await+ {+ Ok(buffer) => buffer,+ Err(error) => {+ log::debug!("failed to open git log related buffer: {error:#}");+ continue;+ }+ };++ let range = buffer.read_with(cx, |buffer, _cx| {+ let snapshot = buffer.snapshot();+ let max_row = GIT_LOG_CONTEXT_LINE_COUNT.min(snapshot.max_point().row);+ let end = snapshot.anchor_after(Point::new(max_row, snapshot.line_len(max_row)));+ snapshot.anchor_before(Point::new(0, 0))..end+ });++ push_context_range(+ ranges_by_buffer,+ buffer,+ range,+ next_order + index,+ ContextSource::GitLog,+ );+ }+}++fn expanded_anchor_range(+ snapshot: &BufferSnapshot,+ range: Range<Anchor>,+ context_line_count: u32,+) -> Range<Anchor> {+ let start = range.start.to_point(snapshot);+ let end = range.end.to_point(snapshot);+ let start_row = start.row.saturating_sub(context_line_count);+ let end_row = end+ .row+ .saturating_add(context_line_count)+ .min(snapshot.max_point().row);+ let start = snapshot.anchor_before(Point::new(start_row, 0));+ let end = snapshot.anchor_after(Point::new(end_row, snapshot.line_len(end_row)));+ start..end+}++fn push_context_range(+ ranges_by_buffer: &mut RangesByBuffer,+ buffer: Entity<Buffer>,+ range: Range<Anchor>,+ order: usize,+ context_source: ContextSource,+) {+ ranges_by_buffer+ .entry(buffer.entity_id())+ .or_insert_with(|| (buffer.clone(), Vec::new()))+ .1+ .push(EditableContextRange {+ range,+ order,+ context_source,+ });+}++fn related_file_for_ranges(+ project: &Project,+ buffer: &Entity<Buffer>,+ ranges: Vec<EditableContextRange>,+ cx: &App,+) -> Option<RelatedFile> {+ let buffer = buffer.read(cx);+ let snapshot = buffer.snapshot();+ let file = snapshot.file()?;+ let worktree = project.worktree_for_id(file.worktree_id(cx), cx)?;+ let path: Arc<Path> = Path::new(&format!(+ "{}/{}",+ worktree.read(cx).root_name().as_unix_str(),+ file.path().as_unix_str()+ ))+ .into();++ let ranges = resolved_context_ranges(ranges, &snapshot);++ let mut excerpts = ranges+ .into_iter()+ .map(|range| RelatedExcerpt {+ row_range: range.range.start.row..range.range.end.row,+ text: snapshot+ .text_for_range(range.range)+ .collect::<String>()+ .into(),+ order: range.order,+ context_source: range.context_source,+ })+ .collect::<Vec<_>>();+ excerpts.sort_by_key(|excerpt| excerpt.order);++ Some(RelatedFile {+ path,+ max_row: snapshot.max_point().row,+ excerpts,+ in_open_source_repo: false,+ })+}++fn resolved_context_ranges(+ ranges: Vec<EditableContextRange>,+ snapshot: &BufferSnapshot,+) -> Vec<ResolvedEditableContextRange> {+ ranges+ .into_iter()+ .filter_map(|range| {+ let start = range.range.start.to_point(snapshot);+ let end = range.range.end.to_point(snapshot);+ if start >= end {+ return None;+ }++ Some(ResolvedEditableContextRange {+ range: start..end,+ order: range.order,+ context_source: range.context_source,+ })+ })+ .collect()+}++#[allow(dead_code)]+fn merge_overlapping_ranges(ranges: &mut Vec<ResolvedEditableContextRange>) {+ ranges.sort_by_key(|range| (range.range.start, range.range.end));+ let mut merged: Vec<ResolvedEditableContextRange> = Vec::new();++ for range in ranges.drain(..) {+ if let Some(last_range) = merged.last_mut()+ && range.range.start <= last_range.range.end+ {+ if context_source_order(range.context_source)+ < context_source_order(last_range.context_source)+ {+ last_range.context_source = range.context_source;+ }+ last_range.range.end = last_range.range.end.max(range.range.end);+ last_range.order = last_range.order.min(range.order);+ continue;+ }++ merged.push(range);+ }++ *ranges = merged;+}++#[allow(dead_code)]+fn push_context_source(context_sources: &mut Vec<ContextSource>, context_source: ContextSource) {+ if !context_sources.contains(&context_source) {+ context_sources.push(context_source);+ context_sources.sort_by_key(|context_source| context_source_order(*context_source));+ }+}++fn context_source_order(context_source: ContextSource) -> usize {+ match context_source {+ ContextSource::Lsp => 0,+ ContextSource::CursorExcerpt => 1,+ ContextSource::CurrentFile => 2,+ ContextSource::EditHistory => 3,+ ContextSource::EditHistoryFile => 4,+ ContextSource::GitLog => 5,+ ContextSource::OracleFile => 6,+ }+}diff --git a/crates/edit_prediction_context/src/git_log_context.rs b/crates/edit_prediction_context/src/git_log_context.rsnew file mode 100644index 00000000000..2ed358b03fb--- /dev/null+++ b/crates/edit_prediction_context/src/git_log_context.rs@@ -0,0 +1,255 @@+// Goal:+// - Build an index that finds files that are frequently edited in the same git commit+// - Lookup by path and get a list of related files, sorted by most frequently edited together+//+// Path => Path => usize+//+// This is a symmetric relationship, so for a => (b, 1), also add b => (a, 1)++use std::collections::HashMap;+use std::env;+use std::path::{Path, PathBuf};+use std::process::ExitCode;++use anyhow::{Context as _, Result, anyhow, bail};+use util::command::new_command;++pub struct GitLogIndex {+ index: HashMap<PathBuf, HashMap<PathBuf, usize>>,+}++impl GitLogIndex {+ pub fn new() -> Self {+ Self {+ index: HashMap::new(),+ }+ }++ pub fn add_related(&mut self, path: PathBuf, related: PathBuf) {+ let count = self+ .index+ .entry(path.clone())+ .or_default()+ .entry(related.clone())+ .or_default();+ *count += 1;++ // add the reverse mapping+ let reverse_count = self+ .index+ .entry(related)+ .or_default()+ .entry(path)+ .or_default();+ *reverse_count += 1;+ }++ pub fn get_related(&self, path: &Path, n: usize) -> Vec<PathBuf> {+ self.get_related_with_counts(path, n)+ .into_iter()+ .map(|(path, _)| path)+ .collect()+ }++ pub fn get_related_with_counts(&self, path: &Path, n: usize) -> Vec<(PathBuf, usize)> {+ let Some(counts) = self.index.get(path) else {+ return Vec::new();+ };++ let mut related: Vec<_> = counts.iter().collect();+ related.sort_by(|(left_path, left_count), (right_path, right_count)| {+ right_count+ .cmp(left_count)+ .then_with(|| left_path.cmp(right_path))+ });+ related+ .into_iter()+ .take(n)+ .map(|(path, count)| (path.clone(), *count))+ .collect()+ }+}++impl Default for GitLogIndex {+ fn default() -> Self {+ Self::new()+ }+}++pub async fn build_git_log_index(worktree_dir: &Path) -> Result<GitLogIndex> {+ let mut index = GitLogIndex::new();++ let output = new_command("git")+ .arg("log")+ .arg("-5000")+ .arg("--pretty=tformat:@@COMMIT %H")+ .arg("--name-only")+ .current_dir(worktree_dir)+ .output()+ .await+ .with_context(|| format!("failed to run git log in {}", worktree_dir.display()))?;++ if !output.status.success() {+ let stderr = String::from_utf8_lossy(&output.stderr);+ bail!(+ "git log failed in {} with status {}: {}",+ worktree_dir.display(),+ output.status,+ stderr.trim()+ );+ }++ let log = String::from_utf8(output.stdout).context("git log output was not valid UTF-8")?;+ let parsed = parse_git_log(&log);+ for files in parsed {+ for i in 0..files.len() {+ for j in (i + 1)..files.len() {+ index.add_related(files[i].clone(), files[j].clone());+ }+ }+ }++ Ok(index)+}++#[allow(dead_code)]+fn main() -> ExitCode {+ match run() {+ Ok(()) => ExitCode::SUCCESS,+ Err(error) => {+ eprintln!("{error:#}");+ ExitCode::FAILURE+ }+ }+}++#[allow(dead_code)]+fn run() -> Result<()> {+ let mut arguments = env::args_os();+ let program_name = arguments+ .next()+ .and_then(|path| PathBuf::from(path).file_name().map(|name| name.to_owned()))+ .and_then(|name| name.into_string().ok())+ .unwrap_or_else(|| "git_log_context".to_string());++ let worktree_dir = arguments.next().ok_or_else(|| {+ print_usage(&program_name);+ anyhow!("missing worktree path")+ })?;+ let query_path = arguments.next().ok_or_else(|| {+ print_usage(&program_name);+ anyhow!("missing query path")+ })?;+ if arguments.next().is_some() {+ print_usage(&program_name);+ bail!("too many arguments");+ }++ let worktree_dir = PathBuf::from(worktree_dir);+ let query_path = normalize_query_path(&worktree_dir, &PathBuf::from(query_path));+ let index = futures::executor::block_on(build_git_log_index(&worktree_dir))?;++ for (path, count) in index.get_related_with_counts(&query_path, 10) {+ println!("{count}\t{}", path.display());+ }++ Ok(())+}++#[allow(dead_code)]+fn print_usage(program_name: &str) {+ eprintln!("Usage: {program_name} <worktree-path> <query-path>");+}++#[allow(dead_code)]+fn normalize_query_path(worktree_dir: &Path, query_path: &Path) -> PathBuf {+ if query_path.is_absolute() {+ query_path+ .strip_prefix(worktree_dir)+ .unwrap_or(query_path)+ .components()+ .collect()+ } else {+ query_path.components().collect()+ }+}++fn parse_git_log(log: &str) -> Vec<Vec<PathBuf>> {+ let mut lines = log.lines().peekable();+ let mut commits = Vec::new();++ while let Some(line) = lines.next() {+ if line.starts_with("@@COMMIT ") {+ // skip blank line+ lines.next();+ let mut files = Vec::new();+ while let Some(next) = lines.peek()+ && !next.starts_with("@@COMMIT ")+ {+ let Some(next) = lines.next() else {+ break;+ };+ if !next.is_empty() {+ files.push(next.into());+ }+ }+ commits.push(files);+ }+ }++ commits+}++#[cfg(test)]+mod tests {+ use super::*;+ use indoc::indoc;++ #[test]+ fn test_git_log_index() {+ let mut index = GitLogIndex::new();+ index.add_related(PathBuf::from("a"), PathBuf::from("b"));+ index.add_related(PathBuf::from("a"), PathBuf::from("b"));+ index.add_related(PathBuf::from("a"), PathBuf::from("c"));+ index.add_related(PathBuf::from("b"), PathBuf::from("c"));++ let related = index.get_related(&PathBuf::from("a"), 100);+ assert_eq!(related, vec![PathBuf::from("b"), PathBuf::from("c")]);+ }++ #[test]+ fn test_parse_git_log() {+ let log = indoc! {"+ @@COMMIT d2e451dd48be67ef8c943e90dabc02e80a6984c9++ crates/edit_prediction/src/edit_prediction.rs+ crates/edit_prediction_cli/src/format_prompt.rs+ crates/edit_prediction_cli/src/main.rs+ crates/edit_prediction_cli/src/predict.rs+ @@COMMIT d666823f348bd151067464fa31676d28bdb96717++ crates/edit_prediction_cli/src/main.rs+ crates/edit_prediction_cli/src/predict.rs+ "};+ let parsed = parse_git_log(log);++ assert_eq!(parsed.len(), 2);+ assert_eq!(+ parsed[0][0],+ PathBuf::from("crates/edit_prediction/src/edit_prediction.rs")+ );+ assert_eq!(+ parsed[0][1],+ PathBuf::from("crates/edit_prediction_cli/src/format_prompt.rs")+ );++ assert_eq!(+ parsed[1][0],+ PathBuf::from("crates/edit_prediction_cli/src/main.rs")+ );+ assert_eq!(+ parsed[1][1],+ PathBuf::from("crates/edit_prediction_cli/src/predict.rs")+ );+ }+}diff --git a/crates/edit_prediction_metrics/src/edit_prediction_metrics.rs b/crates/edit_prediction_metrics/src/edit_prediction_metrics.rsindex 81a37148c79..205075e2f5b 100644--- a/crates/edit_prediction_metrics/src/edit_prediction_metrics.rs+++ b/crates/edit_prediction_metrics/src/edit_prediction_metrics.rs@@ -1,4 +1,6 @@+mod jumps;mod kept_rate;+mod patch;mod patch_metrics;mod prediction_score;mod reversal;@@ -7,6 +9,7 @@ mod tokenize;#[cfg(feature = "tree-sitter")]mod tree_sitter;+pub use jumps::{EditableContextCoverage, Excerpt, editable_context_coverage};pub use kept_rate::AnnotatedToken;pub use kept_rate::KeptRateResult;pub use kept_rate::TokenAnnotation;diff --git a/crates/edit_prediction_metrics/src/jumps.rs b/crates/edit_prediction_metrics/src/jumps.rsnew file mode 100644index 00000000000..8cfeafcc97a--- /dev/null+++ b/crates/edit_prediction_metrics/src/jumps.rs@@ -0,0 +1,505 @@+use std::{+ collections::{BTreeMap, BTreeSet},+ ops::Range,+};++use crate::patch::{Patch, PatchLine};++const LINE_RELEVANCE_WINDOW: u32 = 20;++#[derive(Clone, Debug, Eq, PartialEq, serde::Deserialize, serde::Serialize)]+pub struct Excerpt {+ pub path: String,+ pub row_range: Range<u32>,+ pub content: String,+}++#[derive(Clone, Debug, PartialEq, serde::Deserialize, serde::Serialize)]+pub struct EditableContextCoverage {+ pub lines_tp: usize,+ pub lines_fp: usize,+ pub lines_fn: usize,+ pub lines_precision: f64,+ pub lines_recall: f64,+ pub lines_f1: f64,++ pub files_tp: usize,+ pub files_fp: usize,+ pub files_fn: usize,+ pub files_precision: f64,+ pub files_recall: f64,+ pub files_f1: f64,+}++impl EditableContextCoverage {+ pub fn new(+ lines_tp: usize,+ lines_fp: usize,+ lines_fn: usize,+ files_tp: usize,+ files_fp: usize,+ files_fn: usize,+ ) -> Self {+ Self {+ lines_tp,+ lines_fp,+ lines_fn,+ lines_precision: precision(lines_tp, lines_fp, lines_fn),+ lines_recall: recall(lines_tp, lines_fp, lines_fn),+ lines_f1: f1(lines_tp, lines_fp, lines_fn),+ files_tp,+ files_fp,+ files_fn,+ files_precision: precision(files_tp, files_fp, files_fn),+ files_recall: recall(files_tp, files_fp, files_fn),+ files_f1: f1(files_tp, files_fp, files_fn),+ }+ }+}++/// Measures how much expected edit context was retrieved and how much unrelated context was retrieved.+pub fn editable_context_coverage(+ expected_patch: &str,+ context: &[Excerpt],+) -> EditableContextCoverage {+ let patch = Patch::parse_unified_diff(expected_patch);+ let (expected_files, expected_anchor_lines, relevant_lines) = expected_context(&patch);+ let (retrieved_files, retrieved_lines) = retrieved_context(context);++ let (lines_tp, lines_fp) = classify_retrieved_rows(&relevant_lines, &retrieved_lines);+ let (lines_fn, expected_anchor_line_count) =+ count_missing_rows(&expected_anchor_lines, &retrieved_lines);+ let (files_tp, files_fp, files_fn) = classify_values(&expected_files, &retrieved_files);++ let lines_precision = precision(lines_tp, lines_fp, lines_fn);+ let lines_recall = line_recall(expected_anchor_line_count, lines_fn);+ let lines_f1 = f1_from_precision_and_recall(lines_precision, lines_recall);+ let files_precision = precision(files_tp, files_fp, files_fn);+ let files_recall = recall(files_tp, files_fp, files_fn);+ let files_f1 = f1_from_precision_and_recall(files_precision, files_recall);++ EditableContextCoverage {+ lines_tp,+ lines_fp,+ lines_fn,+ lines_precision,+ lines_recall,+ lines_f1,+ files_tp,+ files_fp,+ files_fn,+ files_precision,+ files_recall,+ files_f1,+ }+}++fn expected_context(+ patch: &Patch,+) -> (+ BTreeSet<String>,+ BTreeMap<String, BTreeSet<u32>>,+ BTreeMap<String, BTreeSet<u32>>,+) {+ let mut expected_files = BTreeSet::new();+ let mut expected_anchor_lines = BTreeMap::new();+ let mut relevant_lines = BTreeMap::new();++ for hunk in &patch.hunks {+ if hunk+ .lines+ .iter()+ .any(|line| matches!(line, PatchLine::Addition(_) | PatchLine::Deletion(_)))+ {+ expected_files.insert(hunk.filename.clone());+ }++ let mut old_row = hunk.old_start.saturating_sub(1).max(0) as u32;+ let mut previous_context_row = None;+ let mut index = 0;++ while index < hunk.lines.len() {+ match &hunk.lines[index] {+ PatchLine::Context(_) => {+ previous_context_row = Some(old_row);+ old_row = old_row.saturating_add(1);+ index += 1;+ }+ PatchLine::Addition(_) | PatchLine::Deletion(_) => {+ let mut deletion_rows = Vec::new();+ let mut has_addition = false;++ while index < hunk.lines.len() {+ match &hunk.lines[index] {+ PatchLine::Addition(_) => {+ has_addition = true;+ index += 1;+ }+ PatchLine::Deletion(_) => {+ deletion_rows.push(old_row);+ old_row = old_row.saturating_add(1);+ index += 1;+ }+ _ => break,+ }+ }++ if deletion_rows.is_empty() {+ if has_addition {+ if let Some(row) = previous_context_row {+ insert_anchor_row(+ &mut expected_anchor_lines,+ &mut relevant_lines,+ &hunk.filename,+ row,+ );+ }+ if matches!(hunk.lines.get(index), Some(PatchLine::Context(_))) {+ insert_anchor_row(+ &mut expected_anchor_lines,+ &mut relevant_lines,+ &hunk.filename,+ old_row,+ );+ }+ }+ } else {+ for row in deletion_rows {+ insert_anchor_row(+ &mut expected_anchor_lines,+ &mut relevant_lines,+ &hunk.filename,+ row,+ );+ }+ }++ previous_context_row = None;+ }+ PatchLine::Garbage(_) => {+ index += 1;+ }+ }+ }+ }++ (expected_files, expected_anchor_lines, relevant_lines)+}++fn retrieved_context(context: &[Excerpt]) -> (BTreeSet<String>, BTreeMap<String, BTreeSet<u32>>) {+ let mut retrieved_files = BTreeSet::new();+ let mut retrieved_lines = BTreeMap::new();++ for excerpt in context {+ retrieved_files.insert(excerpt.path.clone());+ let rows = retrieved_lines+ .entry(excerpt.path.clone())+ .or_insert_with(BTreeSet::new);+ rows.extend(excerpt.row_range.clone());+ }++ (retrieved_files, retrieved_lines)+}++fn insert_anchor_row(+ anchor_lines_by_file: &mut BTreeMap<String, BTreeSet<u32>>,+ relevant_lines_by_file: &mut BTreeMap<String, BTreeSet<u32>>,+ path: &str,+ row: u32,+) {+ insert_row(anchor_lines_by_file, path, row);++ let start = row.saturating_sub(LINE_RELEVANCE_WINDOW);+ let end = row.saturating_add(LINE_RELEVANCE_WINDOW);+ for relevant_row in start..=end {+ insert_row(relevant_lines_by_file, path, relevant_row);+ }+}++fn insert_row(lines_by_file: &mut BTreeMap<String, BTreeSet<u32>>, path: &str, row: u32) {+ lines_by_file+ .entry(path.to_string())+ .or_insert_with(BTreeSet::new)+ .insert(row);+}++fn classify_retrieved_rows(+ relevant: &BTreeMap<String, BTreeSet<u32>>,+ retrieved: &BTreeMap<String, BTreeSet<u32>>,+) -> (usize, usize) {+ let mut true_positives = 0;+ let mut false_positives = 0;++ for (path, rows) in retrieved {+ for row in rows {+ if relevant+ .get(path)+ .is_some_and(|relevant_rows| relevant_rows.contains(row))+ {+ true_positives += 1;+ } else {+ false_positives += 1;+ }+ }+ }++ (true_positives, false_positives)+}++fn count_missing_rows(+ expected: &BTreeMap<String, BTreeSet<u32>>,+ retrieved: &BTreeMap<String, BTreeSet<u32>>,+) -> (usize, usize) {+ let mut false_negatives = 0;+ let mut expected_count = 0;++ for (path, rows) in expected {+ for row in rows {+ expected_count += 1;+ if !retrieved+ .get(path)+ .is_some_and(|retrieved_rows| retrieved_rows.contains(row))+ {+ false_negatives += 1;+ }+ }+ }++ (false_negatives, expected_count)+}++fn classify_values<T: Ord>(+ expected: &BTreeSet<T>,+ retrieved: &BTreeSet<T>,+) -> (usize, usize, usize) {+ let true_positives = expected.intersection(retrieved).count();+ let false_positives = retrieved.difference(expected).count();+ let false_negatives = expected.difference(retrieved).count();+ (true_positives, false_positives, false_negatives)+}++fn precision(true_positives: usize, false_positives: usize, false_negatives: usize) -> f64 {+ if true_positives + false_positives + false_negatives == 0 {+ return 1.0;+ }++ let denominator = true_positives + false_positives;+ if denominator == 0 {+ 1.0+ } else {+ true_positives as f64 / denominator as f64+ }+}++fn recall(true_positives: usize, false_positives: usize, false_negatives: usize) -> f64 {+ if true_positives + false_positives + false_negatives == 0 {+ return 1.0;+ }++ let denominator = true_positives + false_negatives;+ if denominator == 0 {+ 1.0+ } else {+ true_positives as f64 / denominator as f64+ }+}++fn line_recall(expected_anchor_line_count: usize, false_negatives: usize) -> f64 {+ if expected_anchor_line_count == 0 {+ 1.0+ } else {+ (expected_anchor_line_count - false_negatives) as f64 / expected_anchor_line_count as f64+ }+}++fn f1(true_positives: usize, false_positives: usize, false_negatives: usize) -> f64 {+ let precision = precision(true_positives, false_positives, false_negatives);+ let recall = recall(true_positives, false_positives, false_negatives);++ f1_from_precision_and_recall(precision, recall)+}++fn f1_from_precision_and_recall(precision: f64, recall: f64) -> f64 {+ if precision + recall == 0.0 {+ 0.0+ } else {+ 2.0 * precision * recall / (precision + recall)+ }+}++#[cfg(test)]+mod tests {+ use super::*;+ use indoc::indoc;++ fn excerpt(path: &str, row_range: Range<u32>) -> Excerpt {+ Excerpt {+ path: path.to_string(),+ row_range,+ content: String::new(),+ }+ }++ #[test]+ fn deletion_counts_deleted_old_line_as_true_positive() {+ let patch = indoc! {"+ --- a/src/main.rs+ +++ b/src/main.rs+ @@ -2,1 +2,0 @@+ -let value = 1;+ "};++ let score = editable_context_coverage(patch, &[excerpt("src/main.rs", 1..2)]);++ assert_eq!(score, EditableContextCoverage::new(1, 0, 0, 1, 0, 0));+ }++ #[test]+ fn retrieved_lines_inside_relevance_window_are_true_positives() {+ let patch = indoc! {"+ --- a/src/main.rs+ +++ b/src/main.rs+ @@ -4,1 +4,0 @@+ -let value = 4;+ "};++ let score = editable_context_coverage(+ patch,+ &[excerpt("src/main.rs", 0..1), excerpt("src/main.rs", 3..4)],+ );++ assert_eq!(score, EditableContextCoverage::new(2, 0, 0, 1, 0, 0));+ }++ #[test]+ fn replacement_counts_deleted_old_line_without_addition_anchor() {+ let patch = indoc! {"+ --- a/src/main.rs+ +++ b/src/main.rs+ @@ -1,3 +1,3 @@+ fn main() {+ - let value = 1;+ + let value = 2;+ }+ "};++ let score = editable_context_coverage(patch, &[excerpt("src/main.rs", 1..2)]);++ assert_eq!(score, EditableContextCoverage::new(1, 0, 0, 1, 0, 0));+ }++ #[test]+ fn pure_insertion_counts_previous_and_next_old_lines_as_expected_context() {+ let patch = indoc! {"+ --- a/src/main.rs+ +++ b/src/main.rs+ @@ -1,2 +1,3 @@+ line 1+ +inserted+ line 2+ "};++ let score = editable_context_coverage(patch, &[excerpt("src/main.rs", 0..1)]);++ assert_eq!(score, EditableContextCoverage::new(1, 0, 1, 1, 0, 0));+ }++ #[test]+ fn pure_insertion_at_file_boundary_uses_available_neighboring_context() {+ let patch = indoc! {"+ --- a/src/main.rs+ +++ b/src/main.rs+ @@ -1,1 +1,2 @@+ +inserted+ line 1+ "};++ let score = editable_context_coverage(patch, &[excerpt("src/main.rs", 0..1)]);++ assert_eq!(score, EditableContextCoverage::new(1, 0, 0, 1, 0, 0));+ }++ #[test]+ fn counts_false_negatives_and_file_false_positives() {+ let patch = indoc! {"+ --- a/src/main.rs+ +++ b/src/main.rs+ @@ -1,3 +1,3 @@+ -let first = 1;+ +let first = 2;+ let middle = 3;+ -let last = 4;+ +let last = 5;+ "};++ let score = editable_context_coverage(+ patch,+ &[excerpt("src/main.rs", 0..1), excerpt("src/lib.rs", 0..1)],+ );++ assert_eq!(score, EditableContextCoverage::new(1, 1, 1, 1, 1, 0));+ }++ #[test]+ fn overlapping_excerpts_are_counted_once() {+ let patch = indoc! {"+ --- a/src/main.rs+ +++ b/src/main.rs+ @@ -2,1 +2,0 @@+ -let value = 1;+ "};++ let score = editable_context_coverage(+ patch,+ &[excerpt("src/main.rs", 0..2), excerpt("src/main.rs", 1..3)],+ );++ assert_eq!(score, EditableContextCoverage::new(3, 0, 0, 1, 0, 0));+ }++ #[test]+ fn nearby_lines_do_not_satisfy_line_recall_without_exact_anchor_lines() {+ let patch = indoc! {"+ --- a/src/main.rs+ +++ b/src/main.rs+ @@ -1,2 +1,3 @@+ line 1+ +inserted+ line 2+ "};++ let score = editable_context_coverage(patch, &[excerpt("src/main.rs", 2..3)]);++ assert_eq!(score.lines_tp, 1);+ assert_eq!(score.lines_fp, 0);+ assert_eq!(score.lines_fn, 2);+ assert_eq!(score.lines_precision, 1.0);+ assert_eq!(score.lines_recall, 0.0);+ assert_eq!(score.lines_f1, 0.0);+ }++ #[test]+ fn retrieved_lines_outside_relevance_window_are_false_positives() {+ let patch = indoc! {"+ --- a/src/main.rs+ +++ b/src/main.rs+ @@ -1,1 +1,0 @@+ -line 1+ "};++ let score = editable_context_coverage(patch, &[excerpt("src/main.rs", 21..22)]);++ assert_eq!(score, EditableContextCoverage::new(0, 1, 1, 1, 0, 0));+ }++ #[test]+ fn empty_patch_with_no_context_has_perfect_f1() {+ let score = editable_context_coverage(+ indoc! {"+ "},+ &[],+ );++ assert_eq!(score, EditableContextCoverage::new(0, 0, 0, 0, 0, 0));+ }+}diff --git a/crates/edit_prediction_metrics/src/main.rs b/crates/edit_prediction_metrics/src/main.rsindex 0e557c35e7f..c7c55ceff60 100644--- a/crates/edit_prediction_metrics/src/main.rs+++ b/crates/edit_prediction_metrics/src/main.rs@@ -5,10 +5,10 @@ use std::path::Path;use std::process;use edit_prediction_metrics::{- ClassificationMetrics, DeltaChrFMetrics, KeptRateResult, TokenAnnotation,- annotate_kept_rate_tokens, braces_disbalance, compute_kept_rate, count_patch_token_changes,- delta_chr_f, exact_lines_match, extract_changed_lines_from_diff,- has_isolated_whitespace_changes, is_editable_region_correct,+ ClassificationMetrics, DeltaChrFMetrics, EditableContextCoverage, Excerpt, KeptRateResult,+ TokenAnnotation, annotate_kept_rate_tokens, braces_disbalance, compute_kept_rate,+ count_patch_token_changes, delta_chr_f, editable_context_coverage, exact_lines_match,+ extract_changed_lines_from_diff, has_isolated_whitespace_changes, is_editable_region_correct,};use serde::Deserialize;@@ -41,10 +41,18 @@ fn run() -> Result<(), String> {let actual_patch = fs::read_to_string(&actual_patch_path).map_err(|err| format!("failed to read {}: {err}", actual_patch_path.display()))?;- let expected = apply_patch_to_excerpt(&base, &expected_patch, 0)?;- let actual = apply_patch_to_excerpt(&base, &actual_patch, 0)?;-- EvaluationReport::new(base, expected_patch, actual_patch, expected, actual)+ let expected = apply_patch_to_excerpt(&base, &expected_patch, 0, None)?;+ let actual = apply_patch_to_excerpt(&base, &actual_patch, 0, None)?;+ let context = [];++ EvaluationReport::new(+ base,+ expected_patch,+ actual_patch,+ expected,+ actual,+ &context,+ )}CliInput::Json {json_path,@@ -55,26 +63,7 @@ fn run() -> Result<(), String> {let example: JsonExample = serde_json::from_str(&json).map_err(|err| format!("failed to parse {}: {err}", json_path.display()))?;- let base = example.prompt_inputs.cursor_excerpt;- let excerpt_start_row = example.prompt_inputs.excerpt_start_row;- let expected_patch = example- .expected_patches- .into_iter()- .next()- .ok_or_else(|| "JSON input is missing expected_patches[0]".to_string())?;- let actual_patch = example- .predictions- .into_iter()- .nth(prediction_index)- .ok_or_else(|| {- format!("JSON input does not contain predictions[{prediction_index}]")- })?- .actual_patch;-- let expected = apply_patch_to_excerpt(&base, &expected_patch, excerpt_start_row)?;- let actual = apply_patch_to_excerpt(&base, &actual_patch, excerpt_start_row)?;-- EvaluationReport::new(base, expected_patch, actual_patch, expected, actual)+ report_from_json_example(example, prediction_index)?}};@@ -82,6 +71,77 @@ fn run() -> Result<(), String> {Ok(())}+fn get_context_excerpts(example: &JsonExample) -> Vec<Excerpt> {+ let mut context = vec![get_cursor_excerpt(example)];++ if let Some(related) = &example.prompt_inputs.related_files {+ context.extend(related.iter().flat_map(|file| {+ file.excerpts.iter().map(|excerpt| Excerpt {+ path: file.path.clone(),+ row_range: excerpt.row_range.clone(),+ content: excerpt.text.clone(),+ })+ }));+ }++ context+}++fn get_cursor_excerpt(example: &JsonExample) -> Excerpt {+ let content = example.prompt_inputs.cursor_excerpt.clone();+ let start_row = example.prompt_inputs.excerpt_start_row;+ let rows = content.lines().count() as u32;+ let row_range = start_row..start_row + rows;+ Excerpt {+ path: example.cursor_path.clone(),+ row_range,+ content,+ }+}++fn report_from_json_example(+ example: JsonExample,+ prediction_index: usize,+) -> Result<EvaluationReport, String> {+ let context = get_context_excerpts(&example);+ let excerpt_start_row = example.prompt_inputs.excerpt_start_row;+ let cursor_path = example.cursor_path;+ let base = example.prompt_inputs.cursor_excerpt;+ let expected_patch = example+ .expected_patches+ .into_iter()+ .next()+ .ok_or_else(|| "JSON input is missing expected_patches[0]".to_string())?;+ let actual_patch = if example.predictions.is_empty() {+ String::new()+ } else {+ example+ .predictions+ .into_iter()+ .nth(prediction_index)+ .ok_or_else(|| format!("JSON input does not contain predictions[{prediction_index}]"))?+ .actual_patch+ };++ let expected = apply_patch_to_excerpt(+ &base,+ &expected_patch,+ excerpt_start_row,+ Some(&cursor_path),+ )?;+ let actual =+ apply_patch_to_excerpt(&base, &actual_patch, excerpt_start_row, Some(&cursor_path))?;++ Ok(EvaluationReport::new(+ base,+ expected_patch,+ actual_patch,+ expected,+ actual,+ &context,+ ))+}+fn print_usage() {eprintln!("Usage:\n edit_prediction_metrics --base <base.txt> --expected-patch <expected.diff> --actual-patch <actual.diff>\n edit_prediction_metrics --json <example.json> [--prediction-index <n>]"@@ -199,6 +259,7 @@ struct EvaluationReport {editable_region_correct: bool,expected_braces_disbalance: usize,actual_braces_disbalance: usize,+ editable_context_coverage: EditableContextCoverage,}impl EvaluationReport {@@ -208,6 +269,7 @@ impl EvaluationReport {actual_patch: String,expected: String,actual: String,+ context: &[Excerpt],) -> Self {let kept_rate = compute_kept_rate(&base, &actual, &expected);let exact_lines = exact_lines_match(&expected_patch, &actual_patch);@@ -223,6 +285,7 @@ impl EvaluationReport {let editable_region_correct = is_editable_region_correct(&actual_patch);let expected_braces_disbalance = braces_disbalance(&expected);let actual_braces_disbalance = braces_disbalance(&actual);+ let editable_context_coverage = editable_context_coverage(&expected_patch, context);Self {base,@@ -238,6 +301,7 @@ impl EvaluationReport {editable_region_correct,expected_braces_disbalance,actual_braces_disbalance,+ editable_context_coverage,}}}@@ -319,6 +383,27 @@ fn print_report(report: &EvaluationReport) {println!();print_kept_rate_explanation(&report.base, &report.actual, &report.expected);++ println!("Jumps metrics");+ println!("-------------");+ println!(+ "Editable context lines: P={}%, R={}%, F1={}% (tp: {}, fp: {}, fn: {})",+ (report.editable_context_coverage.lines_precision * 100.0).round(),+ (report.editable_context_coverage.lines_recall * 100.0).round(),+ (report.editable_context_coverage.lines_f1 * 100.0).round(),+ report.editable_context_coverage.lines_tp,+ report.editable_context_coverage.lines_fp,+ report.editable_context_coverage.lines_fn+ );+ println!(+ "Editable context files: P={}%, R={}%, F1={}% (tp: {}, fp: {}, fn: {})",+ (report.editable_context_coverage.files_precision * 100.0).round(),+ (report.editable_context_coverage.files_recall * 100.0).round(),+ (report.editable_context_coverage.files_f1 * 100.0).round(),+ report.editable_context_coverage.files_tp,+ report.editable_context_coverage.files_fp,+ report.editable_context_coverage.files_fn+ );}fn print_kept_rate_explanation(base: &str, actual: &str, expected: &str) {@@ -373,7 +458,9 @@ fn visualize_whitespace(token: &str) -> String {#[derive(Debug, Deserialize)]struct JsonExample {prompt_inputs: PromptInputs,+ cursor_path: String,expected_patches: Vec<String>,+ #[serde(default)]predictions: Vec<Prediction>,}@@ -381,6 +468,20 @@ struct JsonExample {struct PromptInputs {cursor_excerpt: String,excerpt_start_row: u32,+ pub related_files: Option<Vec<RelatedFile>>,+}++#[derive(Clone, Debug, PartialEq, Hash, Deserialize)]+pub struct RelatedFile {+ pub path: String,+ pub max_row: u32,+ pub excerpts: Vec<RelatedExcerpt>,+}++#[derive(Clone, Debug, PartialEq, Hash, Deserialize)]+pub struct RelatedExcerpt {+ pub row_range: std::ops::Range<u32>,+ pub text: String,}#[derive(Debug, Deserialize)]@@ -391,6 +492,7 @@ struct Prediction {#[derive(Debug, Clone)]struct ParsedHunk {old_start: u32,+ filename: Option<String>,lines: Vec<HunkLine>,}@@ -405,8 +507,20 @@ fn apply_patch_to_excerpt(base: &str,patch: &str,excerpt_start_row: u32,+ target_path: Option<&str>,) -> Result<String, String> {let hunks = parse_diff_hunks(patch);+ let hunks = if let Some(target_path) = target_path {+ hunks+ .into_iter()+ .filter(|hunk| match hunk.filename.as_deref() {+ Some(filename) => filename == target_path,+ None => true,+ })+ .collect::<Vec<_>>()+ } else {+ hunks+ };let result = try_apply_hunks(base, &hunks, excerpt_start_row);@@ -598,6 +712,7 @@ fn filter_hunk_to_excerpt(Some(ParsedHunk {old_start: filtered_old_start.unwrap_or(excerpt_start_row),+ filename: hunk.filename.clone(),lines: filtered_lines,})}@@ -605,8 +720,14 @@ fn filter_hunk_to_excerpt(fn parse_diff_hunks(diff: &str) -> Vec<ParsedHunk> {let mut hunks = Vec::new();let mut current_hunk: Option<ParsedHunk> = None;+ let mut current_filename = None;for line in diff.lines() {+ if let Some(filename) = parse_diff_filename(line) {+ current_filename = Some(filename);+ continue;+ }+if let Some((old_start, old_count, _new_start, _new_count)) = parse_hunk_header(line) {if let Some(hunk) = current_hunk.take() {hunks.push(hunk);@@ -614,6 +735,7 @@ fn parse_diff_hunks(diff: &str) -> Vec<ParsedHunk> {let _ = old_count;current_hunk = Some(ParsedHunk {old_start,+ filename: current_filename.clone(),lines: Vec::new(),});continue;@@ -664,6 +786,27 @@ fn parse_hunk_range(part: &str) -> Option<(u32, u32)> {}}+fn parse_diff_filename(line: &str) -> Option<String> {+ let path = line+ .strip_prefix("--- ")+ .or_else(|| line.strip_prefix("+++ "))?;+ normalize_diff_path(path)+}++fn normalize_diff_path(path: &str) -> Option<String> {+ let path = path.trim();+ let path = path+ .strip_prefix("a/")+ .or_else(|| path.strip_prefix("b/"))+ .unwrap_or(path);++ if path == "/dev/null" {+ None+ } else {+ Some(path.to_string())+ }+}+#[cfg(test)]mod tests {use super::*;@@ -673,7 +816,7 @@ mod tests {let base = "fn main() {\n println!(\"hello\");\n}\n";let patch = "@@ -1,3 +1,3 @@\n fn main() {\n- println!(\"hello\");\n+ println!(\"world\");\n }\n";- let actual = apply_patch_to_excerpt(base, patch, 0).unwrap();+ let actual = apply_patch_to_excerpt(base, patch, 0, None).unwrap();assert_eq!(actual, "fn main() {\n println!(\"world\");\n}\n");}@@ -682,7 +825,7 @@ mod tests {let base = "b\nc\nd\n";let patch = "@@ -2,2 +2,2 @@\n-b\n-c\n+x\n+y\n";- let actual = apply_patch_to_excerpt(base, patch, 1).unwrap();+ let actual = apply_patch_to_excerpt(base, patch, 1, None).unwrap();assert_eq!(actual, "x\ny\nd\n");}@@ -693,7 +836,7 @@ mod tests {// even though the excerpt starts at file row 100.let patch = "@@ -2,2 +2,2 @@\n-b\n-c\n+x\n+y\n";- let actual = apply_patch_to_excerpt(base, patch, 100).unwrap();+ let actual = apply_patch_to_excerpt(base, patch, 100, None).unwrap();assert_eq!(actual, "a\nx\ny\nd\n");}@@ -704,7 +847,87 @@ mod tests {// hunk targets line 6 (1-based) = row 5 (0-based) = first line.let patch = "@@ -6,2 +6,2 @@\n-a\n-b\n+x\n+y\n";- let actual = apply_patch_to_excerpt(base, patch, 5).unwrap();+ let actual = apply_patch_to_excerpt(base, patch, 5, None).unwrap();assert_eq!(actual, "x\ny\nc\n");}++ #[test]+ fn json_patch_application_ignores_unrelated_file_hunks() {+ let base = "first\nsecond\nthird\n";+ let patch = "--- a/src/other.rs\n+++ b/src/other.rs\n@@ -2,1 +2,1 @@\n-second\n+changed\n";++ let actual = apply_patch_to_excerpt(base, patch, 0, Some("src/main.rs")).unwrap();+ assert_eq!(actual, base);+ }++ #[test]+ fn json_patch_application_applies_matching_file_hunks() {+ let base = "first\nsecond\nthird\n";+ let patch = "--- a/src/main.rs\n+++ b/src/main.rs\n@@ -2,1 +2,1 @@\n-second\n+changed\n";++ let actual = apply_patch_to_excerpt(base, patch, 0, Some("src/main.rs")).unwrap();+ assert_eq!(actual, "first\nchanged\nthird\n");+ }++ #[test]+ fn json_patch_application_applies_headerless_hunks() {+ let base = "first\nsecond\nthird\n";+ let patch = "@@ -2,1 +2,1 @@\n-second\n+changed\n";++ let actual = apply_patch_to_excerpt(base, patch, 0, Some("src/main.rs")).unwrap();+ assert_eq!(actual, "first\nchanged\nthird\n");+ }++ fn json_example(predictions: Option<&str>) -> String {+ let predictions = predictions+ .map(|predictions| {+ format!(+ r#",+ "predictions": {predictions}"#+ )+ })+ .unwrap_or_default();++ format!(+ r#"{{+ "prompt_inputs": {{+ "cursor_excerpt": "first\nsecond\nthird\n",+ "excerpt_start_row": 0+ }},+ "cursor_path": "src/main.rs",+ "expected_patches": [+ "--- a/src/main.rs\n+++ b/src/main.rs\n@@ -2,1 +2,1 @@\n-second\n+changed\n"+ ]{predictions}+}}"#+ )+ }++ fn report_from_json(predictions: Option<&str>) -> EvaluationReport {+ let example = serde_json::from_str(&json_example(predictions)).unwrap();+ report_from_json_example(example, 0).unwrap()+ }++ #[test]+ fn json_report_with_missing_predictions_uses_expected_patch_for_context_coverage() {+ let report = report_from_json(None);++ assert_eq!(report.actual, "first\nsecond\nthird\n");+ assert_eq!(report.actual_changed_lines, 0);+ assert_eq!(+ report.editable_context_coverage,+ EditableContextCoverage::new(3, 0, 0, 1, 0, 0)+ );+ }++ #[test]+ fn json_report_with_empty_predictions_uses_expected_patch_for_context_coverage() {+ let report = report_from_json(Some("[]"));++ assert_eq!(report.actual, "first\nsecond\nthird\n");+ assert_eq!(report.actual_changed_lines, 0);+ assert_eq!(+ report.editable_context_coverage,+ EditableContextCoverage::new(3, 0, 0, 1, 0, 0)+ );+ }}diff --git a/crates/edit_prediction_metrics/src/patch.rs b/crates/edit_prediction_metrics/src/patch.rsnew file mode 100644index 00000000000..611544cec58--- /dev/null+++ b/crates/edit_prediction_metrics/src/patch.rs@@ -0,0 +1,127 @@+#[derive(Debug, Default, Clone)]+pub struct Patch {+ pub hunks: Vec<Hunk>,+}++impl Patch {+ pub fn parse_unified_diff(unified_diff: &str) -> Patch {+ let mut current_file = String::new();+ let mut is_filename_inherited = false;+ let mut hunk = Hunk::default();+ let mut patch = Patch::default();+ let mut in_header = true;++ for line in unified_diff.lines() {+ if line.starts_with("--- ") || line.starts_with("+++") || line.starts_with("@@") {+ in_header = false;+ }++ if in_header {+ continue;+ }++ if line.starts_with("@@") {+ if !hunk.lines.is_empty() {+ patch.hunks.push(hunk);+ }+ hunk = Hunk::from_header(line, ¤t_file, is_filename_inherited);+ is_filename_inherited = true;+ } else if let Some(path) = line.strip_prefix("--- ") {+ is_filename_inherited = false;+ let path = path.trim().strip_prefix("a/").unwrap_or(path);+ if path != "/dev/null" {+ current_file = path.into();+ }+ } else if let Some(path) = line.strip_prefix("+++ ") {+ is_filename_inherited = false;+ let path = path.trim().strip_prefix("b/").unwrap_or(path);+ if path != "/dev/null" {+ current_file = path.into();+ }+ } else if let Some(line) = line.strip_prefix('+') {+ hunk.lines.push(PatchLine::Addition(line.to_string()));+ } else if let Some(line) = line.strip_prefix('-') {+ hunk.lines.push(PatchLine::Deletion(line.to_string()));+ } else if let Some(line) = line.strip_prefix(' ') {+ hunk.lines.push(PatchLine::Context(line.to_string()));+ } else {+ hunk.lines.push(PatchLine::Garbage(line.to_string()));+ }+ }++ if !hunk.lines.is_empty() {+ patch.hunks.push(hunk);+ }++ patch+ }+}++#[derive(Debug, Default, Clone)]+pub struct Hunk {+ pub old_start: isize,+ pub new_start: isize,+ pub lines: Vec<PatchLine>,+ pub filename: String,+}++impl Hunk {+ pub fn from_header(header: &str, filename: &str, _is_filename_inherited: bool) -> Self {+ let (old_start, _, new_start, _, _) = Self::parse_hunk_header(header);+ Self {+ old_start,+ new_start,+ lines: Vec::new(),+ filename: filename.to_string(),+ }+ }++ fn parse_hunk_header(line: &str) -> (isize, isize, isize, isize, String) {+ let header_part = line.trim_start_matches("@@").trim();+ let parts: Vec<&str> = header_part.split_whitespace().collect();++ if parts.len() < 2 {+ return (0, 0, 0, 0, String::new());+ }++ let old_part = parts[0].trim_start_matches('-');+ let new_part = parts[1].trim_start_matches('+');++ let (old_start, old_count) = Hunk::parse_hunk_header_range(old_part);+ let (new_start, new_count) = Hunk::parse_hunk_header_range(new_part);++ let comment = if parts.len() > 2 {+ parts[2..]+ .join(" ")+ .trim_start_matches("@@")+ .trim()+ .to_string()+ } else {+ String::new()+ };++ (+ old_start as isize,+ old_count as isize,+ new_start as isize,+ new_count as isize,+ comment,+ )+ }++ fn parse_hunk_header_range(part: &str) -> (usize, usize) {+ if let Some((start, count)) = part.split_once(',') {+ (start.parse().unwrap_or(0), count.parse().unwrap_or(0))+ } else {+ (part.parse().unwrap_or(0), 1)+ }+ }+}++#[derive(Clone, Debug, Eq, PartialEq)]+pub enum PatchLine {+ Context(String),+ Addition(String),+ Deletion(String),+ Garbage(String),+}diff --git a/crates/edit_prediction_metrics/src/patch_metrics.rs b/crates/edit_prediction_metrics/src/patch_metrics.rsindex 85470da91c5..c913f9a7e4b 100644--- a/crates/edit_prediction_metrics/src/patch_metrics.rs+++ b/crates/edit_prediction_metrics/src/patch_metrics.rs@@ -1,6 +1,9 @@use std::collections::HashMap;-use crate::tokenize::tokenize;+use crate::{+ patch::{Patch, PatchLine},+ tokenize::tokenize,+};use serde::Serialize;use similar::{DiffTag, TextDiff};@@ -716,129 +719,6 @@ pub fn reconstruct_texts_from_diff(patch_str: &str) -> (String, String) {(old_lines.join("\n"), new_lines.join("\n"))}-#[derive(Debug, Default, Clone)]-struct Patch {- hunks: Vec<Hunk>,-}--impl Patch {- fn parse_unified_diff(unified_diff: &str) -> Patch {- let mut current_file = String::new();- let mut is_filename_inherited = false;- let mut hunk = Hunk::default();- let mut patch = Patch::default();- let mut in_header = true;-- for line in unified_diff.lines() {- if line.starts_with("--- ") || line.starts_with("+++") || line.starts_with("@@") {- in_header = false;- }-- if in_header {- continue;- }-- if line.starts_with("@@") {- if !hunk.lines.is_empty() {- patch.hunks.push(hunk);- }- hunk = Hunk::from_header(line, ¤t_file, is_filename_inherited);- is_filename_inherited = true;- } else if let Some(path) = line.strip_prefix("--- ") {- is_filename_inherited = false;- let path = path.trim().strip_prefix("a/").unwrap_or(path);- if path != "/dev/null" {- current_file = path.into();- }- } else if let Some(path) = line.strip_prefix("+++ ") {- is_filename_inherited = false;- let path = path.trim().strip_prefix("b/").unwrap_or(path);- if path != "/dev/null" {- current_file = path.into();- }- } else if let Some(line) = line.strip_prefix('+') {- hunk.lines.push(PatchLine::Addition(line.to_string()));- } else if let Some(line) = line.strip_prefix('-') {- hunk.lines.push(PatchLine::Deletion(line.to_string()));- } else if let Some(line) = line.strip_prefix(' ') {- hunk.lines.push(PatchLine::Context(line.to_string()));- } else {- hunk.lines.push(PatchLine::Garbage(line.to_string()));- }- }-- if !hunk.lines.is_empty() {- patch.hunks.push(hunk);- }-- patch- }-}--#[derive(Debug, Default, Clone)]-struct Hunk {- new_start: isize,- lines: Vec<PatchLine>,-}--impl Hunk {- fn from_header(header: &str, _filename: &str, _is_filename_inherited: bool) -> Self {- let (_, _, new_start, _, _) = Self::parse_hunk_header(header);- Self {- new_start,- lines: Vec::new(),- }- }-- fn parse_hunk_header(line: &str) -> (isize, isize, isize, isize, String) {- let header_part = line.trim_start_matches("@@").trim();- let parts: Vec<&str> = header_part.split_whitespace().collect();-- if parts.len() < 2 {- return (0, 0, 0, 0, String::new());- }-- let old_part = parts[0].trim_start_matches('-');- let new_part = parts[1].trim_start_matches('+');-- let (old_start, old_count) = Hunk::parse_hunk_header_range(old_part);- let (new_start, new_count) = Hunk::parse_hunk_header_range(new_part);-- let comment = if parts.len() > 2 {- parts[2..]- .join(" ")- .trim_start_matches("@@")- .trim()- .to_string()- } else {- String::new()- };-- (- old_start as isize,- old_count as isize,- new_start as isize,- new_count as isize,- comment,- )- }-- fn parse_hunk_header_range(part: &str) -> (usize, usize) {- if let Some((start, count)) = part.split_once(',') {- (start.parse().unwrap_or(0), count.parse().unwrap_or(0))- } else {- (part.parse().unwrap_or(0), 1)- }- }-}--#[derive(Clone, Debug, Eq, PartialEq)]-enum PatchLine {- Context(String),- Addition(String),- Deletion(String),- Garbage(String),-}#[cfg(test)]mod test_optimization {diff --git a/crates/edit_prediction_metrics/src/prediction_score.rs b/crates/edit_prediction_metrics/src/prediction_score.rsindex 942ce3c9d1a..684a93ba9de 100644--- a/crates/edit_prediction_metrics/src/prediction_score.rs+++ b/crates/edit_prediction_metrics/src/prediction_score.rs@@ -5,12 +5,15 @@ use std::path::Path;use std::sync::Arc;use zeta_prompt::udiff::{apply_diff_to_string, apply_diff_to_string_with_hunk_offset};-use crate::patch_metrics::{- ClassificationMetrics, DeltaChrFMetrics, braces_disbalance, count_patch_token_changes,- delta_chr_f, delta_chr_f_beta, exact_lines_match, has_isolated_whitespace_changes,- is_editable_region_correct,-};use crate::reversal::compute_prediction_reversal_ratio_from_history;+use crate::{+ jumps::{EditableContextCoverage, Excerpt, editable_context_coverage},+ patch_metrics::{+ ClassificationMetrics, DeltaChrFMetrics, braces_disbalance, count_patch_token_changes,+ delta_chr_f, delta_chr_f_beta, exact_lines_match, has_isolated_whitespace_changes,+ is_editable_region_correct,+ },+};#[derive(Clone, Debug, Serialize, Deserialize)]pub struct PredictionScore {@@ -61,6 +64,8 @@ pub struct PredictionScore {pub cumulative_logprob: Option<f64>,#[serde(default, skip_serializing_if = "Option::is_none")]pub avg_logprob: Option<f64>,+ #[serde(default, skip_serializing_if = "Option::is_none")]+ pub editable_context_coverage: Option<EditableContextCoverage>,}impl PredictionScore {@@ -91,6 +96,7 @@ impl PredictionScore {discarded_chars: None,cumulative_logprob: None,avg_logprob: None,+ editable_context_coverage: None,}}@@ -193,11 +199,26 @@ pub struct PredictionScoringInput<'a> {pub reversal_context: Option<PredictionReversalContext<'a>>,pub cumulative_logprob: Option<f64>,pub avg_logprob: Option<f64>,+ pub context: Option<&'a [Excerpt]>,}pub fn score_prediction(input: PredictionScoringInput<'_>) -> PredictionScore {+ let editable_context_coverage = input.context.and_then(|context| {+ input+ .expected_patches+ .iter()+ .map(|expected| editable_context_coverage(&expected.patch, context))+ .max_by(|left, right| {+ left.lines_f1+ .total_cmp(&right.lines_f1)+ .then_with(|| left.files_f1.total_cmp(&right.files_f1))+ })+ });+let Some(actual_patch) = input.actual_patch else {- return PredictionScore::zero();+ let mut score = PredictionScore::zero();+ score.editable_context_coverage = editable_context_coverage;+ return score;};let token_changes = count_patch_token_changes(actual_patch);@@ -208,6 +229,7 @@ pub fn score_prediction(input: PredictionScoringInput<'_>) -> PredictionScore {let mut score = PredictionScore::zero();score.inserted_tokens = token_changes.inserted_tokens;score.deleted_tokens = token_changes.deleted_tokens;+ score.editable_context_coverage = editable_context_coverage;return score;}};@@ -302,6 +324,7 @@ pub fn score_prediction(input: PredictionScoringInput<'_>) -> PredictionScore {discarded_chars,cumulative_logprob: input.cumulative_logprob,avg_logprob: input.avg_logprob,+ editable_context_coverage,}}@@ -343,6 +366,7 @@ mod tests {reversal_context: None,cumulative_logprob: None,avg_logprob: None,+ context: None,});assert_eq!(score.delta_chr_f, 0.0);diff --git a/crates/edit_prediction_metrics/src/summary.rs b/crates/edit_prediction_metrics/src/summary.rsindex 249ae185755..9c4028a300b 100644--- a/crates/edit_prediction_metrics/src/summary.rs+++ b/crates/edit_prediction_metrics/src/summary.rs@@ -56,6 +56,30 @@ pub struct SummaryJson {pub total_correctly_deleted_chars: Option<usize>,#[serde(skip_serializing_if = "Option::is_none")]pub total_discarded_chars: Option<usize>,+ #[serde(skip_serializing_if = "Option::is_none")]+ pub avg_editable_context_lines_precision: Option<f64>,+ #[serde(skip_serializing_if = "Option::is_none")]+ pub avg_editable_context_lines_recall: Option<f64>,+ #[serde(skip_serializing_if = "Option::is_none")]+ pub avg_editable_context_lines_f1: Option<f64>,+ #[serde(skip_serializing_if = "Option::is_none")]+ pub editable_context_lines_tp: Option<usize>,+ #[serde(skip_serializing_if = "Option::is_none")]+ pub editable_context_lines_fp: Option<usize>,+ #[serde(skip_serializing_if = "Option::is_none")]+ pub editable_context_lines_fn: Option<usize>,+ #[serde(skip_serializing_if = "Option::is_none")]+ pub avg_editable_context_files_precision: Option<f64>,+ #[serde(skip_serializing_if = "Option::is_none")]+ pub avg_editable_context_files_recall: Option<f64>,+ #[serde(skip_serializing_if = "Option::is_none")]+ pub avg_editable_context_files_f1: Option<f64>,+ #[serde(skip_serializing_if = "Option::is_none")]+ pub editable_context_files_tp: Option<usize>,+ #[serde(skip_serializing_if = "Option::is_none")]+ pub editable_context_files_fp: Option<usize>,+ #[serde(skip_serializing_if = "Option::is_none")]+ pub editable_context_files_fn: Option<usize>,}pub fn compute_summary<'a>(@@ -91,6 +115,19 @@ pub fn compute_summary<'a>(let mut discarded_chars_count: usize = 0;let mut recall_rate_sum: f64 = 0.0;let mut recall_rate_count: usize = 0;+ let mut editable_context_lines_precision_sum: f64 = 0.0;+ let mut editable_context_lines_recall_sum: f64 = 0.0;+ let mut editable_context_lines_f1_sum: f64 = 0.0;+ let mut editable_context_files_precision_sum: f64 = 0.0;+ let mut editable_context_files_recall_sum: f64 = 0.0;+ let mut editable_context_files_f1_sum: f64 = 0.0;+ let mut editable_context_coverage_count: usize = 0;+ let mut editable_context_lines_tp: usize = 0;+ let mut editable_context_lines_fp: usize = 0;+ let mut editable_context_lines_fn: usize = 0;+ let mut editable_context_files_tp: usize = 0;+ let mut editable_context_files_fp: usize = 0;+ let mut editable_context_files_fn: usize = 0;for prediction in predictions {let score = prediction.score;@@ -149,6 +186,21 @@ pub fn compute_summary<'a>(recall_rate_sum += recall_rate;recall_rate_count += 1;}+ if let Some(coverage) = &score.editable_context_coverage {+ editable_context_lines_precision_sum += coverage.lines_precision;+ editable_context_lines_recall_sum += coverage.lines_recall;+ editable_context_lines_f1_sum += coverage.lines_f1;+ editable_context_files_precision_sum += coverage.files_precision;+ editable_context_files_recall_sum += coverage.files_recall;+ editable_context_files_f1_sum += coverage.files_f1;+ editable_context_coverage_count += 1;+ editable_context_lines_tp += coverage.lines_tp;+ editable_context_lines_fp += coverage.lines_fp;+ editable_context_lines_fn += coverage.lines_fn;+ editable_context_files_tp += coverage.files_tp;+ editable_context_files_fp += coverage.files_fp;+ editable_context_files_fn += coverage.files_fn;+ }if let Some(exact_match) = score.cursor_exact_match {cursor_total += 1;@@ -252,6 +304,67 @@ pub fn compute_summary<'a>(None};+ let avg_editable_context_lines_precision = if editable_context_coverage_count > 0 {+ Some(editable_context_lines_precision_sum / editable_context_coverage_count as f64)+ } else {+ None+ };+ let avg_editable_context_lines_recall = if editable_context_coverage_count > 0 {+ Some(editable_context_lines_recall_sum / editable_context_coverage_count as f64)+ } else {+ None+ };+ let avg_editable_context_lines_f1 = if editable_context_coverage_count > 0 {+ Some(editable_context_lines_f1_sum / editable_context_coverage_count as f64)+ } else {+ None+ };+ let editable_context_lines_tp = if editable_context_coverage_count > 0 {+ Some(editable_context_lines_tp)+ } else {+ None+ };+ let editable_context_lines_fp = if editable_context_coverage_count > 0 {+ Some(editable_context_lines_fp)+ } else {+ None+ };+ let editable_context_lines_fn = if editable_context_coverage_count > 0 {+ Some(editable_context_lines_fn)+ } else {+ None+ };+ let avg_editable_context_files_precision = if editable_context_coverage_count > 0 {+ Some(editable_context_files_precision_sum / editable_context_coverage_count as f64)+ } else {+ None+ };+ let avg_editable_context_files_recall = if editable_context_coverage_count > 0 {+ Some(editable_context_files_recall_sum / editable_context_coverage_count as f64)+ } else {+ None+ };+ let avg_editable_context_files_f1 = if editable_context_coverage_count > 0 {+ Some(editable_context_files_f1_sum / editable_context_coverage_count as f64)+ } else {+ None+ };+ let editable_context_files_tp = if editable_context_coverage_count > 0 {+ Some(editable_context_files_tp)+ } else {+ None+ };+ let editable_context_files_fp = if editable_context_coverage_count > 0 {+ Some(editable_context_files_fp)+ } else {+ None+ };+ let editable_context_files_fn = if editable_context_coverage_count > 0 {+ Some(editable_context_files_fn)+ } else {+ None+ };+SummaryJson {total_examples: total_scores,avg_delta_chr_f,@@ -289,5 +402,17 @@ pub fn compute_summary<'a>(total_kept_chars,total_correctly_deleted_chars,total_discarded_chars,+ avg_editable_context_lines_precision,+ avg_editable_context_lines_recall,+ avg_editable_context_lines_f1,+ editable_context_lines_tp,+ editable_context_lines_fp,+ editable_context_lines_fn,+ avg_editable_context_files_precision,+ avg_editable_context_files_recall,+ avg_editable_context_files_f1,+ editable_context_files_tp,+ editable_context_files_fp,+ editable_context_files_fn,}}diff --git a/crates/zeta_prompt/src/zeta_prompt.rs b/crates/zeta_prompt/src/zeta_prompt.rsindex e685645e687..d513a9e66a4 100644--- a/crates/zeta_prompt/src/zeta_prompt.rs+++ b/crates/zeta_prompt/src/zeta_prompt.rs@@ -240,6 +240,21 @@ pub struct RelatedExcerpt {pub text: Arc<str>,#[serde(default)]pub order: usize,+ #[serde(default)]+ pub context_source: ContextSource,+}++#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, Serialize, Deserialize)]+#[serde(rename_all = "snake_case")]+pub enum ContextSource {+ #[default]+ Lsp,+ CursorExcerpt,+ CurrentFile,+ EditHistory,+ EditHistoryFile,+ GitLog,+ OracleFile,}pub fn prompt_input_contains_special_tokens(input: &ZetaPromptInput, format: ZetaFormat) -> bool {@@ -4851,6 +4866,7 @@ mod tests {row_range: 0..content.lines().count() as u32,text: content.into(),order: 0,+ context_source: ContextSource::Lsp,}],in_open_source_repo: false,}@@ -4970,16 +4986,19 @@ mod tests {row_range: 0..10,text: "first excerpt\n".into(),order: 0,+ context_source: ContextSource::Lsp,},RelatedExcerpt {row_range: 10..20,text: "second excerpt\n".into(),order: 0,+ context_source: ContextSource::Lsp,},RelatedExcerpt {row_range: 20..30,text: "third excerpt\n".into(),order: 0,+ context_source: ContextSource::Lsp,},],}],@@ -5039,6 +5058,7 @@ mod tests {row_range: 0..10,text: "low priority content\n".into(),order: 5,+ context_source: ContextSource::Lsp,}],},RelatedFile {@@ -5049,6 +5069,7 @@ mod tests {row_range: 0..10,text: "high priority content\n".into(),order: 1,+ context_source: ContextSource::Lsp,}],},],@@ -5113,16 +5134,19 @@ mod tests {row_range: 0..5,text: "mod header\n".into(),order: 1,+ context_source: ContextSource::Lsp,},RelatedExcerpt {row_range: 5..15,text: "important fn\n".into(),order: 1,+ context_source: ContextSource::Lsp,},RelatedExcerpt {row_range: 15..30,text: "less important fn\n".into(),order: 3,+ context_source: ContextSource::Lsp,},],}],@@ -5522,6 +5546,7 @@ mod tests {row_range: 0..5,text: "low prio\n".into(),order: 10,+ context_source: ContextSource::Lsp,}],},RelatedFile {@@ -5532,6 +5557,7 @@ mod tests {row_range: 0..5,text: "high prio\n".into(),order: 1,+ context_source: ContextSource::Lsp,}],},],