Skip to content

armgit / zed

Public Imported from GitHub
0

Code at the speed of thought – Zed is a high-performance, multiplayer code editor from the creators of Atom and Tree-sitter.

ep: Editable context collectors and metrics (#58400)

Oleksiy Syvokonade2b219f788
diff --git a/Cargo.lock b/Cargo.lock
index b07750b7ba8..830cfca9688 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -5524,6 +5524,7 @@ dependencies = [
"debug_adapter_extension",
"dirs",
"edit_prediction",
+ "edit_prediction_context",
"edit_prediction_metrics",
"extension",
"flate2",
diff --git a/crates/edit_prediction/src/edit_prediction.rs b/crates/edit_prediction/src/edit_prediction.rs
index ab88dbf2fa6..5e48063cd6d 100644
--- a/crates/edit_prediction/src/edit_prediction.rs
+++ b/crates/edit_prediction/src/edit_prediction.rs
@@ -55,6 +55,8 @@ use std::env;
use std::rc::Rc;
use text::{AnchorRangeExt, Edit};
use workspace::{AppState, Workspace};
+#[cfg(feature = "cli-support")]
+use zeta_prompt::ContextSource;
use zeta_prompt::{ZetaFormat, ZetaPromptInput};
use std::mem;
@@ -3047,6 +3049,51 @@ impl EditPredictionStore {
});
}
+ #[cfg(feature = "cli-support")]
+ pub fn collect_editable_context(
+ &mut self,
+ project: Entity<Project>,
+ buffer: Entity<language::Buffer>,
+ cursor_position: language::Anchor,
+ oracle_paths: Vec<Arc<Path>>,
+ context_sources: Vec<ContextSource>,
+ cx: &mut Context<Self>,
+ ) -> Task<anyhow::Result<Vec<RelatedFile>>> {
+ use edit_prediction_context::{EditHistoryContextEntry, collect_editable_context};
+
+ let buffers_by_id = project.read(cx).opened_buffers(cx).into_iter().fold(
+ HashMap::default(),
+ |mut buffers_by_id, buffer| {
+ buffers_by_id.insert(buffer.read(cx).remote_id(), buffer.clone());
+ buffers_by_id
+ },
+ );
+ let edit_history = self
+ .edit_history_for_project(&project, cx)
+ .into_iter()
+ .filter_map(|event| {
+ let buffer = buffers_by_id.get(&event.old_snapshot.remote_id())?.clone();
+ Some(EditHistoryContextEntry {
+ buffer,
+ edited_range: event.total_edit_range,
+ })
+ })
+ .collect();
+
+ cx.spawn(async move |_, cx| {
+ collect_editable_context(
+ project,
+ buffer,
+ cursor_position,
+ edit_history,
+ oracle_paths,
+ context_sources,
+ cx,
+ )
+ .await
+ })
+ }
+
#[cfg(feature = "cli-support")]
pub fn set_context_for_buffer(
&mut self,
diff --git a/crates/edit_prediction_cli/Cargo.toml b/crates/edit_prediction_cli/Cargo.toml
index 81e91fa262d..733deb895c0 100644
--- a/crates/edit_prediction_cli/Cargo.toml
+++ b/crates/edit_prediction_cli/Cargo.toml
@@ -61,6 +61,7 @@ terminal_view.workspace = true
util.workspace = true
watch.workspace = true
edit_prediction = { workspace = true, features = ["cli-support"] }
+edit_prediction_context.workspace = true
edit_prediction_metrics = { workspace = true, features = ["tree-sitter"] }
telemetry_events.workspace = true
wasmtime.workspace = true
@@ -90,4 +91,3 @@ pretty_assertions.workspace = true
project = { workspace = true, features = ["test-support"] }
tempfile.workspace = true
workspace = { workspace = true, features = ["test-support"] }
-
diff --git a/crates/edit_prediction_cli/src/format_prompt.rs b/crates/edit_prediction_cli/src/format_prompt.rs
index e0354a78aca..933560b476b 100644
--- a/crates/edit_prediction_cli/src/format_prompt.rs
+++ b/crates/edit_prediction_cli/src/format_prompt.rs
@@ -3,7 +3,7 @@ use crate::{
example::{ActualCursor, Example, ExamplePrompt},
headless::EpAppState,
progress::{ExampleProgress, Step},
- retrieve_context::run_context_retrieval,
+ retrieve_context::{ContextRetrievalType, run_context_retrieval},
};
use anyhow::{Context as _, Result, anyhow};
use gpui::AsyncApp;
@@ -30,7 +30,15 @@ pub async fn run_format_prompt(
example_progress: &ExampleProgress,
cx: AsyncApp,
) -> Result<()> {
- run_context_retrieval(example, app_state.clone(), example_progress, cx.clone()).await?;
+ run_context_retrieval(
+ example,
+ app_state.clone(),
+ example_progress,
+ ContextRetrievalType::Lsp,
+ false,
+ cx.clone(),
+ )
+ .await?;
let step_progress = example_progress.start(Step::FormatPrompt);
diff --git a/crates/edit_prediction_cli/src/main.rs b/crates/edit_prediction_cli/src/main.rs
index 41e7b397745..69167ce22b4 100644
--- a/crates/edit_prediction_cli/src/main.rs
+++ b/crates/edit_prediction_cli/src/main.rs
@@ -59,7 +59,7 @@ use crate::paths::{FAILED_EXAMPLES_DIR, RUN_DIR};
use crate::predict::run_prediction;
use crate::progress::Progress;
use crate::pull_examples::{fetch_settled_examples_after, parse_settled_after_input};
-use crate::retrieve_context::run_context_retrieval;
+use crate::retrieve_context::{ContextRetrievalType, run_context_retrieval};
use crate::score::run_scoring;
use crate::split_commit::SplitCommitArgs;
use crate::split_dataset::SplitArgs;
@@ -100,7 +100,7 @@ struct EpArgs {
output: Option<PathBuf>,
#[arg(long, short, global = true)]
in_place: bool,
- #[arg(long, short, global = true)]
+ #[arg(long, global = true)]
failfast: bool,
/// How to handle failed examples in output: keep them or skip them.
/// Failed examples are always logged to the run's failed directory.
@@ -125,6 +125,16 @@ pub enum FailedHandling {
SkipNoFiles,
}
+#[derive(Args, Debug, Clone)]
+struct ContextArgs {
+ /// Which context collector to run.
+ #[arg(long = "type", value_enum, default_value_t = ContextRetrievalType::Lsp)]
+ context_type: ContextRetrievalType,
+ /// Recompute context even if the example already has related files.
+ #[arg(long, short = 'f', default_value_t = false)]
+ force: bool,
+}
+
const INPUTS_HELP: &str = r#"
Inputs can be file paths or special specifiers:
@@ -197,7 +207,7 @@ enum Command {
/// Create git worktrees for each example and load file contents
LoadProject,
/// Retrieve context for input examples.
- Context,
+ Context(ContextArgs),
/// Generate a prompt string for a specific model
FormatPrompt(FormatPromptArgs),
/// Runs edit prediction
@@ -239,7 +249,13 @@ impl Display for Command {
match self {
Command::Read(_) => write!(f, "read"),
Command::LoadProject => write!(f, "load-project"),
- Command::Context => write!(f, "context"),
+ Command::Context(args) => {
+ write!(f, "context --type={}", args.context_type)?;
+ if args.force {
+ write!(f, " --force")?;
+ }
+ Ok(())
+ }
Command::FormatPrompt(args) => {
write!(f, "format-prompt --provider={}", args.provider)
}
@@ -253,10 +269,16 @@ impl Display for Command {
None => write!(f, "score"),
},
Command::Distill => write!(f, "distill"),
- Command::Eval(args) => match &args.predict.provider {
- Some(provider) => write!(f, "eval --provider={}", provider),
- None => write!(f, "eval"),
- },
+ Command::Eval(args) => {
+ write!(f, "eval")?;
+ if args.context_only {
+ write!(f, " --context-only")?;
+ }
+ if let Some(provider) = &args.predict.provider {
+ write!(f, " --provider={}", provider)?;
+ }
+ Ok(())
+ }
Command::Synthesize(args) => {
write!(f, "synthesize --repos {}", args.repos.join(" "))
}
@@ -309,6 +331,9 @@ struct PredictArgs {
struct EvalArgs {
#[clap(flatten)]
predict: PredictArgs,
+ /// Only compute editable context coverage from expected patches and retrieved context.
+ #[clap(long)]
+ context_only: bool,
/// Path to write summary scores as JSON
#[clap(long)]
summary_json: Option<PathBuf>,
@@ -1133,7 +1158,9 @@ fn main() {
predict::sync_batches(args.provider.as_ref()).await?;
}
Command::Eval(args) => {
- predict::sync_batches(args.predict.provider.as_ref()).await?;
+ if !args.context_only {
+ predict::sync_batches(args.predict.provider.as_ref()).await?;
+ }
}
Command::Qa(args) => {
qa::sync_batches(args).await?;
@@ -1222,11 +1249,13 @@ fn main() {
)
.await?;
}
- Command::Context => {
+ Command::Context(args) => {
run_context_retrieval(
example,
app_state.clone(),
&example_progress,
+ args.context_type,
+ args.force,
cx.clone(),
)
.await?;
@@ -1264,18 +1293,30 @@ fn main() {
app_state.clone(),
&example_progress,
cx.clone(),
+ false,
+ None,
)
.await?;
}
Command::Eval(args) => {
- run_scoring(
- example,
- &args.predict,
- app_state.clone(),
- &example_progress,
- cx.clone(),
- )
- .await?;
+ if args.context_only {
+ score::run_context_coverage_scoring(
+ example,
+ &example_progress,
+ Some(score::EVAL_RETRIEVED_CONTEXT_BYTE_LIMIT),
+ )?;
+ } else {
+ run_scoring(
+ example,
+ &args.predict,
+ app_state.clone(),
+ &example_progress,
+ cx.clone(),
+ true,
+ Some(score::EVAL_RETRIEVED_CONTEXT_BYTE_LIMIT),
+ )
+ .await?;
+ }
}
Command::Qa(args) => {
qa::run_qa(example, args, &example_progress).await?;
@@ -1394,15 +1435,17 @@ fn main() {
}
}
Command::Eval(args) => {
- predict::sync_batches(args.predict.provider.as_ref()).await?;
- if args.predict.wait {
- predict::wait_for_batches(args.predict.provider.as_ref()).await?;
- let mut examples =
- std::mem::take(&mut *finished_examples.lock().unwrap());
- predict::reprocess_after_batch_wait(&mut examples, &args.predict)
- .await?;
- rewrite_output(&examples, write_path, is_markdown)?;
- *finished_examples.lock().unwrap() = examples;
+ if !args.context_only {
+ predict::sync_batches(args.predict.provider.as_ref()).await?;
+ if args.predict.wait {
+ predict::wait_for_batches(args.predict.provider.as_ref()).await?;
+ let mut examples =
+ std::mem::take(&mut *finished_examples.lock().unwrap());
+ predict::reprocess_after_batch_wait(&mut examples, &args.predict)
+ .await?;
+ rewrite_output(&examples, write_path, is_markdown)?;
+ *finished_examples.lock().unwrap() = examples;
+ }
}
}
Command::Qa(args) => {
@@ -1425,7 +1468,12 @@ fn main() {
match &command {
Command::Eval(args) => {
let examples = finished_examples.lock().unwrap();
- score::print_report(&examples, args.verbose);
+ score::print_report(
+ &examples,
+ args.verbose,
+ args.context_only,
+ Some(score::EVAL_RETRIEVED_CONTEXT_BYTE_LIMIT),
+ );
if let Some(summary_path) = &args.summary_json {
score::write_summary_json(&examples, summary_path)?;
}
diff --git a/crates/edit_prediction_cli/src/predict.rs b/crates/edit_prediction_cli/src/predict.rs
index a78d4d12d70..c21066aa812 100644
--- a/crates/edit_prediction_cli/src/predict.rs
+++ b/crates/edit_prediction_cli/src/predict.rs
@@ -9,7 +9,7 @@ use crate::{
parse_output::parse_prediction_output,
paths::{LATEST_EXAMPLE_RUN_DIR, RUN_DIR},
progress::{ExampleProgress, InfoStyle, Progress, Step, StepProgress},
- retrieve_context::run_context_retrieval,
+ retrieve_context::{ContextRetrievalType, run_context_retrieval},
};
use anyhow::Context as _;
use cloud_llm_client::predict_edits_v3::{RawCompletionRequest, RawCompletionResponse};
@@ -68,7 +68,15 @@ pub async fn run_prediction(
if let PredictionProvider::Teacher(backend, _)
| PredictionProvider::TeacherNonBatching(backend, _) = provider
{
- run_context_retrieval(example, app_state.clone(), example_progress, cx.clone()).await?;
+ run_context_retrieval(
+ example,
+ app_state.clone(),
+ example_progress,
+ ContextRetrievalType::Lsp,
+ false,
+ cx.clone(),
+ )
+ .await?;
run_format_prompt(
example,
&FormatPromptArgs { provider },
@@ -111,7 +119,15 @@ pub async fn run_prediction(
}
run_load_project(example, app_state.clone(), example_progress, cx.clone()).await?;
- run_context_retrieval(example, app_state.clone(), example_progress, cx.clone()).await?;
+ run_context_retrieval(
+ example,
+ app_state.clone(),
+ example_progress,
+ ContextRetrievalType::Lsp,
+ false,
+ cx.clone(),
+ )
+ .await?;
let step_progress = example_progress.start(Step::Predict);
diff --git a/crates/edit_prediction_cli/src/retrieve_context.rs b/crates/edit_prediction_cli/src/retrieve_context.rs
index f02509ceb06..91d9ab0761b 100644
--- a/crates/edit_prediction_cli/src/retrieve_context.rs
+++ b/crates/edit_prediction_cli/src/retrieve_context.rs
@@ -5,25 +5,88 @@ use crate::{
progress::{ExampleProgress, InfoStyle, Step, StepProgress},
};
use anyhow::Context as _;
+use clap::ValueEnum;
use collections::HashSet;
-use edit_prediction::{DebugEvent, EditPredictionStore};
+use edit_prediction::{DebugEvent, EditPredictionStore, udiff::refresh_worktree_entries};
use futures::{FutureExt as _, StreamExt as _, channel::mpsc};
use gpui::{AsyncApp, Entity};
use language::Buffer;
use project::Project;
-use std::sync::Arc;
-use std::time::Duration;
+use std::{
+ path::{Path, PathBuf},
+ sync::Arc,
+ time::Duration,
+};
+use zeta_prompt::{ContextSource, udiff::DiffLine};
+
+#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, ValueEnum)]
+pub enum ContextRetrievalType {
+ Lsp,
+ Editable,
+ CurrentFile,
+ EditHistory,
+ EditHistoryFile,
+ GitLog,
+ OracleFile,
+ #[default]
+ All,
+ None,
+}
+
+impl std::fmt::Display for ContextRetrievalType {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ match self {
+ ContextRetrievalType::Lsp => write!(f, "lsp"),
+ ContextRetrievalType::Editable => write!(f, "editable"),
+ ContextRetrievalType::CurrentFile => write!(f, "current-file"),
+ ContextRetrievalType::EditHistory => write!(f, "edit-history"),
+ ContextRetrievalType::EditHistoryFile => write!(f, "edit-history-file"),
+ ContextRetrievalType::GitLog => write!(f, "git-log"),
+ ContextRetrievalType::OracleFile => write!(f, "oracle-file"),
+ ContextRetrievalType::All => write!(f, "all"),
+ ContextRetrievalType::None => write!(f, "none"),
+ }
+ }
+}
+
+impl ContextRetrievalType {
+ fn includes_lsp(self) -> bool {
+ matches!(self, ContextRetrievalType::Lsp | ContextRetrievalType::All)
+ }
+
+ fn editable_context_sources(self) -> Option<Vec<ContextSource>> {
+ match self {
+ ContextRetrievalType::Editable | ContextRetrievalType::All => Some(vec![
+ ContextSource::CursorExcerpt,
+ ContextSource::CurrentFile,
+ ContextSource::EditHistory,
+ ContextSource::EditHistoryFile,
+ ContextSource::GitLog,
+ ContextSource::OracleFile,
+ ]),
+ ContextRetrievalType::CurrentFile => Some(vec![ContextSource::CurrentFile]),
+ ContextRetrievalType::EditHistory => Some(vec![ContextSource::EditHistory]),
+ ContextRetrievalType::EditHistoryFile => Some(vec![ContextSource::EditHistoryFile]),
+ ContextRetrievalType::GitLog => Some(vec![ContextSource::GitLog]),
+ ContextRetrievalType::OracleFile => Some(vec![ContextSource::OracleFile]),
+ ContextRetrievalType::Lsp | ContextRetrievalType::None => None,
+ }
+ }
+}
pub async fn run_context_retrieval(
example: &mut Example,
app_state: Arc<EpAppState>,
example_progress: &ExampleProgress,
+ context_type: ContextRetrievalType,
+ force: bool,
mut cx: AsyncApp,
) -> anyhow::Result<()> {
- if example
- .prompt_inputs
- .as_ref()
- .is_some_and(|inputs| inputs.related_files.is_some())
+ if (!force
+ && example
+ .prompt_inputs
+ .as_ref()
+ .is_some_and(|inputs| inputs.related_files.is_some()))
|| example.spec.repository_url.is_empty()
{
return Ok(());
@@ -36,32 +99,61 @@ pub async fn run_context_retrieval(
let state = example.state.as_ref().unwrap();
let project = state.project.clone();
- let _lsp_handle = project.update(&mut cx, |project, cx| {
- project.register_buffer_with_language_servers(&state.buffer, cx)
- });
- wait_for_language_servers_to_start(&project, &state.buffer, &step_progress, &mut cx).await?;
-
let ep_store = cx
.update(|cx| EditPredictionStore::try_global(cx))
.context("EditPredictionStore not initialized")?;
- let mut events = ep_store.update(&mut cx, |store, cx| {
- store.register_buffer(&state.buffer, &project, cx);
- store.refresh_context(&project, &state.buffer, state.cursor_position, cx);
- store.debug_info(&project, cx)
- });
+ let mut context_files = Vec::new();
+
+ if context_type.includes_lsp() {
+ let _lsp_handle = project.update(&mut cx, |project, cx| {
+ project.register_buffer_with_language_servers(&state.buffer, cx)
+ });
+ wait_for_language_servers_to_start(&project, &state.buffer, &step_progress, &mut cx)
+ .await?;
+
+ let mut events = ep_store.update(&mut cx, |store, cx| {
+ store.register_buffer(&state.buffer, &project, cx);
+ store.refresh_context(&project, &state.buffer, state.cursor_position, cx);
+ store.debug_info(&project, cx)
+ });
- while let Some(event) = events.next().await {
- match event {
- DebugEvent::ContextRetrievalFinished(_) => {
- break;
+ while let Some(event) = events.next().await {
+ match event {
+ DebugEvent::ContextRetrievalFinished(_) => {
+ break;
+ }
+ _ => {}
}
- _ => {}
}
+
+ context_files
+ .extend(ep_store.update(&mut cx, |store, cx| store.context_for_project(&project, cx)));
}
- let context_files =
- ep_store.update(&mut cx, |store, cx| store.context_for_project(&project, cx));
+ if let Some(context_sources) = context_type.editable_context_sources() {
+ let oracle_paths = if context_sources.contains(&ContextSource::OracleFile) {
+ let oracle_paths = oracle_paths_from_expected_patches(example);
+ refresh_paths(&project, &oracle_paths, &mut cx).await?;
+ oracle_paths
+ } else {
+ Vec::new()
+ };
+
+ let editable_context = ep_store
+ .update(&mut cx, |store, cx| {
+ store.collect_editable_context(
+ project.clone(),
+ state.buffer.clone(),
+ state.cursor_position,
+ oracle_paths,
+ context_sources,
+ cx,
+ )
+ })
+ .await?;
+ merge_context_files(&mut context_files, editable_context);
+ }
let excerpt_count: usize = context_files.iter().map(|f| f.excerpts.len()).sum();
step_progress.set_info(format!("{} excerpts", excerpt_count), InfoStyle::Normal);
@@ -72,6 +164,73 @@ pub async fn run_context_retrieval(
Ok(())
}
+fn merge_context_files(
+ context_files: &mut Vec<zeta_prompt::RelatedFile>,
+ new_files: Vec<zeta_prompt::RelatedFile>,
+) {
+ for mut new_file in new_files {
+ if let Some(existing_file) = context_files
+ .iter_mut()
+ .find(|existing_file| existing_file.path == new_file.path)
+ {
+ existing_file.max_row = existing_file.max_row.max(new_file.max_row);
+ existing_file.excerpts.append(&mut new_file.excerpts);
+ existing_file
+ .excerpts
+ .sort_by_key(|excerpt| (excerpt.order, excerpt.row_range.start));
+ existing_file.in_open_source_repo =
+ existing_file.in_open_source_repo && new_file.in_open_source_repo;
+ } else {
+ context_files.push(new_file);
+ }
+ }
+}
+
+fn oracle_paths_from_expected_patches(example: &Example) -> Vec<Arc<Path>> {
+ let mut seen_paths = HashSet::default();
+ let mut paths = Vec::new();
+
+ for patch in &example.spec.expected_patches {
+ for path in paths_from_diff(patch) {
+ if seen_paths.insert(path.clone()) {
+ paths.push(path.into());
+ }
+ }
+ }
+
+ paths
+}
+
+fn paths_from_diff(diff: &str) -> Vec<PathBuf> {
+ diff.lines()
+ .filter_map(|line| match DiffLine::parse(line) {
+ DiffLine::OldPath { path } | DiffLine::NewPath { path }
+ if path.as_ref() != "/dev/null" =>
+ {
+ Some(Path::new(path.as_ref()).to_path_buf())
+ }
+ _ => None,
+ })
+ .collect()
+}
+
+async fn refresh_paths(
+ project: &Entity<Project>,
+ paths: &[Arc<Path>],
+ cx: &mut AsyncApp,
+) -> anyhow::Result<()> {
+ if paths.is_empty() {
+ return Ok(());
+ }
+
+ let Some(worktree) = project.read_with(cx, |project, cx| project.visible_worktrees(cx).next())
+ else {
+ return Ok(());
+ };
+
+ refresh_worktree_entries(&worktree, paths.iter().map(|path| path.as_ref()), cx).await
+}
+
async fn wait_for_language_servers_to_start(
project: &Entity<Project>,
buffer: &Entity<Buffer>,
diff --git a/crates/edit_prediction_cli/src/score.rs b/crates/edit_prediction_cli/src/score.rs
index cbaeb338650..e8227a0d7c4 100644
--- a/crates/edit_prediction_cli/src/score.rs
+++ b/crates/edit_prediction_cli/src/score.rs
@@ -8,8 +8,9 @@ use crate::{
progress::{ExampleProgress, Step},
};
use anyhow::Context as _;
+use edit_prediction_context::limit_retrieved_context_to_bytes;
use edit_prediction_metrics::{
- ActualPredictionCursor, PredictionReversalContext, PredictionScoringInput,
+ ActualPredictionCursor, Excerpt, PredictionReversalContext, PredictionScoringInput,
};
use gpui::{AppContext as _, AsyncApp};
use std::fs::File;
@@ -17,14 +18,20 @@ use std::io::BufWriter;
use std::path::Path;
use std::sync::Arc;
+pub const EVAL_RETRIEVED_CONTEXT_BYTE_LIMIT: usize = 4000;
+
pub async fn run_scoring(
example: &mut Example,
args: &PredictArgs,
app_state: Arc<EpAppState>,
example_progress: &ExampleProgress,
cx: AsyncApp,
+ allow_missing_predictions: bool,
+ retrieved_context_byte_limit: Option<usize>,
) -> anyhow::Result<()> {
- run_prediction(example, args, app_state, example_progress, cx.clone()).await?;
+ if !(allow_missing_predictions && args.provider.is_none() && example.predictions.is_empty()) {
+ run_prediction(example, args, app_state, example_progress, cx.clone()).await?;
+ }
let progress = example_progress.start(Step::Score);
@@ -70,8 +77,32 @@ pub async fn run_scoring(
})?;
let cursor_path = example_for_scoring.spec.cursor_path.as_ref();
+ let context = context_excerpts(
+ &example_for_scoring,
+ prompt_inputs,
+ retrieved_context_byte_limit,
+ );
let mut scores = vec![];
+ if allow_missing_predictions && example_for_scoring.predictions.is_empty() {
+ scores.push(edit_prediction_metrics::score_prediction(
+ PredictionScoringInput {
+ original_text,
+ expected_patches: &prepared_expected_patches,
+ actual_patch: None,
+ actual_cursor: None,
+ reversal_context: Some(PredictionReversalContext {
+ edit_history: &prompt_inputs.events,
+ excerpt_start_row: prompt_inputs.excerpt_start_row,
+ cursor_path,
+ }),
+ cumulative_logprob: None,
+ avg_logprob: None,
+ context: Some(&context),
+ },
+ ));
+ }
+
for prediction in &example_for_scoring.predictions {
let actual_patch = prediction.actual_patch.clone().or_else(|| {
parse_prediction_output(
@@ -103,6 +134,7 @@ pub async fn run_scoring(
}),
cumulative_logprob: prediction.cumulative_logprob,
avg_logprob: prediction.avg_logprob,
+ context: Some(&context),
},
));
}
@@ -113,11 +145,120 @@ pub async fn run_scoring(
Ok(())
}
-pub fn print_report(examples: &[Example], verbose: bool) {
+pub fn run_context_coverage_scoring(
+ example: &mut Example,
+ example_progress: &ExampleProgress,
+ retrieved_context_byte_limit: Option<usize>,
+) -> anyhow::Result<()> {
+ let progress = example_progress.start(Step::Score);
+
+ progress.set_substatus("computing context coverage");
+ let prompt_inputs = example
+ .prompt_inputs
+ .as_ref()
+ .context("prompt_inputs is required for context coverage scoring")?;
+ let context = context_excerpts(example, prompt_inputs, retrieved_context_byte_limit);
+
+ let editable_context_coverage = example
+ .spec
+ .expected_patches_with_cursor_positions()
+ .iter()
+ .map(|(expected_patch, _)| {
+ edit_prediction_metrics::editable_context_coverage(expected_patch, &context)
+ })
+ .max_by(|left, right| {
+ left.lines_f1
+ .total_cmp(&right.lines_f1)
+ .then_with(|| left.files_f1.total_cmp(&right.files_f1))
+ });
+
+ let mut score = edit_prediction_metrics::PredictionScore::zero();
+ score.editable_context_coverage = editable_context_coverage;
+ example.score = vec![score];
+
+ Ok(())
+}
+
+fn context_excerpts(
+ _example: &Example,
+ prompt_inputs: &zeta_prompt::ZetaPromptInput,
+ retrieved_context_byte_limit: Option<usize>,
+) -> Vec<Excerpt> {
+ let mut context = Vec::new();
+
+ if let Some(excerpt_start_row) = prompt_inputs.excerpt_start_row {
+ let row_count = prompt_inputs.cursor_excerpt.lines().count() as u32;
+
+ context.push(Excerpt {
+ path: prompt_inputs.cursor_path.to_string_lossy().to_string(),
+ row_range: excerpt_start_row..excerpt_start_row.saturating_add(row_count),
+ content: prompt_inputs.cursor_excerpt.to_string(),
+ });
+ }
+
+ if let Some(related_files) = &prompt_inputs.related_files {
+ let related_files = if let Some(max_bytes) = retrieved_context_byte_limit {
+ limit_retrieved_context_to_bytes(related_files, max_bytes)
+ } else {
+ related_files.clone()
+ };
+ for related_file in &related_files {
+ for excerpt in &related_file.excerpts {
+ // First component is a project name which is not present in expected patch, skip it
+ let path = related_file
+ .path
+ .iter()
+ .skip(1)
+ .collect::<std::path::PathBuf>()
+ .to_string_lossy()
+ .to_string();
+ context.push(Excerpt {
+ path,
+ row_range: excerpt.row_range.clone(),
+ content: excerpt.text.to_string(),
+ });
+ }
+ }
+ }
+
+ context
+}
+
+fn retrieved_context_bytes(
+ example: &Example,
+ retrieved_context_byte_limit: Option<usize>,
+) -> Option<usize> {
+ let related_files = example.prompt_inputs.as_ref()?.related_files.as_ref()?;
+ let related_files = if let Some(max_bytes) = retrieved_context_byte_limit {
+ limit_retrieved_context_to_bytes(related_files, max_bytes)
+ } else {
+ related_files.clone()
+ };
+ Some(
+ related_files
+ .iter()
+ .flat_map(|file| file.excerpts.iter())
+ .map(|excerpt| excerpt.text.len())
+ .sum::<usize>(),
+ )
+}
+
+pub fn print_report(
+ examples: &[Example],
+ verbose: bool,
+ context_only: bool,
+ retrieved_context_byte_limit: Option<usize>,
+) {
const MAX_EXAMPLES_DEFAULT: usize = 20;
use crate::metrics::ClassificationMetrics;
const LINE_WIDTH: usize = 101;
+
+ if context_only {
+ print_context_coverage_report(examples, verbose, retrieved_context_byte_limit);
+ return;
+ }
+
let separator = "─".repeat(LINE_WIDTH);
println!("{}", separator);
@@ -154,14 +295,30 @@ pub fn print_report(examples: &[Example], verbose: bool) {
let mut discarded_chars_total: usize = 0;
let mut recall_rate_sum: f64 = 0.0;
let mut recall_rate_count: usize = 0;
+ let mut editable_context_coverage_count: usize = 0;
+ let mut editable_context_lines_precision_sum = 0.0;
+ let mut editable_context_lines_recall_sum = 0.0;
+ let mut editable_context_lines_f1_sum = 0.0;
+ let mut editable_context_files_precision_sum = 0.0;
+ let mut editable_context_files_recall_sum = 0.0;
+ let mut editable_context_files_f1_sum = 0.0;
+ let mut total_editable_context_lines = ClassificationMetrics::default();
+ let mut total_editable_context_files = ClassificationMetrics::default();
let mut patch_inserted_tokens: Vec<usize> = Vec::new();
let mut patch_deleted_tokens: Vec<usize> = Vec::new();
let mut predictions_with_patch: usize = 0;
+ let mut retrieved_context_bytes_sum = 0.0;
+ let mut retrieved_context_bytes_count = 0;
let mut printed_lines: usize = 0;
let mut skipped_lines: usize = 0;
for example in examples {
+ if let Some(bytes) = retrieved_context_bytes(example, retrieved_context_byte_limit) {
+ retrieved_context_bytes_sum += bytes as f64;
+ retrieved_context_bytes_count += 1;
+ }
+
for (score_idx, score) in example.score.iter().enumerate() {
let exact_lines = score.exact_lines_counts();
@@ -264,6 +421,25 @@ pub fn print_report(examples: &[Example], verbose: bool) {
recall_rate_sum += rr;
recall_rate_count += 1;
}
+ if let Some(coverage) = &score.editable_context_coverage {
+ editable_context_coverage_count += 1;
+ editable_context_lines_precision_sum += coverage.lines_precision;
+ editable_context_lines_recall_sum += coverage.lines_recall;
+ editable_context_lines_f1_sum += coverage.lines_f1;
+ editable_context_files_precision_sum += coverage.files_precision;
+ editable_context_files_recall_sum += coverage.files_recall;
+ editable_context_files_f1_sum += coverage.files_f1;
+ total_editable_context_lines.accumulate(&ClassificationMetrics {
+ true_positives: coverage.lines_tp,
+ false_positives: coverage.lines_fp,
+ false_negatives: coverage.lines_fn,
+ });
+ total_editable_context_files.accumulate(&ClassificationMetrics {
+ true_positives: coverage.files_tp,
+ false_positives: coverage.files_fp,
+ false_negatives: coverage.files_fn,
+ });
+ }
// Accumulate token change metrics (only for predictions that produced a patch)
let has_patch = example
@@ -414,6 +590,36 @@ pub fn print_report(examples: &[Example], verbose: bool) {
recall_rate_count
);
}
+ if retrieved_context_bytes_count > 0 {
+ println!(
+ "Retrieved context size: {:.0} bytes avg ({} examples)",
+ retrieved_context_bytes_sum / retrieved_context_bytes_count as f64,
+ retrieved_context_bytes_count
+ );
+ }
+ if editable_context_coverage_count > 0 {
+ let count = editable_context_coverage_count as f64;
+ println!(
+ "Editable context lines: P={:.1}%, R={:.1}%, F1={:.1}% avg ({} evaluated, TP={}, FP={}, FN={})",
+ editable_context_lines_precision_sum / count * 100.0,
+ editable_context_lines_recall_sum / count * 100.0,
+ editable_context_lines_f1_sum / count * 100.0,
+ editable_context_coverage_count,
+ total_editable_context_lines.true_positives,
+ total_editable_context_lines.false_positives,
+ total_editable_context_lines.false_negatives
+ );
+ println!(
+ "Editable context files: P={:.1}%, R={:.1}%, F1={:.1}% avg ({} evaluated, TP={}, FP={}, FN={})",
+ editable_context_files_precision_sum / count * 100.0,
+ editable_context_files_recall_sum / count * 100.0,
+ editable_context_files_f1_sum / count * 100.0,
+ editable_context_coverage_count,
+ total_editable_context_files.true_positives,
+ total_editable_context_files.false_positives,
+ total_editable_context_files.false_negatives
+ );
+ }
// Print token change percentile summary (only for predictions with a patch)
if !patch_inserted_tokens.is_empty() {
@@ -470,6 +676,148 @@ pub fn print_report(examples: &[Example], verbose: bool) {
println!("\n");
}
+fn print_context_coverage_report(
+ examples: &[Example],
+ verbose: bool,
+ retrieved_context_byte_limit: Option<usize>,
+) {
+ const MAX_EXAMPLES_DEFAULT: usize = 20;
+ const LINE_WIDTH: usize = 120;
+
+ use crate::metrics::ClassificationMetrics;
+
+ let separator = "─".repeat(LINE_WIDTH);
+ println!("{}", separator);
+ println!(
+ "{:<40} {:>6} {:>6} {:>6} {:>5} {:>5} {:>5} {:>6} {:>6} {:>6} {:>5} {:>5} {:>5}",
+ "Example",
+ "LineP",
+ "LineR",
+ "LineF1",
+ "LTP",
+ "LFP",
+ "LFN",
+ "FileP",
+ "FileR",
+ "FileF1",
+ "FTP",
+ "FFP",
+ "FFN"
+ );
+ println!("{}", separator);
+
+ let mut total_lines = ClassificationMetrics::default();
+ let mut total_files = ClassificationMetrics::default();
+ let mut line_precision_sum = 0.0;
+ let mut line_recall_sum = 0.0;
+ let mut line_f1_sum = 0.0;
+ let mut file_precision_sum = 0.0;
+ let mut file_recall_sum = 0.0;
+ let mut file_f1_sum = 0.0;
+ let mut total_scores = 0;
+ let mut retrieved_context_bytes_sum = 0.0;
+ let mut retrieved_context_bytes_count = 0;
+ let mut printed_lines = 0;
+ let mut skipped_lines = 0;
+
+ for example in examples {
+ if let Some(bytes) = retrieved_context_bytes(example, retrieved_context_byte_limit) {
+ retrieved_context_bytes_sum += bytes as f64;
+ retrieved_context_bytes_count += 1;
+ }
+
+ for score in &example.score {
+ let Some(coverage) = &score.editable_context_coverage else {
+ continue;
+ };
+
+ if verbose || printed_lines < MAX_EXAMPLES_DEFAULT {
+ println!(
+ "{:<40} {:>5.1}% {:>5.1}% {:>5.1}% {:>5} {:>5} {:>5} {:>5.1}% {:>5.1}% {:>5.1}% {:>5} {:>5} {:>5}",
+ truncate_name(&example.spec.name, 40),
+ coverage.lines_precision * 100.0,
+ coverage.lines_recall * 100.0,
+ coverage.lines_f1 * 100.0,
+ coverage.lines_tp,
+ coverage.lines_fp,
+ coverage.lines_fn,
+ coverage.files_precision * 100.0,
+ coverage.files_recall * 100.0,
+ coverage.files_f1 * 100.0,
+ coverage.files_tp,
+ coverage.files_fp,
+ coverage.files_fn
+ );
+ printed_lines += 1;
+ } else {
+ skipped_lines += 1;
+ }
+
+ total_scores += 1;
+ line_precision_sum += coverage.lines_precision;
+ line_recall_sum += coverage.lines_recall;
+ line_f1_sum += coverage.lines_f1;
+ file_precision_sum += coverage.files_precision;
+ file_recall_sum += coverage.files_recall;
+ file_f1_sum += coverage.files_f1;
+ total_lines.accumulate(&ClassificationMetrics {
+ true_positives: coverage.lines_tp,
+ false_positives: coverage.lines_fp,
+ false_negatives: coverage.lines_fn,
+ });
+ total_files.accumulate(&ClassificationMetrics {
+ true_positives: coverage.files_tp,
+ false_positives: coverage.files_fp,
+ false_negatives: coverage.files_fn,
+ });
+ }
+ }
+
+ if skipped_lines > 0 {
+ println!(
+ "{:<40} (use --verbose to see all {} examples)",
+ format!("... and {} more", skipped_lines),
+ printed_lines + skipped_lines
+ );
+ }
+
+ println!("{}", separator);
+
+ if total_scores > 0 {
+ let count = total_scores as f64;
+ println!(
+ "{:<40} {:>5.1}% {:>5.1}% {:>5.1}% {:>5} {:>5} {:>5} {:>5.1}% {:>5.1}% {:>5.1}% {:>5} {:>5} {:>5}",
+ "TOTAL / AVERAGE",
+ line_precision_sum / count * 100.0,
+ line_recall_sum / count * 100.0,
+ line_f1_sum / count * 100.0,
+ total_lines.true_positives,
+ total_lines.false_positives,
+ total_lines.false_negatives,
+ file_precision_sum / count * 100.0,
+ file_recall_sum / count * 100.0,
+ file_f1_sum / count * 100.0,
+ total_files.true_positives,
+ total_files.false_positives,
+ total_files.false_negatives
+ );
+ println!("{}", separator);
+ println!(
+ "Evaluated editable context coverage for {} examples",
+ total_scores
+ );
+ if retrieved_context_bytes_count > 0 {
+ println!(
+ "Retrieved context size: {:.0} bytes avg ({} examples)",
+ retrieved_context_bytes_sum / retrieved_context_bytes_count as f64,
+ retrieved_context_bytes_count
+ );
+ }
+ }
+
+ println!("\n");
+}
+
fn percentile(sorted_values: &[usize], p: usize) -> usize {
if sorted_values.is_empty() {
return 0;
diff --git a/crates/edit_prediction_context/src/edit_prediction_context.rs b/crates/edit_prediction_context/src/edit_prediction_context.rs
index 5671df6092e..97a1729ec64 100644
--- a/crates/edit_prediction_context/src/edit_prediction_context.rs
+++ b/crates/edit_prediction_context/src/edit_prediction_context.rs
@@ -22,10 +22,16 @@ use util::{RangeExt as _, ResultExt};
mod assemble_excerpts;
#[cfg(test)]
mod edit_prediction_context_tests;
+mod editable_context;
#[cfg(test)]
mod fake_definition_lsp;
+mod git_log_context;
-pub use zeta_prompt::{RelatedExcerpt, RelatedFile};
+pub use editable_context::{
+ EditHistoryContextEntry, collect_editable_context, limit_retrieved_context_to_bytes,
+};
+
+pub use zeta_prompt::{ContextSource, RelatedExcerpt, RelatedFile};
const IDENTIFIER_LINE_COUNT: u32 = 3;
@@ -573,6 +579,7 @@ impl RelatedBuffer {
row_range: start.row..end.row,
text: buffer.text_for_range(start..end).collect::<String>().into(),
order,
+ context_source: ContextSource::Lsp,
}
})
.collect::<Vec<_>>();
diff --git a/crates/edit_prediction_context/src/edit_prediction_context_tests.rs b/crates/edit_prediction_context/src/edit_prediction_context_tests.rs
index 32dc37b9532..5b8e9c6aecb 100644
--- a/crates/edit_prediction_context/src/edit_prediction_context_tests.rs
+++ b/crates/edit_prediction_context/src/edit_prediction_context_tests.rs
@@ -308,6 +308,7 @@ async fn test_assemble_excerpts(cx: &mut TestAppContext) {
row_range,
text: buffer.text_for_range(start..end).collect::<String>().into(),
order,
+ context_source: ContextSource::Lsp,
}
})
.collect();
diff --git a/crates/edit_prediction_context/src/editable_context.rs b/crates/edit_prediction_context/src/editable_context.rs
new file mode 100644
index 00000000000..c1beaaf634d
--- /dev/null
+++ b/crates/edit_prediction_context/src/editable_context.rs
@@ -0,0 +1,616 @@
+use collections::{HashMap, HashSet};
+use gpui::{App, AppContext as _, AsyncApp, Entity, EntityId};
+use language::{Buffer, BufferSnapshot, Point, ToPoint as _};
+use project::{Project, ProjectPath};
+use std::{
+ ops::Range,
+ path::{Path, PathBuf},
+ sync::Arc,
+};
+use text::Anchor;
+use util::{paths::PathStyle, rel_path::RelPath};
+use zeta_prompt::{ContextSource, RelatedExcerpt, RelatedFile};
+
+use crate::git_log_context::build_git_log_index;
+
+/// This module contains collectors for editable context:
+/// excerpts or full files that are likely to be edited.
+const CURSOR_CONTEXT_LINE_COUNT: u32 = 20;
+const EDIT_HISTORY_CONTEXT_LINE_COUNT: u32 = 20;
+const GIT_LOG_CONTEXT_LINE_COUNT: u32 = 10000;
+const GIT_LOG_CONTEXT_FILE_COUNT: usize = 10;
+
+type RangesByBuffer = HashMap<EntityId, (Entity<Buffer>, Vec<EditableContextRange>)>;
+
+#[derive(Clone)]
+pub struct EditHistoryContextEntry {
+ pub buffer: Entity<Buffer>,
+ pub edited_range: Range<Anchor>,
+}
+
+struct EditableContextRange {
+ range: Range<Anchor>,
+ order: usize,
+ context_source: ContextSource,
+}
+
+struct ResolvedEditableContextRange {
+ range: Range<Point>,
+ order: usize,
+ context_source: ContextSource,
+}
+
+pub async fn collect_editable_context(
+ project: Entity<Project>,
+ active_buffer: Entity<Buffer>,
+ cursor_position: Anchor,
+ edit_history: Vec<EditHistoryContextEntry>,
+ oracle_paths: Vec<Arc<Path>>,
+ context_sources: Vec<ContextSource>,
+ cx: &mut AsyncApp,
+) -> anyhow::Result<Vec<RelatedFile>> {
+ let mut ranges_by_buffer = RangesByBuffer::default();
+
+ if context_sources.contains(&ContextSource::CursorExcerpt) {
+ collect_cursor_excerpt_context(
+ &mut ranges_by_buffer,
+ active_buffer.clone(),
+ cursor_position,
+ cx,
+ );
+ }
+ if context_sources.contains(&ContextSource::CurrentFile) {
+ collect_current_file_context(&mut ranges_by_buffer, active_buffer.clone(), cx);
+ }
+ if context_sources.contains(&ContextSource::EditHistory) {
+ collect_edit_history_context(&mut ranges_by_buffer, &edit_history, cx);
+ }
+ if context_sources.contains(&ContextSource::EditHistoryFile) {
+ collect_edit_history_file_context(&mut ranges_by_buffer, &edit_history, cx);
+ }
+ if context_sources.contains(&ContextSource::GitLog) {
+ collect_git_log_context(&mut ranges_by_buffer, project.clone(), active_buffer, cx).await;
+ }
+ if context_sources.contains(&ContextSource::OracleFile) {
+ collect_oracle_file_context(&mut ranges_by_buffer, project.clone(), oracle_paths, cx).await;
+ }
+
+ Ok(cx.update(|cx| {
+ let project = project.read(cx);
+ let mut related_files = ranges_by_buffer
+ .into_values()
+ .filter_map(|(buffer, ranges)| related_file_for_ranges(&project, &buffer, ranges, cx))
+ .collect::<Vec<_>>();
+ related_files.sort_by_key(|file| {
+ file.excerpts
+ .iter()
+ .map(|excerpt| excerpt.order)
+ .min()
+ .unwrap_or(usize::MAX)
+ });
+ related_files
+ }))
+}
+
+pub fn limit_retrieved_context_to_bytes(
+ related_files: &[RelatedFile],
+ max_bytes: usize,
+) -> Vec<RelatedFile> {
+ struct ExcerptCandidate {
+ file_index: usize,
+ excerpt_index: usize,
+ order: usize,
+ }
+
+ let mut candidates = related_files
+ .iter()
+ .enumerate()
+ .flat_map(|(file_index, file)| {
+ file.excerpts
+ .iter()
+ .enumerate()
+ .map(move |(excerpt_index, excerpt)| ExcerptCandidate {
+ file_index,
+ excerpt_index,
+ order: excerpt.order,
+ })
+ })
+ .collect::<Vec<_>>();
+ candidates.sort_by_key(|candidate| {
+ (
+ candidate.order,
+ candidate.file_index,
+ candidate.excerpt_index,
+ )
+ });
+
+ let mut selected_excerpts = related_files
+ .iter()
+ .map(|file| vec![false; file.excerpts.len()])
+ .collect::<Vec<_>>();
+ let mut covered_ranges_by_file = vec![Vec::<Range<u32>>::new(); related_files.len()];
+ let mut selected_bytes: usize = 0;
+
+ for candidate in candidates {
+ let file = &related_files[candidate.file_index];
+ let excerpt = &file.excerpts[candidate.excerpt_index];
+ let added_bytes =
+ uncovered_excerpt_bytes(excerpt, &covered_ranges_by_file[candidate.file_index]);
+ if added_bytes == 0 || selected_bytes.saturating_add(added_bytes) > max_bytes {
+ continue;
+ }
+
+ selected_bytes += added_bytes;
+ selected_excerpts[candidate.file_index][candidate.excerpt_index] = true;
+ push_covered_range(
+ &mut covered_ranges_by_file[candidate.file_index],
+ excerpt.row_range.clone(),
+ );
+ }
+
+ related_files
+ .iter()
+ .enumerate()
+ .filter_map(|(file_index, file)| {
+ let excerpts = file
+ .excerpts
+ .iter()
+ .enumerate()
+ .filter_map(|(excerpt_index, excerpt)| {
+ selected_excerpts[file_index][excerpt_index].then(|| excerpt.clone())
+ })
+ .collect::<Vec<_>>();
+ if excerpts.is_empty() {
+ return None;
+ }
+
+ Some(RelatedFile {
+ path: file.path.clone(),
+ max_row: file.max_row,
+ excerpts,
+ in_open_source_repo: file.in_open_source_repo,
+ })
+ })
+ .collect()
+}
+
+fn uncovered_excerpt_bytes(excerpt: &RelatedExcerpt, covered_ranges: &[Range<u32>]) -> usize {
+ let mut bytes = 0;
+
+ for (row, line) in (excerpt.row_range.start..).zip(excerpt.text.split_inclusive('\n')) {
+ if row >= excerpt.row_range.end {
+ break;
+ }
+ if !covered_ranges
+ .iter()
+ .any(|covered_range| covered_range.contains(&row))
+ {
+ bytes += line.len();
+ }
+ }
+
+ bytes
+}
+
+fn push_covered_range(covered_ranges: &mut Vec<Range<u32>>, range: Range<u32>) {
+ covered_ranges.push(range);
+ covered_ranges.sort_by_key(|range| (range.start, range.end));
+
+ let mut merged_ranges: Vec<Range<u32>> = Vec::new();
+ for range in covered_ranges.drain(..) {
+ if let Some(last_range) = merged_ranges.last_mut()
+ && range.start <= last_range.end
+ {
+ last_range.end = last_range.end.max(range.end);
+ continue;
+ }
+
+ merged_ranges.push(range);
+ }
+
+ *covered_ranges = merged_ranges;
+}
+
+fn collect_cursor_excerpt_context(
+ ranges_by_buffer: &mut RangesByBuffer,
+ active_buffer: Entity<Buffer>,
+ cursor_position: Anchor,
+ cx: &mut AsyncApp,
+) {
+ let cursor_range = active_buffer.read_with(cx, |buffer, _cx| {
+ let snapshot = buffer.snapshot();
+ expanded_anchor_range(
+ &snapshot,
+ cursor_position..cursor_position,
+ CURSOR_CONTEXT_LINE_COUNT,
+ )
+ });
+
+ push_context_range(
+ ranges_by_buffer,
+ active_buffer,
+ cursor_range,
+ 0,
+ ContextSource::CursorExcerpt,
+ );
+}
+
+fn collect_current_file_context(
+ ranges_by_buffer: &mut RangesByBuffer,
+ active_buffer: Entity<Buffer>,
+ cx: &mut AsyncApp,
+) {
+ collect_full_buffer_context(
+ ranges_by_buffer,
+ active_buffer,
+ 0,
+ ContextSource::CurrentFile,
+ cx,
+ );
+}
+
+fn collect_edit_history_context(
+ ranges_by_buffer: &mut RangesByBuffer,
+ edit_history: &[EditHistoryContextEntry],
+ cx: &mut AsyncApp,
+) {
+ for (index, entry) in edit_history.iter().enumerate() {
+ let edit_history_range = entry.buffer.read_with(cx, |buffer, _cx| {
+ expanded_anchor_range(
+ &buffer.snapshot(),
+ entry.edited_range.clone(),
+ EDIT_HISTORY_CONTEXT_LINE_COUNT,
+ )
+ });
+
+ push_context_range(
+ ranges_by_buffer,
+ entry.buffer.clone(),
+ edit_history_range,
+ index + 1,
+ ContextSource::EditHistory,
+ );
+ }
+}
+
+fn collect_edit_history_file_context(
+ ranges_by_buffer: &mut RangesByBuffer,
+ edit_history: &[EditHistoryContextEntry],
+ cx: &mut AsyncApp,
+) {
+ let next_order = next_context_order(ranges_by_buffer);
+ let mut seen_buffers = HashSet::default();
+ let mut index = 0;
+
+ for entry in edit_history {
+ if !seen_buffers.insert(entry.buffer.entity_id()) {
+ continue;
+ }
+
+ collect_full_buffer_context(
+ ranges_by_buffer,
+ entry.buffer.clone(),
+ next_order + index,
+ ContextSource::EditHistoryFile,
+ cx,
+ );
+ index += 1;
+ }
+}
+
+async fn collect_oracle_file_context(
+ ranges_by_buffer: &mut RangesByBuffer,
+ project: Entity<Project>,
+ oracle_paths: Vec<Arc<Path>>,
+ cx: &mut AsyncApp,
+) {
+ let next_order = next_context_order(ranges_by_buffer);
+ let mut seen_buffers = HashSet::default();
+ let mut index = 0;
+
+ for path in oracle_paths {
+ let buffer = match open_buffer_for_path(&project, &path, cx).await {
+ Ok(Some(buffer)) => buffer,
+ Ok(None) => {
+ log::debug!("failed to find oracle file path: {}", path.display());
+ continue;
+ }
+ Err(error) => {
+ log::debug!(
+ "failed to open oracle file path {}: {error:#}",
+ path.display()
+ );
+ continue;
+ }
+ };
+
+ if !seen_buffers.insert(buffer.entity_id()) {
+ continue;
+ }
+
+ collect_full_buffer_context(
+ ranges_by_buffer,
+ buffer,
+ next_order + index,
+ ContextSource::OracleFile,
+ cx,
+ );
+ index += 1;
+ }
+}
+
+async fn open_buffer_for_path(
+ project: &Entity<Project>,
+ path: &Path,
+ cx: &mut AsyncApp,
+) -> anyhow::Result<Option<Entity<Buffer>>> {
+ let path = path.to_path_buf();
+ let path_without_prefix: PathBuf = path.components().skip(1).collect();
+ let project_path = project.update(cx, |project, cx| {
+ project.find_project_path(&path, cx).or_else(|| {
+ if path_without_prefix.as_os_str().is_empty() {
+ None
+ } else {
+ project.find_project_path(&path_without_prefix, cx)
+ }
+ })
+ });
+
+ let Some(project_path) = project_path else {
+ return Ok(None);
+ };
+
+ project
+ .update(cx, |project, cx| project.open_buffer(project_path, cx))
+ .await
+ .map(Some)
+}
+
+fn collect_full_buffer_context(
+ ranges_by_buffer: &mut RangesByBuffer,
+ buffer: Entity<Buffer>,
+ order: usize,
+ context_source: ContextSource,
+ cx: &mut AsyncApp,
+) {
+ let range = buffer.read_with(cx, |buffer, _cx| full_file_anchor_range(&buffer.snapshot()));
+ push_context_range(ranges_by_buffer, buffer, range, order, context_source);
+}
+
+fn full_file_anchor_range(snapshot: &BufferSnapshot) -> Range<Anchor> {
+ let start = snapshot.anchor_before(Point::new(0, 0));
+ let max_point = snapshot.max_point();
+ let end = snapshot.anchor_after(max_point);
+ start..end
+}
+
+fn next_context_order(ranges_by_buffer: &RangesByBuffer) -> usize {
+ ranges_by_buffer
+ .values()
+ .flat_map(|(_, ranges)| ranges.iter().map(|range| range.order))
+ .max()
+ .map_or(0, |order| order + 1)
+}
+
+async fn collect_git_log_context(
+ ranges_by_buffer: &mut RangesByBuffer,
+ project: Entity<Project>,
+ active_buffer: Entity<Buffer>,
+ cx: &mut AsyncApp,
+) {
+ let Some((worktree_id, active_path, worktree_abs_path)) = cx.update(|cx| {
+ let buffer = active_buffer.read(cx);
+ let file = buffer.file()?;
+ let project = project.read(cx);
+ if !project.is_local() {
+ return None;
+ }
+ let worktree = project.worktree_for_id(file.worktree_id(cx), cx)?;
+ let worktree = worktree.read(cx);
+ if !worktree.is_local() {
+ return None;
+ }
+ Some((
+ file.worktree_id(cx),
+ file.path().clone(),
+ worktree.abs_path(),
+ ))
+ }) else {
+ return;
+ };
+
+ let index_result = cx
+ .background_spawn(async move { build_git_log_index(&worktree_abs_path).await })
+ .await;
+ let index = match index_result {
+ Ok(index) => index,
+ Err(error) => {
+ log::debug!("failed to build git log context index: {error:#}");
+ return;
+ }
+ };
+
+ let next_order = next_context_order(ranges_by_buffer);
+
+ for (index, related_path) in index
+ .get_related(active_path.as_std_path(), GIT_LOG_CONTEXT_FILE_COUNT)
+ .into_iter()
+ .enumerate()
+ {
+ let Ok(related_path) = RelPath::new(&related_path, PathStyle::Posix) else {
+ continue;
+ };
+ let project_path = ProjectPath {
+ worktree_id,
+ path: related_path.into_owned().into(),
+ };
+ let buffer = match project
+ .update(cx, |project, cx| project.open_buffer(project_path, cx))
+ .await
+ {
+ Ok(buffer) => buffer,
+ Err(error) => {
+ log::debug!("failed to open git log related buffer: {error:#}");
+ continue;
+ }
+ };
+
+ let range = buffer.read_with(cx, |buffer, _cx| {
+ let snapshot = buffer.snapshot();
+ let max_row = GIT_LOG_CONTEXT_LINE_COUNT.min(snapshot.max_point().row);
+ let end = snapshot.anchor_after(Point::new(max_row, snapshot.line_len(max_row)));
+ snapshot.anchor_before(Point::new(0, 0))..end
+ });
+
+ push_context_range(
+ ranges_by_buffer,
+ buffer,
+ range,
+ next_order + index,
+ ContextSource::GitLog,
+ );
+ }
+}
+
+fn expanded_anchor_range(
+ snapshot: &BufferSnapshot,
+ range: Range<Anchor>,
+ context_line_count: u32,
+) -> Range<Anchor> {
+ let start = range.start.to_point(snapshot);
+ let end = range.end.to_point(snapshot);
+ let start_row = start.row.saturating_sub(context_line_count);
+ let end_row = end
+ .row
+ .saturating_add(context_line_count)
+ .min(snapshot.max_point().row);
+ let start = snapshot.anchor_before(Point::new(start_row, 0));
+ let end = snapshot.anchor_after(Point::new(end_row, snapshot.line_len(end_row)));
+ start..end
+}
+
+fn push_context_range(
+ ranges_by_buffer: &mut RangesByBuffer,
+ buffer: Entity<Buffer>,
+ range: Range<Anchor>,
+ order: usize,
+ context_source: ContextSource,
+) {
+ ranges_by_buffer
+ .entry(buffer.entity_id())
+ .or_insert_with(|| (buffer.clone(), Vec::new()))
+ .1
+ .push(EditableContextRange {
+ range,
+ order,
+ context_source,
+ });
+}
+
+fn related_file_for_ranges(
+ project: &Project,
+ buffer: &Entity<Buffer>,
+ ranges: Vec<EditableContextRange>,
+ cx: &App,
+) -> Option<RelatedFile> {
+ let buffer = buffer.read(cx);
+ let snapshot = buffer.snapshot();
+ let file = snapshot.file()?;
+ let worktree = project.worktree_for_id(file.worktree_id(cx), cx)?;
+ let path: Arc<Path> = Path::new(&format!(
+ "{}/{}",
+ worktree.read(cx).root_name().as_unix_str(),
+ file.path().as_unix_str()
+ ))
+ .into();
+
+ let ranges = resolved_context_ranges(ranges, &snapshot);
+
+ let mut excerpts = ranges
+ .into_iter()
+ .map(|range| RelatedExcerpt {
+ row_range: range.range.start.row..range.range.end.row,
+ text: snapshot
+ .text_for_range(range.range)
+ .collect::<String>()
+ .into(),
+ order: range.order,
+ context_source: range.context_source,
+ })
+ .collect::<Vec<_>>();
+ excerpts.sort_by_key(|excerpt| excerpt.order);
+
+ Some(RelatedFile {
+ path,
+ max_row: snapshot.max_point().row,
+ excerpts,
+ in_open_source_repo: false,
+ })
+}
+
+fn resolved_context_ranges(
+ ranges: Vec<EditableContextRange>,
+ snapshot: &BufferSnapshot,
+) -> Vec<ResolvedEditableContextRange> {
+ ranges
+ .into_iter()
+ .filter_map(|range| {
+ let start = range.range.start.to_point(snapshot);
+ let end = range.range.end.to_point(snapshot);
+ if start >= end {
+ return None;
+ }
+
+ Some(ResolvedEditableContextRange {
+ range: start..end,
+ order: range.order,
+ context_source: range.context_source,
+ })
+ })
+ .collect()
+}
+
+#[allow(dead_code)]
+fn merge_overlapping_ranges(ranges: &mut Vec<ResolvedEditableContextRange>) {
+ ranges.sort_by_key(|range| (range.range.start, range.range.end));
+ let mut merged: Vec<ResolvedEditableContextRange> = Vec::new();
+
+ for range in ranges.drain(..) {
+ if let Some(last_range) = merged.last_mut()
+ && range.range.start <= last_range.range.end
+ {
+ if context_source_order(range.context_source)
+ < context_source_order(last_range.context_source)
+ {
+ last_range.context_source = range.context_source;
+ }
+ last_range.range.end = last_range.range.end.max(range.range.end);
+ last_range.order = last_range.order.min(range.order);
+ continue;
+ }
+
+ merged.push(range);
+ }
+
+ *ranges = merged;
+}
+
+#[allow(dead_code)]
+fn push_context_source(context_sources: &mut Vec<ContextSource>, context_source: ContextSource) {
+ if !context_sources.contains(&context_source) {
+ context_sources.push(context_source);
+ context_sources.sort_by_key(|context_source| context_source_order(*context_source));
+ }
+}
+
+fn context_source_order(context_source: ContextSource) -> usize {
+ match context_source {
+ ContextSource::Lsp => 0,
+ ContextSource::CursorExcerpt => 1,
+ ContextSource::CurrentFile => 2,
+ ContextSource::EditHistory => 3,
+ ContextSource::EditHistoryFile => 4,
+ ContextSource::GitLog => 5,
+ ContextSource::OracleFile => 6,
+ }
+}
diff --git a/crates/edit_prediction_context/src/git_log_context.rs b/crates/edit_prediction_context/src/git_log_context.rs
new file mode 100644
index 00000000000..2ed358b03fb
--- /dev/null
+++ b/crates/edit_prediction_context/src/git_log_context.rs
@@ -0,0 +1,255 @@
+// Goal:
+// - Build an index that finds files that are frequently edited in the same git commit
+// - Lookup by path and get a list of related files, sorted by most frequently edited together
+//
+// Path => Path => usize
+//
+// This is a symmetric relationship, so for a => (b, 1), also add b => (a, 1)
+
+use std::collections::HashMap;
+use std::env;
+use std::path::{Path, PathBuf};
+use std::process::ExitCode;
+
+use anyhow::{Context as _, Result, anyhow, bail};
+use util::command::new_command;
+
+pub struct GitLogIndex {
+ index: HashMap<PathBuf, HashMap<PathBuf, usize>>,
+}
+
+impl GitLogIndex {
+ pub fn new() -> Self {
+ Self {
+ index: HashMap::new(),
+ }
+ }
+
+ pub fn add_related(&mut self, path: PathBuf, related: PathBuf) {
+ let count = self
+ .index
+ .entry(path.clone())
+ .or_default()
+ .entry(related.clone())
+ .or_default();
+ *count += 1;
+
+ // add the reverse mapping
+ let reverse_count = self
+ .index
+ .entry(related)
+ .or_default()
+ .entry(path)
+ .or_default();
+ *reverse_count += 1;
+ }
+
+ pub fn get_related(&self, path: &Path, n: usize) -> Vec<PathBuf> {
+ self.get_related_with_counts(path, n)
+ .into_iter()
+ .map(|(path, _)| path)
+ .collect()
+ }
+
+ pub fn get_related_with_counts(&self, path: &Path, n: usize) -> Vec<(PathBuf, usize)> {
+ let Some(counts) = self.index.get(path) else {
+ return Vec::new();
+ };
+
+ let mut related: Vec<_> = counts.iter().collect();
+ related.sort_by(|(left_path, left_count), (right_path, right_count)| {
+ right_count
+ .cmp(left_count)
+ .then_with(|| left_path.cmp(right_path))
+ });
+ related
+ .into_iter()
+ .take(n)
+ .map(|(path, count)| (path.clone(), *count))
+ .collect()
+ }
+}
+
+impl Default for GitLogIndex {
+ fn default() -> Self {
+ Self::new()
+ }
+}
+
+pub async fn build_git_log_index(worktree_dir: &Path) -> Result<GitLogIndex> {
+ let mut index = GitLogIndex::new();
+
+ let output = new_command("git")
+ .arg("log")
+ .arg("-5000")
+ .arg("--pretty=tformat:@@COMMIT %H")
+ .arg("--name-only")
+ .current_dir(worktree_dir)
+ .output()
+ .await
+ .with_context(|| format!("failed to run git log in {}", worktree_dir.display()))?;
+
+ if !output.status.success() {
+ let stderr = String::from_utf8_lossy(&output.stderr);
+ bail!(
+ "git log failed in {} with status {}: {}",
+ worktree_dir.display(),
+ output.status,
+ stderr.trim()
+ );
+ }
+
+ let log = String::from_utf8(output.stdout).context("git log output was not valid UTF-8")?;
+ let parsed = parse_git_log(&log);
+ for files in parsed {
+ for i in 0..files.len() {
+ for j in (i + 1)..files.len() {
+ index.add_related(files[i].clone(), files[j].clone());
+ }
+ }
+ }
+
+ Ok(index)
+}
+
+#[allow(dead_code)]
+fn main() -> ExitCode {
+ match run() {
+ Ok(()) => ExitCode::SUCCESS,
+ Err(error) => {
+ eprintln!("{error:#}");
+ ExitCode::FAILURE
+ }
+ }
+}
+
+#[allow(dead_code)]
+fn run() -> Result<()> {
+ let mut arguments = env::args_os();
+ let program_name = arguments
+ .next()
+ .and_then(|path| PathBuf::from(path).file_name().map(|name| name.to_owned()))
+ .and_then(|name| name.into_string().ok())
+ .unwrap_or_else(|| "git_log_context".to_string());
+
+ let worktree_dir = arguments.next().ok_or_else(|| {
+ print_usage(&program_name);
+ anyhow!("missing worktree path")
+ })?;
+ let query_path = arguments.next().ok_or_else(|| {
+ print_usage(&program_name);
+ anyhow!("missing query path")
+ })?;
+ if arguments.next().is_some() {
+ print_usage(&program_name);
+ bail!("too many arguments");
+ }
+
+ let worktree_dir = PathBuf::from(worktree_dir);
+ let query_path = normalize_query_path(&worktree_dir, &PathBuf::from(query_path));
+ let index = futures::executor::block_on(build_git_log_index(&worktree_dir))?;
+
+ for (path, count) in index.get_related_with_counts(&query_path, 10) {
+ println!("{count}\t{}", path.display());
+ }
+
+ Ok(())
+}
+
+#[allow(dead_code)]
+fn print_usage(program_name: &str) {
+ eprintln!("Usage: {program_name} <worktree-path> <query-path>");
+}
+
+#[allow(dead_code)]
+fn normalize_query_path(worktree_dir: &Path, query_path: &Path) -> PathBuf {
+ if query_path.is_absolute() {
+ query_path
+ .strip_prefix(worktree_dir)
+ .unwrap_or(query_path)
+ .components()
+ .collect()
+ } else {
+ query_path.components().collect()
+ }
+}
+
+fn parse_git_log(log: &str) -> Vec<Vec<PathBuf>> {
+ let mut lines = log.lines().peekable();
+ let mut commits = Vec::new();
+
+ while let Some(line) = lines.next() {
+ if line.starts_with("@@COMMIT ") {
+ // skip blank line
+ lines.next();
+ let mut files = Vec::new();
+ while let Some(next) = lines.peek()
+ && !next.starts_with("@@COMMIT ")
+ {
+ let Some(next) = lines.next() else {
+ break;
+ };
+ if !next.is_empty() {
+ files.push(next.into());
+ }
+ }
+ commits.push(files);
+ }
+ }
+
+ commits
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use indoc::indoc;
+
+ #[test]
+ fn test_git_log_index() {
+ let mut index = GitLogIndex::new();
+ index.add_related(PathBuf::from("a"), PathBuf::from("b"));
+ index.add_related(PathBuf::from("a"), PathBuf::from("b"));
+ index.add_related(PathBuf::from("a"), PathBuf::from("c"));
+ index.add_related(PathBuf::from("b"), PathBuf::from("c"));
+
+ let related = index.get_related(&PathBuf::from("a"), 100);
+ assert_eq!(related, vec![PathBuf::from("b"), PathBuf::from("c")]);
+ }
+
+ #[test]
+ fn test_parse_git_log() {
+ let log = indoc! {"
+ @@COMMIT d2e451dd48be67ef8c943e90dabc02e80a6984c9
+
+ crates/edit_prediction/src/edit_prediction.rs
+ crates/edit_prediction_cli/src/format_prompt.rs
+ crates/edit_prediction_cli/src/main.rs
+ crates/edit_prediction_cli/src/predict.rs
+ @@COMMIT d666823f348bd151067464fa31676d28bdb96717
+
+ crates/edit_prediction_cli/src/main.rs
+ crates/edit_prediction_cli/src/predict.rs
+ "};
+ let parsed = parse_git_log(log);
+
+ assert_eq!(parsed.len(), 2);
+ assert_eq!(
+ parsed[0][0],
+ PathBuf::from("crates/edit_prediction/src/edit_prediction.rs")
+ );
+ assert_eq!(
+ parsed[0][1],
+ PathBuf::from("crates/edit_prediction_cli/src/format_prompt.rs")
+ );
+
+ assert_eq!(
+ parsed[1][0],
+ PathBuf::from("crates/edit_prediction_cli/src/main.rs")
+ );
+ assert_eq!(
+ parsed[1][1],
+ PathBuf::from("crates/edit_prediction_cli/src/predict.rs")
+ );
+ }
+}
diff --git a/crates/edit_prediction_metrics/src/edit_prediction_metrics.rs b/crates/edit_prediction_metrics/src/edit_prediction_metrics.rs
index 81a37148c79..205075e2f5b 100644
--- a/crates/edit_prediction_metrics/src/edit_prediction_metrics.rs
+++ b/crates/edit_prediction_metrics/src/edit_prediction_metrics.rs
@@ -1,4 +1,6 @@
+mod jumps;
mod kept_rate;
+mod patch;
mod patch_metrics;
mod prediction_score;
mod reversal;
@@ -7,6 +9,7 @@ mod tokenize;
#[cfg(feature = "tree-sitter")]
mod tree_sitter;
+pub use jumps::{EditableContextCoverage, Excerpt, editable_context_coverage};
pub use kept_rate::AnnotatedToken;
pub use kept_rate::KeptRateResult;
pub use kept_rate::TokenAnnotation;
diff --git a/crates/edit_prediction_metrics/src/jumps.rs b/crates/edit_prediction_metrics/src/jumps.rs
new file mode 100644
index 00000000000..8cfeafcc97a
--- /dev/null
+++ b/crates/edit_prediction_metrics/src/jumps.rs
@@ -0,0 +1,505 @@
+use std::{
+ collections::{BTreeMap, BTreeSet},
+ ops::Range,
+};
+
+use crate::patch::{Patch, PatchLine};
+
+const LINE_RELEVANCE_WINDOW: u32 = 20;
+
+#[derive(Clone, Debug, Eq, PartialEq, serde::Deserialize, serde::Serialize)]
+pub struct Excerpt {
+ pub path: String,
+ pub row_range: Range<u32>,
+ pub content: String,
+}
+
+#[derive(Clone, Debug, PartialEq, serde::Deserialize, serde::Serialize)]
+pub struct EditableContextCoverage {
+ pub lines_tp: usize,
+ pub lines_fp: usize,
+ pub lines_fn: usize,
+ pub lines_precision: f64,
+ pub lines_recall: f64,
+ pub lines_f1: f64,
+
+ pub files_tp: usize,
+ pub files_fp: usize,
+ pub files_fn: usize,
+ pub files_precision: f64,
+ pub files_recall: f64,
+ pub files_f1: f64,
+}
+
+impl EditableContextCoverage {
+ pub fn new(
+ lines_tp: usize,
+ lines_fp: usize,
+ lines_fn: usize,
+ files_tp: usize,
+ files_fp: usize,
+ files_fn: usize,
+ ) -> Self {
+ Self {
+ lines_tp,
+ lines_fp,
+ lines_fn,
+ lines_precision: precision(lines_tp, lines_fp, lines_fn),
+ lines_recall: recall(lines_tp, lines_fp, lines_fn),
+ lines_f1: f1(lines_tp, lines_fp, lines_fn),
+ files_tp,
+ files_fp,
+ files_fn,
+ files_precision: precision(files_tp, files_fp, files_fn),
+ files_recall: recall(files_tp, files_fp, files_fn),
+ files_f1: f1(files_tp, files_fp, files_fn),
+ }
+ }
+}
+
+/// Measures how much expected edit context was retrieved and how much unrelated context was retrieved.
+pub fn editable_context_coverage(
+ expected_patch: &str,
+ context: &[Excerpt],
+) -> EditableContextCoverage {
+ let patch = Patch::parse_unified_diff(expected_patch);
+ let (expected_files, expected_anchor_lines, relevant_lines) = expected_context(&patch);
+ let (retrieved_files, retrieved_lines) = retrieved_context(context);
+
+ let (lines_tp, lines_fp) = classify_retrieved_rows(&relevant_lines, &retrieved_lines);
+ let (lines_fn, expected_anchor_line_count) =
+ count_missing_rows(&expected_anchor_lines, &retrieved_lines);
+ let (files_tp, files_fp, files_fn) = classify_values(&expected_files, &retrieved_files);
+
+ let lines_precision = precision(lines_tp, lines_fp, lines_fn);
+ let lines_recall = line_recall(expected_anchor_line_count, lines_fn);
+ let lines_f1 = f1_from_precision_and_recall(lines_precision, lines_recall);
+ let files_precision = precision(files_tp, files_fp, files_fn);
+ let files_recall = recall(files_tp, files_fp, files_fn);
+ let files_f1 = f1_from_precision_and_recall(files_precision, files_recall);
+
+ EditableContextCoverage {
+ lines_tp,
+ lines_fp,
+ lines_fn,
+ lines_precision,
+ lines_recall,
+ lines_f1,
+ files_tp,
+ files_fp,
+ files_fn,
+ files_precision,
+ files_recall,
+ files_f1,
+ }
+}
+
+fn expected_context(
+ patch: &Patch,
+) -> (
+ BTreeSet<String>,
+ BTreeMap<String, BTreeSet<u32>>,
+ BTreeMap<String, BTreeSet<u32>>,
+) {
+ let mut expected_files = BTreeSet::new();
+ let mut expected_anchor_lines = BTreeMap::new();
+ let mut relevant_lines = BTreeMap::new();
+
+ for hunk in &patch.hunks {
+ if hunk
+ .lines
+ .iter()
+ .any(|line| matches!(line, PatchLine::Addition(_) | PatchLine::Deletion(_)))
+ {
+ expected_files.insert(hunk.filename.clone());
+ }
+
+ let mut old_row = hunk.old_start.saturating_sub(1).max(0) as u32;
+ let mut previous_context_row = None;
+ let mut index = 0;
+
+ while index < hunk.lines.len() {
+ match &hunk.lines[index] {
+ PatchLine::Context(_) => {
+ previous_context_row = Some(old_row);
+ old_row = old_row.saturating_add(1);
+ index += 1;
+ }
+ PatchLine::Addition(_) | PatchLine::Deletion(_) => {
+ let mut deletion_rows = Vec::new();
+ let mut has_addition = false;
+
+ while index < hunk.lines.len() {
+ match &hunk.lines[index] {
+ PatchLine::Addition(_) => {
+ has_addition = true;
+ index += 1;
+ }
+ PatchLine::Deletion(_) => {
+ deletion_rows.push(old_row);
+ old_row = old_row.saturating_add(1);
+ index += 1;
+ }
+ _ => break,
+ }
+ }
+
+ if deletion_rows.is_empty() {
+ if has_addition {
+ if let Some(row) = previous_context_row {
+ insert_anchor_row(
+ &mut expected_anchor_lines,
+ &mut relevant_lines,
+ &hunk.filename,
+ row,
+ );
+ }
+ if matches!(hunk.lines.get(index), Some(PatchLine::Context(_))) {
+ insert_anchor_row(
+ &mut expected_anchor_lines,
+ &mut relevant_lines,
+ &hunk.filename,
+ old_row,
+ );
+ }
+ }
+ } else {
+ for row in deletion_rows {
+ insert_anchor_row(
+ &mut expected_anchor_lines,
+ &mut relevant_lines,
+ &hunk.filename,
+ row,
+ );
+ }
+ }
+
+ previous_context_row = None;
+ }
+ PatchLine::Garbage(_) => {
+ index += 1;
+ }
+ }
+ }
+ }
+
+ (expected_files, expected_anchor_lines, relevant_lines)
+}
+
+fn retrieved_context(context: &[Excerpt]) -> (BTreeSet<String>, BTreeMap<String, BTreeSet<u32>>) {
+ let mut retrieved_files = BTreeSet::new();
+ let mut retrieved_lines = BTreeMap::new();
+
+ for excerpt in context {
+ retrieved_files.insert(excerpt.path.clone());
+ let rows = retrieved_lines
+ .entry(excerpt.path.clone())
+ .or_insert_with(BTreeSet::new);
+ rows.extend(excerpt.row_range.clone());
+ }
+
+ (retrieved_files, retrieved_lines)
+}
+
+fn insert_anchor_row(
+ anchor_lines_by_file: &mut BTreeMap<String, BTreeSet<u32>>,
+ relevant_lines_by_file: &mut BTreeMap<String, BTreeSet<u32>>,
+ path: &str,
+ row: u32,
+) {
+ insert_row(anchor_lines_by_file, path, row);
+
+ let start = row.saturating_sub(LINE_RELEVANCE_WINDOW);
+ let end = row.saturating_add(LINE_RELEVANCE_WINDOW);
+ for relevant_row in start..=end {
+ insert_row(relevant_lines_by_file, path, relevant_row);
+ }
+}
+
+fn insert_row(lines_by_file: &mut BTreeMap<String, BTreeSet<u32>>, path: &str, row: u32) {
+ lines_by_file
+ .entry(path.to_string())
+ .or_insert_with(BTreeSet::new)
+ .insert(row);
+}
+
+fn classify_retrieved_rows(
+ relevant: &BTreeMap<String, BTreeSet<u32>>,
+ retrieved: &BTreeMap<String, BTreeSet<u32>>,
+) -> (usize, usize) {
+ let mut true_positives = 0;
+ let mut false_positives = 0;
+
+ for (path, rows) in retrieved {
+ for row in rows {
+ if relevant
+ .get(path)
+ .is_some_and(|relevant_rows| relevant_rows.contains(row))
+ {
+ true_positives += 1;
+ } else {
+ false_positives += 1;
+ }
+ }
+ }
+
+ (true_positives, false_positives)
+}
+
+fn count_missing_rows(
+ expected: &BTreeMap<String, BTreeSet<u32>>,
+ retrieved: &BTreeMap<String, BTreeSet<u32>>,
+) -> (usize, usize) {
+ let mut false_negatives = 0;
+ let mut expected_count = 0;
+
+ for (path, rows) in expected {
+ for row in rows {
+ expected_count += 1;
+ if !retrieved
+ .get(path)
+ .is_some_and(|retrieved_rows| retrieved_rows.contains(row))
+ {
+ false_negatives += 1;
+ }
+ }
+ }
+
+ (false_negatives, expected_count)
+}
+
+fn classify_values<T: Ord>(
+ expected: &BTreeSet<T>,
+ retrieved: &BTreeSet<T>,
+) -> (usize, usize, usize) {
+ let true_positives = expected.intersection(retrieved).count();
+ let false_positives = retrieved.difference(expected).count();
+ let false_negatives = expected.difference(retrieved).count();
+ (true_positives, false_positives, false_negatives)
+}
+
+fn precision(true_positives: usize, false_positives: usize, false_negatives: usize) -> f64 {
+ if true_positives + false_positives + false_negatives == 0 {
+ return 1.0;
+ }
+
+ let denominator = true_positives + false_positives;
+ if denominator == 0 {
+ 1.0
+ } else {
+ true_positives as f64 / denominator as f64
+ }
+}
+
+fn recall(true_positives: usize, false_positives: usize, false_negatives: usize) -> f64 {
+ if true_positives + false_positives + false_negatives == 0 {
+ return 1.0;
+ }
+
+ let denominator = true_positives + false_negatives;
+ if denominator == 0 {
+ 1.0
+ } else {
+ true_positives as f64 / denominator as f64
+ }
+}
+
+fn line_recall(expected_anchor_line_count: usize, false_negatives: usize) -> f64 {
+ if expected_anchor_line_count == 0 {
+ 1.0
+ } else {
+ (expected_anchor_line_count - false_negatives) as f64 / expected_anchor_line_count as f64
+ }
+}
+
+fn f1(true_positives: usize, false_positives: usize, false_negatives: usize) -> f64 {
+ let precision = precision(true_positives, false_positives, false_negatives);
+ let recall = recall(true_positives, false_positives, false_negatives);
+
+ f1_from_precision_and_recall(precision, recall)
+}
+
+fn f1_from_precision_and_recall(precision: f64, recall: f64) -> f64 {
+ if precision + recall == 0.0 {
+ 0.0
+ } else {
+ 2.0 * precision * recall / (precision + recall)
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use indoc::indoc;
+
+ fn excerpt(path: &str, row_range: Range<u32>) -> Excerpt {
+ Excerpt {
+ path: path.to_string(),
+ row_range,
+ content: String::new(),
+ }
+ }
+
+ #[test]
+ fn deletion_counts_deleted_old_line_as_true_positive() {
+ let patch = indoc! {"
+ --- a/src/main.rs
+ +++ b/src/main.rs
+ @@ -2,1 +2,0 @@
+ -let value = 1;
+ "};
+
+ let score = editable_context_coverage(patch, &[excerpt("src/main.rs", 1..2)]);
+
+ assert_eq!(score, EditableContextCoverage::new(1, 0, 0, 1, 0, 0));
+ }
+
+ #[test]
+ fn retrieved_lines_inside_relevance_window_are_true_positives() {
+ let patch = indoc! {"
+ --- a/src/main.rs
+ +++ b/src/main.rs
+ @@ -4,1 +4,0 @@
+ -let value = 4;
+ "};
+
+ let score = editable_context_coverage(
+ patch,
+ &[excerpt("src/main.rs", 0..1), excerpt("src/main.rs", 3..4)],
+ );
+
+ assert_eq!(score, EditableContextCoverage::new(2, 0, 0, 1, 0, 0));
+ }
+
+ #[test]
+ fn replacement_counts_deleted_old_line_without_addition_anchor() {
+ let patch = indoc! {"
+ --- a/src/main.rs
+ +++ b/src/main.rs
+ @@ -1,3 +1,3 @@
+ fn main() {
+ - let value = 1;
+ + let value = 2;
+ }
+ "};
+
+ let score = editable_context_coverage(patch, &[excerpt("src/main.rs", 1..2)]);
+
+ assert_eq!(score, EditableContextCoverage::new(1, 0, 0, 1, 0, 0));
+ }
+
+ #[test]
+ fn pure_insertion_counts_previous_and_next_old_lines_as_expected_context() {
+ let patch = indoc! {"
+ --- a/src/main.rs
+ +++ b/src/main.rs
+ @@ -1,2 +1,3 @@
+ line 1
+ +inserted
+ line 2
+ "};
+
+ let score = editable_context_coverage(patch, &[excerpt("src/main.rs", 0..1)]);
+
+ assert_eq!(score, EditableContextCoverage::new(1, 0, 1, 1, 0, 0));
+ }
+
+ #[test]
+ fn pure_insertion_at_file_boundary_uses_available_neighboring_context() {
+ let patch = indoc! {"
+ --- a/src/main.rs
+ +++ b/src/main.rs
+ @@ -1,1 +1,2 @@
+ +inserted
+ line 1
+ "};
+
+ let score = editable_context_coverage(patch, &[excerpt("src/main.rs", 0..1)]);
+
+ assert_eq!(score, EditableContextCoverage::new(1, 0, 0, 1, 0, 0));
+ }
+
+ #[test]
+ fn counts_false_negatives_and_file_false_positives() {
+ let patch = indoc! {"
+ --- a/src/main.rs
+ +++ b/src/main.rs
+ @@ -1,3 +1,3 @@
+ -let first = 1;
+ +let first = 2;
+ let middle = 3;
+ -let last = 4;
+ +let last = 5;
+ "};
+
+ let score = editable_context_coverage(
+ patch,
+ &[excerpt("src/main.rs", 0..1), excerpt("src/lib.rs", 0..1)],
+ );
+
+ assert_eq!(score, EditableContextCoverage::new(1, 1, 1, 1, 1, 0));
+ }
+
+ #[test]
+ fn overlapping_excerpts_are_counted_once() {
+ let patch = indoc! {"
+ --- a/src/main.rs
+ +++ b/src/main.rs
+ @@ -2,1 +2,0 @@
+ -let value = 1;
+ "};
+
+ let score = editable_context_coverage(
+ patch,
+ &[excerpt("src/main.rs", 0..2), excerpt("src/main.rs", 1..3)],
+ );
+
+ assert_eq!(score, EditableContextCoverage::new(3, 0, 0, 1, 0, 0));
+ }
+
+ #[test]
+ fn nearby_lines_do_not_satisfy_line_recall_without_exact_anchor_lines() {
+ let patch = indoc! {"
+ --- a/src/main.rs
+ +++ b/src/main.rs
+ @@ -1,2 +1,3 @@
+ line 1
+ +inserted
+ line 2
+ "};
+
+ let score = editable_context_coverage(patch, &[excerpt("src/main.rs", 2..3)]);
+
+ assert_eq!(score.lines_tp, 1);
+ assert_eq!(score.lines_fp, 0);
+ assert_eq!(score.lines_fn, 2);
+ assert_eq!(score.lines_precision, 1.0);
+ assert_eq!(score.lines_recall, 0.0);
+ assert_eq!(score.lines_f1, 0.0);
+ }
+
+ #[test]
+ fn retrieved_lines_outside_relevance_window_are_false_positives() {
+ let patch = indoc! {"
+ --- a/src/main.rs
+ +++ b/src/main.rs
+ @@ -1,1 +1,0 @@
+ -line 1
+ "};
+
+ let score = editable_context_coverage(patch, &[excerpt("src/main.rs", 21..22)]);
+
+ assert_eq!(score, EditableContextCoverage::new(0, 1, 1, 1, 0, 0));
+ }
+
+ #[test]
+ fn empty_patch_with_no_context_has_perfect_f1() {
+ let score = editable_context_coverage(
+ indoc! {"
+ "},
+ &[],
+ );
+
+ assert_eq!(score, EditableContextCoverage::new(0, 0, 0, 0, 0, 0));
+ }
+}
diff --git a/crates/edit_prediction_metrics/src/main.rs b/crates/edit_prediction_metrics/src/main.rs
index 0e557c35e7f..c7c55ceff60 100644
--- a/crates/edit_prediction_metrics/src/main.rs
+++ b/crates/edit_prediction_metrics/src/main.rs
@@ -5,10 +5,10 @@ use std::path::Path;
use std::process;
use edit_prediction_metrics::{
- ClassificationMetrics, DeltaChrFMetrics, KeptRateResult, TokenAnnotation,
- annotate_kept_rate_tokens, braces_disbalance, compute_kept_rate, count_patch_token_changes,
- delta_chr_f, exact_lines_match, extract_changed_lines_from_diff,
- has_isolated_whitespace_changes, is_editable_region_correct,
+ ClassificationMetrics, DeltaChrFMetrics, EditableContextCoverage, Excerpt, KeptRateResult,
+ TokenAnnotation, annotate_kept_rate_tokens, braces_disbalance, compute_kept_rate,
+ count_patch_token_changes, delta_chr_f, editable_context_coverage, exact_lines_match,
+ extract_changed_lines_from_diff, has_isolated_whitespace_changes, is_editable_region_correct,
};
use serde::Deserialize;
@@ -41,10 +41,18 @@ fn run() -> Result<(), String> {
let actual_patch = fs::read_to_string(&actual_patch_path)
.map_err(|err| format!("failed to read {}: {err}", actual_patch_path.display()))?;
- let expected = apply_patch_to_excerpt(&base, &expected_patch, 0)?;
- let actual = apply_patch_to_excerpt(&base, &actual_patch, 0)?;
-
- EvaluationReport::new(base, expected_patch, actual_patch, expected, actual)
+ let expected = apply_patch_to_excerpt(&base, &expected_patch, 0, None)?;
+ let actual = apply_patch_to_excerpt(&base, &actual_patch, 0, None)?;
+ let context = [];
+
+ EvaluationReport::new(
+ base,
+ expected_patch,
+ actual_patch,
+ expected,
+ actual,
+ &context,
+ )
}
CliInput::Json {
json_path,
@@ -55,26 +63,7 @@ fn run() -> Result<(), String> {
let example: JsonExample = serde_json::from_str(&json)
.map_err(|err| format!("failed to parse {}: {err}", json_path.display()))?;
- let base = example.prompt_inputs.cursor_excerpt;
- let excerpt_start_row = example.prompt_inputs.excerpt_start_row;
- let expected_patch = example
- .expected_patches
- .into_iter()
- .next()
- .ok_or_else(|| "JSON input is missing expected_patches[0]".to_string())?;
- let actual_patch = example
- .predictions
- .into_iter()
- .nth(prediction_index)
- .ok_or_else(|| {
- format!("JSON input does not contain predictions[{prediction_index}]")
- })?
- .actual_patch;
-
- let expected = apply_patch_to_excerpt(&base, &expected_patch, excerpt_start_row)?;
- let actual = apply_patch_to_excerpt(&base, &actual_patch, excerpt_start_row)?;
-
- EvaluationReport::new(base, expected_patch, actual_patch, expected, actual)
+ report_from_json_example(example, prediction_index)?
}
};
@@ -82,6 +71,77 @@ fn run() -> Result<(), String> {
Ok(())
}
+fn get_context_excerpts(example: &JsonExample) -> Vec<Excerpt> {
+ let mut context = vec![get_cursor_excerpt(example)];
+
+ if let Some(related) = &example.prompt_inputs.related_files {
+ context.extend(related.iter().flat_map(|file| {
+ file.excerpts.iter().map(|excerpt| Excerpt {
+ path: file.path.clone(),
+ row_range: excerpt.row_range.clone(),
+ content: excerpt.text.clone(),
+ })
+ }));
+ }
+
+ context
+}
+
+fn get_cursor_excerpt(example: &JsonExample) -> Excerpt {
+ let content = example.prompt_inputs.cursor_excerpt.clone();
+ let start_row = example.prompt_inputs.excerpt_start_row;
+ let rows = content.lines().count() as u32;
+ let row_range = start_row..start_row + rows;
+ Excerpt {
+ path: example.cursor_path.clone(),
+ row_range,
+ content,
+ }
+}
+
+fn report_from_json_example(
+ example: JsonExample,
+ prediction_index: usize,
+) -> Result<EvaluationReport, String> {
+ let context = get_context_excerpts(&example);
+ let excerpt_start_row = example.prompt_inputs.excerpt_start_row;
+ let cursor_path = example.cursor_path;
+ let base = example.prompt_inputs.cursor_excerpt;
+ let expected_patch = example
+ .expected_patches
+ .into_iter()
+ .next()
+ .ok_or_else(|| "JSON input is missing expected_patches[0]".to_string())?;
+ let actual_patch = if example.predictions.is_empty() {
+ String::new()
+ } else {
+ example
+ .predictions
+ .into_iter()
+ .nth(prediction_index)
+ .ok_or_else(|| format!("JSON input does not contain predictions[{prediction_index}]"))?
+ .actual_patch
+ };
+
+ let expected = apply_patch_to_excerpt(
+ &base,
+ &expected_patch,
+ excerpt_start_row,
+ Some(&cursor_path),
+ )?;
+ let actual =
+ apply_patch_to_excerpt(&base, &actual_patch, excerpt_start_row, Some(&cursor_path))?;
+
+ Ok(EvaluationReport::new(
+ base,
+ expected_patch,
+ actual_patch,
+ expected,
+ actual,
+ &context,
+ ))
+}
+
fn print_usage() {
eprintln!(
"Usage:\n edit_prediction_metrics --base <base.txt> --expected-patch <expected.diff> --actual-patch <actual.diff>\n edit_prediction_metrics --json <example.json> [--prediction-index <n>]"
@@ -199,6 +259,7 @@ struct EvaluationReport {
editable_region_correct: bool,
expected_braces_disbalance: usize,
actual_braces_disbalance: usize,
+ editable_context_coverage: EditableContextCoverage,
}
impl EvaluationReport {
@@ -208,6 +269,7 @@ impl EvaluationReport {
actual_patch: String,
expected: String,
actual: String,
+ context: &[Excerpt],
) -> Self {
let kept_rate = compute_kept_rate(&base, &actual, &expected);
let exact_lines = exact_lines_match(&expected_patch, &actual_patch);
@@ -223,6 +285,7 @@ impl EvaluationReport {
let editable_region_correct = is_editable_region_correct(&actual_patch);
let expected_braces_disbalance = braces_disbalance(&expected);
let actual_braces_disbalance = braces_disbalance(&actual);
+ let editable_context_coverage = editable_context_coverage(&expected_patch, context);
Self {
base,
@@ -238,6 +301,7 @@ impl EvaluationReport {
editable_region_correct,
expected_braces_disbalance,
actual_braces_disbalance,
+ editable_context_coverage,
}
}
}
@@ -319,6 +383,27 @@ fn print_report(report: &EvaluationReport) {
println!();
print_kept_rate_explanation(&report.base, &report.actual, &report.expected);
+
+ println!("Jumps metrics");
+ println!("-------------");
+ println!(
+ "Editable context lines: P={}%, R={}%, F1={}% (tp: {}, fp: {}, fn: {})",
+ (report.editable_context_coverage.lines_precision * 100.0).round(),
+ (report.editable_context_coverage.lines_recall * 100.0).round(),
+ (report.editable_context_coverage.lines_f1 * 100.0).round(),
+ report.editable_context_coverage.lines_tp,
+ report.editable_context_coverage.lines_fp,
+ report.editable_context_coverage.lines_fn
+ );
+ println!(
+ "Editable context files: P={}%, R={}%, F1={}% (tp: {}, fp: {}, fn: {})",
+ (report.editable_context_coverage.files_precision * 100.0).round(),
+ (report.editable_context_coverage.files_recall * 100.0).round(),
+ (report.editable_context_coverage.files_f1 * 100.0).round(),
+ report.editable_context_coverage.files_tp,
+ report.editable_context_coverage.files_fp,
+ report.editable_context_coverage.files_fn
+ );
}
fn print_kept_rate_explanation(base: &str, actual: &str, expected: &str) {
@@ -373,7 +458,9 @@ fn visualize_whitespace(token: &str) -> String {
#[derive(Debug, Deserialize)]
struct JsonExample {
prompt_inputs: PromptInputs,
+ cursor_path: String,
expected_patches: Vec<String>,
+ #[serde(default)]
predictions: Vec<Prediction>,
}
@@ -381,6 +468,20 @@ struct JsonExample {
struct PromptInputs {
cursor_excerpt: String,
excerpt_start_row: u32,
+ pub related_files: Option<Vec<RelatedFile>>,
+}
+
+#[derive(Clone, Debug, PartialEq, Hash, Deserialize)]
+pub struct RelatedFile {
+ pub path: String,
+ pub max_row: u32,
+ pub excerpts: Vec<RelatedExcerpt>,
+}
+
+#[derive(Clone, Debug, PartialEq, Hash, Deserialize)]
+pub struct RelatedExcerpt {
+ pub row_range: std::ops::Range<u32>,
+ pub text: String,
}
#[derive(Debug, Deserialize)]
@@ -391,6 +492,7 @@ struct Prediction {
#[derive(Debug, Clone)]
struct ParsedHunk {
old_start: u32,
+ filename: Option<String>,
lines: Vec<HunkLine>,
}
@@ -405,8 +507,20 @@ fn apply_patch_to_excerpt(
base: &str,
patch: &str,
excerpt_start_row: u32,
+ target_path: Option<&str>,
) -> Result<String, String> {
let hunks = parse_diff_hunks(patch);
+ let hunks = if let Some(target_path) = target_path {
+ hunks
+ .into_iter()
+ .filter(|hunk| match hunk.filename.as_deref() {
+ Some(filename) => filename == target_path,
+ None => true,
+ })
+ .collect::<Vec<_>>()
+ } else {
+ hunks
+ };
let result = try_apply_hunks(base, &hunks, excerpt_start_row);
@@ -598,6 +712,7 @@ fn filter_hunk_to_excerpt(
Some(ParsedHunk {
old_start: filtered_old_start.unwrap_or(excerpt_start_row),
+ filename: hunk.filename.clone(),
lines: filtered_lines,
})
}
@@ -605,8 +720,14 @@ fn filter_hunk_to_excerpt(
fn parse_diff_hunks(diff: &str) -> Vec<ParsedHunk> {
let mut hunks = Vec::new();
let mut current_hunk: Option<ParsedHunk> = None;
+ let mut current_filename = None;
for line in diff.lines() {
+ if let Some(filename) = parse_diff_filename(line) {
+ current_filename = Some(filename);
+ continue;
+ }
+
if let Some((old_start, old_count, _new_start, _new_count)) = parse_hunk_header(line) {
if let Some(hunk) = current_hunk.take() {
hunks.push(hunk);
@@ -614,6 +735,7 @@ fn parse_diff_hunks(diff: &str) -> Vec<ParsedHunk> {
let _ = old_count;
current_hunk = Some(ParsedHunk {
old_start,
+ filename: current_filename.clone(),
lines: Vec::new(),
});
continue;
@@ -664,6 +786,27 @@ fn parse_hunk_range(part: &str) -> Option<(u32, u32)> {
}
}
+fn parse_diff_filename(line: &str) -> Option<String> {
+ let path = line
+ .strip_prefix("--- ")
+ .or_else(|| line.strip_prefix("+++ "))?;
+ normalize_diff_path(path)
+}
+
+fn normalize_diff_path(path: &str) -> Option<String> {
+ let path = path.trim();
+ let path = path
+ .strip_prefix("a/")
+ .or_else(|| path.strip_prefix("b/"))
+ .unwrap_or(path);
+
+ if path == "/dev/null" {
+ None
+ } else {
+ Some(path.to_string())
+ }
+}
+
#[cfg(test)]
mod tests {
use super::*;
@@ -673,7 +816,7 @@ mod tests {
let base = "fn main() {\n println!(\"hello\");\n}\n";
let patch = "@@ -1,3 +1,3 @@\n fn main() {\n- println!(\"hello\");\n+ println!(\"world\");\n }\n";
- let actual = apply_patch_to_excerpt(base, patch, 0).unwrap();
+ let actual = apply_patch_to_excerpt(base, patch, 0, None).unwrap();
assert_eq!(actual, "fn main() {\n println!(\"world\");\n}\n");
}
@@ -682,7 +825,7 @@ mod tests {
let base = "b\nc\nd\n";
let patch = "@@ -2,2 +2,2 @@\n-b\n-c\n+x\n+y\n";
- let actual = apply_patch_to_excerpt(base, patch, 1).unwrap();
+ let actual = apply_patch_to_excerpt(base, patch, 1, None).unwrap();
assert_eq!(actual, "x\ny\nd\n");
}
@@ -693,7 +836,7 @@ mod tests {
// even though the excerpt starts at file row 100.
let patch = "@@ -2,2 +2,2 @@\n-b\n-c\n+x\n+y\n";
- let actual = apply_patch_to_excerpt(base, patch, 100).unwrap();
+ let actual = apply_patch_to_excerpt(base, patch, 100, None).unwrap();
assert_eq!(actual, "a\nx\ny\nd\n");
}
@@ -704,7 +847,87 @@ mod tests {
// hunk targets line 6 (1-based) = row 5 (0-based) = first line.
let patch = "@@ -6,2 +6,2 @@\n-a\n-b\n+x\n+y\n";
- let actual = apply_patch_to_excerpt(base, patch, 5).unwrap();
+ let actual = apply_patch_to_excerpt(base, patch, 5, None).unwrap();
assert_eq!(actual, "x\ny\nc\n");
}
+
+ #[test]
+ fn json_patch_application_ignores_unrelated_file_hunks() {
+ let base = "first\nsecond\nthird\n";
+ let patch = "--- a/src/other.rs\n+++ b/src/other.rs\n@@ -2,1 +2,1 @@\n-second\n+changed\n";
+
+ let actual = apply_patch_to_excerpt(base, patch, 0, Some("src/main.rs")).unwrap();
+ assert_eq!(actual, base);
+ }
+
+ #[test]
+ fn json_patch_application_applies_matching_file_hunks() {
+ let base = "first\nsecond\nthird\n";
+ let patch = "--- a/src/main.rs\n+++ b/src/main.rs\n@@ -2,1 +2,1 @@\n-second\n+changed\n";
+
+ let actual = apply_patch_to_excerpt(base, patch, 0, Some("src/main.rs")).unwrap();
+ assert_eq!(actual, "first\nchanged\nthird\n");
+ }
+
+ #[test]
+ fn json_patch_application_applies_headerless_hunks() {
+ let base = "first\nsecond\nthird\n";
+ let patch = "@@ -2,1 +2,1 @@\n-second\n+changed\n";
+
+ let actual = apply_patch_to_excerpt(base, patch, 0, Some("src/main.rs")).unwrap();
+ assert_eq!(actual, "first\nchanged\nthird\n");
+ }
+
+ fn json_example(predictions: Option<&str>) -> String {
+ let predictions = predictions
+ .map(|predictions| {
+ format!(
+ r#",
+ "predictions": {predictions}"#
+ )
+ })
+ .unwrap_or_default();
+
+ format!(
+ r#"{{
+ "prompt_inputs": {{
+ "cursor_excerpt": "first\nsecond\nthird\n",
+ "excerpt_start_row": 0
+ }},
+ "cursor_path": "src/main.rs",
+ "expected_patches": [
+ "--- a/src/main.rs\n+++ b/src/main.rs\n@@ -2,1 +2,1 @@\n-second\n+changed\n"
+ ]{predictions}
+}}"#
+ )
+ }
+
+ fn report_from_json(predictions: Option<&str>) -> EvaluationReport {
+ let example = serde_json::from_str(&json_example(predictions)).unwrap();
+ report_from_json_example(example, 0).unwrap()
+ }
+
+ #[test]
+ fn json_report_with_missing_predictions_uses_expected_patch_for_context_coverage() {
+ let report = report_from_json(None);
+
+ assert_eq!(report.actual, "first\nsecond\nthird\n");
+ assert_eq!(report.actual_changed_lines, 0);
+ assert_eq!(
+ report.editable_context_coverage,
+ EditableContextCoverage::new(3, 0, 0, 1, 0, 0)
+ );
+ }
+
+ #[test]
+ fn json_report_with_empty_predictions_uses_expected_patch_for_context_coverage() {
+ let report = report_from_json(Some("[]"));
+
+ assert_eq!(report.actual, "first\nsecond\nthird\n");
+ assert_eq!(report.actual_changed_lines, 0);
+ assert_eq!(
+ report.editable_context_coverage,
+ EditableContextCoverage::new(3, 0, 0, 1, 0, 0)
+ );
+ }
}
diff --git a/crates/edit_prediction_metrics/src/patch.rs b/crates/edit_prediction_metrics/src/patch.rs
new file mode 100644
index 00000000000..611544cec58
--- /dev/null
+++ b/crates/edit_prediction_metrics/src/patch.rs
@@ -0,0 +1,127 @@
+#[derive(Debug, Default, Clone)]
+pub struct Patch {
+ pub hunks: Vec<Hunk>,
+}
+
+impl Patch {
+ pub fn parse_unified_diff(unified_diff: &str) -> Patch {
+ let mut current_file = String::new();
+ let mut is_filename_inherited = false;
+ let mut hunk = Hunk::default();
+ let mut patch = Patch::default();
+ let mut in_header = true;
+
+ for line in unified_diff.lines() {
+ if line.starts_with("--- ") || line.starts_with("+++") || line.starts_with("@@") {
+ in_header = false;
+ }
+
+ if in_header {
+ continue;
+ }
+
+ if line.starts_with("@@") {
+ if !hunk.lines.is_empty() {
+ patch.hunks.push(hunk);
+ }
+ hunk = Hunk::from_header(line, &current_file, is_filename_inherited);
+ is_filename_inherited = true;
+ } else if let Some(path) = line.strip_prefix("--- ") {
+ is_filename_inherited = false;
+ let path = path.trim().strip_prefix("a/").unwrap_or(path);
+ if path != "/dev/null" {
+ current_file = path.into();
+ }
+ } else if let Some(path) = line.strip_prefix("+++ ") {
+ is_filename_inherited = false;
+ let path = path.trim().strip_prefix("b/").unwrap_or(path);
+ if path != "/dev/null" {
+ current_file = path.into();
+ }
+ } else if let Some(line) = line.strip_prefix('+') {
+ hunk.lines.push(PatchLine::Addition(line.to_string()));
+ } else if let Some(line) = line.strip_prefix('-') {
+ hunk.lines.push(PatchLine::Deletion(line.to_string()));
+ } else if let Some(line) = line.strip_prefix(' ') {
+ hunk.lines.push(PatchLine::Context(line.to_string()));
+ } else {
+ hunk.lines.push(PatchLine::Garbage(line.to_string()));
+ }
+ }
+
+ if !hunk.lines.is_empty() {
+ patch.hunks.push(hunk);
+ }
+
+ patch
+ }
+}
+
+#[derive(Debug, Default, Clone)]
+pub struct Hunk {
+ pub old_start: isize,
+ pub new_start: isize,
+ pub lines: Vec<PatchLine>,
+ pub filename: String,
+}
+
+impl Hunk {
+ pub fn from_header(header: &str, filename: &str, _is_filename_inherited: bool) -> Self {
+ let (old_start, _, new_start, _, _) = Self::parse_hunk_header(header);
+ Self {
+ old_start,
+ new_start,
+ lines: Vec::new(),
+ filename: filename.to_string(),
+ }
+ }
+
+ fn parse_hunk_header(line: &str) -> (isize, isize, isize, isize, String) {
+ let header_part = line.trim_start_matches("@@").trim();
+ let parts: Vec<&str> = header_part.split_whitespace().collect();
+
+ if parts.len() < 2 {
+ return (0, 0, 0, 0, String::new());
+ }
+
+ let old_part = parts[0].trim_start_matches('-');
+ let new_part = parts[1].trim_start_matches('+');
+
+ let (old_start, old_count) = Hunk::parse_hunk_header_range(old_part);
+ let (new_start, new_count) = Hunk::parse_hunk_header_range(new_part);
+
+ let comment = if parts.len() > 2 {
+ parts[2..]
+ .join(" ")
+ .trim_start_matches("@@")
+ .trim()
+ .to_string()
+ } else {
+ String::new()
+ };
+
+ (
+ old_start as isize,
+ old_count as isize,
+ new_start as isize,
+ new_count as isize,
+ comment,
+ )
+ }
+
+ fn parse_hunk_header_range(part: &str) -> (usize, usize) {
+ if let Some((start, count)) = part.split_once(',') {
+ (start.parse().unwrap_or(0), count.parse().unwrap_or(0))
+ } else {
+ (part.parse().unwrap_or(0), 1)
+ }
+ }
+}
+
+#[derive(Clone, Debug, Eq, PartialEq)]
+pub enum PatchLine {
+ Context(String),
+ Addition(String),
+ Deletion(String),
+ Garbage(String),
+}
diff --git a/crates/edit_prediction_metrics/src/patch_metrics.rs b/crates/edit_prediction_metrics/src/patch_metrics.rs
index 85470da91c5..c913f9a7e4b 100644
--- a/crates/edit_prediction_metrics/src/patch_metrics.rs
+++ b/crates/edit_prediction_metrics/src/patch_metrics.rs
@@ -1,6 +1,9 @@
use std::collections::HashMap;
-use crate::tokenize::tokenize;
+use crate::{
+ patch::{Patch, PatchLine},
+ tokenize::tokenize,
+};
use serde::Serialize;
use similar::{DiffTag, TextDiff};
@@ -716,129 +719,6 @@ pub fn reconstruct_texts_from_diff(patch_str: &str) -> (String, String) {
(old_lines.join("\n"), new_lines.join("\n"))
}
-#[derive(Debug, Default, Clone)]
-struct Patch {
- hunks: Vec<Hunk>,
-}
-
-impl Patch {
- fn parse_unified_diff(unified_diff: &str) -> Patch {
- let mut current_file = String::new();
- let mut is_filename_inherited = false;
- let mut hunk = Hunk::default();
- let mut patch = Patch::default();
- let mut in_header = true;
-
- for line in unified_diff.lines() {
- if line.starts_with("--- ") || line.starts_with("+++") || line.starts_with("@@") {
- in_header = false;
- }
-
- if in_header {
- continue;
- }
-
- if line.starts_with("@@") {
- if !hunk.lines.is_empty() {
- patch.hunks.push(hunk);
- }
- hunk = Hunk::from_header(line, &current_file, is_filename_inherited);
- is_filename_inherited = true;
- } else if let Some(path) = line.strip_prefix("--- ") {
- is_filename_inherited = false;
- let path = path.trim().strip_prefix("a/").unwrap_or(path);
- if path != "/dev/null" {
- current_file = path.into();
- }
- } else if let Some(path) = line.strip_prefix("+++ ") {
- is_filename_inherited = false;
- let path = path.trim().strip_prefix("b/").unwrap_or(path);
- if path != "/dev/null" {
- current_file = path.into();
- }
- } else if let Some(line) = line.strip_prefix('+') {
- hunk.lines.push(PatchLine::Addition(line.to_string()));
- } else if let Some(line) = line.strip_prefix('-') {
- hunk.lines.push(PatchLine::Deletion(line.to_string()));
- } else if let Some(line) = line.strip_prefix(' ') {
- hunk.lines.push(PatchLine::Context(line.to_string()));
- } else {
- hunk.lines.push(PatchLine::Garbage(line.to_string()));
- }
- }
-
- if !hunk.lines.is_empty() {
- patch.hunks.push(hunk);
- }
-
- patch
- }
-}
-
-#[derive(Debug, Default, Clone)]
-struct Hunk {
- new_start: isize,
- lines: Vec<PatchLine>,
-}
-
-impl Hunk {
- fn from_header(header: &str, _filename: &str, _is_filename_inherited: bool) -> Self {
- let (_, _, new_start, _, _) = Self::parse_hunk_header(header);
- Self {
- new_start,
- lines: Vec::new(),
- }
- }
-
- fn parse_hunk_header(line: &str) -> (isize, isize, isize, isize, String) {
- let header_part = line.trim_start_matches("@@").trim();
- let parts: Vec<&str> = header_part.split_whitespace().collect();
-
- if parts.len() < 2 {
- return (0, 0, 0, 0, String::new());
- }
-
- let old_part = parts[0].trim_start_matches('-');
- let new_part = parts[1].trim_start_matches('+');
-
- let (old_start, old_count) = Hunk::parse_hunk_header_range(old_part);
- let (new_start, new_count) = Hunk::parse_hunk_header_range(new_part);
-
- let comment = if parts.len() > 2 {
- parts[2..]
- .join(" ")
- .trim_start_matches("@@")
- .trim()
- .to_string()
- } else {
- String::new()
- };
-
- (
- old_start as isize,
- old_count as isize,
- new_start as isize,
- new_count as isize,
- comment,
- )
- }
-
- fn parse_hunk_header_range(part: &str) -> (usize, usize) {
- if let Some((start, count)) = part.split_once(',') {
- (start.parse().unwrap_or(0), count.parse().unwrap_or(0))
- } else {
- (part.parse().unwrap_or(0), 1)
- }
- }
-}
-
-#[derive(Clone, Debug, Eq, PartialEq)]
-enum PatchLine {
- Context(String),
- Addition(String),
- Deletion(String),
- Garbage(String),
-}
#[cfg(test)]
mod test_optimization {
diff --git a/crates/edit_prediction_metrics/src/prediction_score.rs b/crates/edit_prediction_metrics/src/prediction_score.rs
index 942ce3c9d1a..684a93ba9de 100644
--- a/crates/edit_prediction_metrics/src/prediction_score.rs
+++ b/crates/edit_prediction_metrics/src/prediction_score.rs
@@ -5,12 +5,15 @@ use std::path::Path;
use std::sync::Arc;
use zeta_prompt::udiff::{apply_diff_to_string, apply_diff_to_string_with_hunk_offset};
-use crate::patch_metrics::{
- ClassificationMetrics, DeltaChrFMetrics, braces_disbalance, count_patch_token_changes,
- delta_chr_f, delta_chr_f_beta, exact_lines_match, has_isolated_whitespace_changes,
- is_editable_region_correct,
-};
use crate::reversal::compute_prediction_reversal_ratio_from_history;
+use crate::{
+ jumps::{EditableContextCoverage, Excerpt, editable_context_coverage},
+ patch_metrics::{
+ ClassificationMetrics, DeltaChrFMetrics, braces_disbalance, count_patch_token_changes,
+ delta_chr_f, delta_chr_f_beta, exact_lines_match, has_isolated_whitespace_changes,
+ is_editable_region_correct,
+ },
+};
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct PredictionScore {
@@ -61,6 +64,8 @@ pub struct PredictionScore {
pub cumulative_logprob: Option<f64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub avg_logprob: Option<f64>,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub editable_context_coverage: Option<EditableContextCoverage>,
}
impl PredictionScore {
@@ -91,6 +96,7 @@ impl PredictionScore {
discarded_chars: None,
cumulative_logprob: None,
avg_logprob: None,
+ editable_context_coverage: None,
}
}
@@ -193,11 +199,26 @@ pub struct PredictionScoringInput<'a> {
pub reversal_context: Option<PredictionReversalContext<'a>>,
pub cumulative_logprob: Option<f64>,
pub avg_logprob: Option<f64>,
+ pub context: Option<&'a [Excerpt]>,
}
pub fn score_prediction(input: PredictionScoringInput<'_>) -> PredictionScore {
+ let editable_context_coverage = input.context.and_then(|context| {
+ input
+ .expected_patches
+ .iter()
+ .map(|expected| editable_context_coverage(&expected.patch, context))
+ .max_by(|left, right| {
+ left.lines_f1
+ .total_cmp(&right.lines_f1)
+ .then_with(|| left.files_f1.total_cmp(&right.files_f1))
+ })
+ });
+
let Some(actual_patch) = input.actual_patch else {
- return PredictionScore::zero();
+ let mut score = PredictionScore::zero();
+ score.editable_context_coverage = editable_context_coverage;
+ return score;
};
let token_changes = count_patch_token_changes(actual_patch);
@@ -208,6 +229,7 @@ pub fn score_prediction(input: PredictionScoringInput<'_>) -> PredictionScore {
let mut score = PredictionScore::zero();
score.inserted_tokens = token_changes.inserted_tokens;
score.deleted_tokens = token_changes.deleted_tokens;
+ score.editable_context_coverage = editable_context_coverage;
return score;
}
};
@@ -302,6 +324,7 @@ pub fn score_prediction(input: PredictionScoringInput<'_>) -> PredictionScore {
discarded_chars,
cumulative_logprob: input.cumulative_logprob,
avg_logprob: input.avg_logprob,
+ editable_context_coverage,
}
}
@@ -343,6 +366,7 @@ mod tests {
reversal_context: None,
cumulative_logprob: None,
avg_logprob: None,
+ context: None,
});
assert_eq!(score.delta_chr_f, 0.0);
diff --git a/crates/edit_prediction_metrics/src/summary.rs b/crates/edit_prediction_metrics/src/summary.rs
index 249ae185755..9c4028a300b 100644
--- a/crates/edit_prediction_metrics/src/summary.rs
+++ b/crates/edit_prediction_metrics/src/summary.rs
@@ -56,6 +56,30 @@ pub struct SummaryJson {
pub total_correctly_deleted_chars: Option<usize>,
#[serde(skip_serializing_if = "Option::is_none")]
pub total_discarded_chars: Option<usize>,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub avg_editable_context_lines_precision: Option<f64>,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub avg_editable_context_lines_recall: Option<f64>,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub avg_editable_context_lines_f1: Option<f64>,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub editable_context_lines_tp: Option<usize>,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub editable_context_lines_fp: Option<usize>,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub editable_context_lines_fn: Option<usize>,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub avg_editable_context_files_precision: Option<f64>,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub avg_editable_context_files_recall: Option<f64>,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub avg_editable_context_files_f1: Option<f64>,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub editable_context_files_tp: Option<usize>,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub editable_context_files_fp: Option<usize>,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub editable_context_files_fn: Option<usize>,
}
pub fn compute_summary<'a>(
@@ -91,6 +115,19 @@ pub fn compute_summary<'a>(
let mut discarded_chars_count: usize = 0;
let mut recall_rate_sum: f64 = 0.0;
let mut recall_rate_count: usize = 0;
+ let mut editable_context_lines_precision_sum: f64 = 0.0;
+ let mut editable_context_lines_recall_sum: f64 = 0.0;
+ let mut editable_context_lines_f1_sum: f64 = 0.0;
+ let mut editable_context_files_precision_sum: f64 = 0.0;
+ let mut editable_context_files_recall_sum: f64 = 0.0;
+ let mut editable_context_files_f1_sum: f64 = 0.0;
+ let mut editable_context_coverage_count: usize = 0;
+ let mut editable_context_lines_tp: usize = 0;
+ let mut editable_context_lines_fp: usize = 0;
+ let mut editable_context_lines_fn: usize = 0;
+ let mut editable_context_files_tp: usize = 0;
+ let mut editable_context_files_fp: usize = 0;
+ let mut editable_context_files_fn: usize = 0;
for prediction in predictions {
let score = prediction.score;
@@ -149,6 +186,21 @@ pub fn compute_summary<'a>(
recall_rate_sum += recall_rate;
recall_rate_count += 1;
}
+ if let Some(coverage) = &score.editable_context_coverage {
+ editable_context_lines_precision_sum += coverage.lines_precision;
+ editable_context_lines_recall_sum += coverage.lines_recall;
+ editable_context_lines_f1_sum += coverage.lines_f1;
+ editable_context_files_precision_sum += coverage.files_precision;
+ editable_context_files_recall_sum += coverage.files_recall;
+ editable_context_files_f1_sum += coverage.files_f1;
+ editable_context_coverage_count += 1;
+ editable_context_lines_tp += coverage.lines_tp;
+ editable_context_lines_fp += coverage.lines_fp;
+ editable_context_lines_fn += coverage.lines_fn;
+ editable_context_files_tp += coverage.files_tp;
+ editable_context_files_fp += coverage.files_fp;
+ editable_context_files_fn += coverage.files_fn;
+ }
if let Some(exact_match) = score.cursor_exact_match {
cursor_total += 1;
@@ -252,6 +304,67 @@ pub fn compute_summary<'a>(
None
};
+ let avg_editable_context_lines_precision = if editable_context_coverage_count > 0 {
+ Some(editable_context_lines_precision_sum / editable_context_coverage_count as f64)
+ } else {
+ None
+ };
+ let avg_editable_context_lines_recall = if editable_context_coverage_count > 0 {
+ Some(editable_context_lines_recall_sum / editable_context_coverage_count as f64)
+ } else {
+ None
+ };
+ let avg_editable_context_lines_f1 = if editable_context_coverage_count > 0 {
+ Some(editable_context_lines_f1_sum / editable_context_coverage_count as f64)
+ } else {
+ None
+ };
+ let editable_context_lines_tp = if editable_context_coverage_count > 0 {
+ Some(editable_context_lines_tp)
+ } else {
+ None
+ };
+ let editable_context_lines_fp = if editable_context_coverage_count > 0 {
+ Some(editable_context_lines_fp)
+ } else {
+ None
+ };
+ let editable_context_lines_fn = if editable_context_coverage_count > 0 {
+ Some(editable_context_lines_fn)
+ } else {
+ None
+ };
+ let avg_editable_context_files_precision = if editable_context_coverage_count > 0 {
+ Some(editable_context_files_precision_sum / editable_context_coverage_count as f64)
+ } else {
+ None
+ };
+ let avg_editable_context_files_recall = if editable_context_coverage_count > 0 {
+ Some(editable_context_files_recall_sum / editable_context_coverage_count as f64)
+ } else {
+ None
+ };
+ let avg_editable_context_files_f1 = if editable_context_coverage_count > 0 {
+ Some(editable_context_files_f1_sum / editable_context_coverage_count as f64)
+ } else {
+ None
+ };
+ let editable_context_files_tp = if editable_context_coverage_count > 0 {
+ Some(editable_context_files_tp)
+ } else {
+ None
+ };
+ let editable_context_files_fp = if editable_context_coverage_count > 0 {
+ Some(editable_context_files_fp)
+ } else {
+ None
+ };
+ let editable_context_files_fn = if editable_context_coverage_count > 0 {
+ Some(editable_context_files_fn)
+ } else {
+ None
+ };
+
SummaryJson {
total_examples: total_scores,
avg_delta_chr_f,
@@ -289,5 +402,17 @@ pub fn compute_summary<'a>(
total_kept_chars,
total_correctly_deleted_chars,
total_discarded_chars,
+ avg_editable_context_lines_precision,
+ avg_editable_context_lines_recall,
+ avg_editable_context_lines_f1,
+ editable_context_lines_tp,
+ editable_context_lines_fp,
+ editable_context_lines_fn,
+ avg_editable_context_files_precision,
+ avg_editable_context_files_recall,
+ avg_editable_context_files_f1,
+ editable_context_files_tp,
+ editable_context_files_fp,
+ editable_context_files_fn,
}
}
diff --git a/crates/zeta_prompt/src/zeta_prompt.rs b/crates/zeta_prompt/src/zeta_prompt.rs
index e685645e687..d513a9e66a4 100644
--- a/crates/zeta_prompt/src/zeta_prompt.rs
+++ b/crates/zeta_prompt/src/zeta_prompt.rs
@@ -240,6 +240,21 @@ pub struct RelatedExcerpt {
pub text: Arc<str>,
#[serde(default)]
pub order: usize,
+ #[serde(default)]
+ pub context_source: ContextSource,
+}
+
+#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum ContextSource {
+ #[default]
+ Lsp,
+ CursorExcerpt,
+ CurrentFile,
+ EditHistory,
+ EditHistoryFile,
+ GitLog,
+ OracleFile,
}
pub fn prompt_input_contains_special_tokens(input: &ZetaPromptInput, format: ZetaFormat) -> bool {
@@ -4851,6 +4866,7 @@ mod tests {
row_range: 0..content.lines().count() as u32,
text: content.into(),
order: 0,
+ context_source: ContextSource::Lsp,
}],
in_open_source_repo: false,
}
@@ -4970,16 +4986,19 @@ mod tests {
row_range: 0..10,
text: "first excerpt\n".into(),
order: 0,
+ context_source: ContextSource::Lsp,
},
RelatedExcerpt {
row_range: 10..20,
text: "second excerpt\n".into(),
order: 0,
+ context_source: ContextSource::Lsp,
},
RelatedExcerpt {
row_range: 20..30,
text: "third excerpt\n".into(),
order: 0,
+ context_source: ContextSource::Lsp,
},
],
}],
@@ -5039,6 +5058,7 @@ mod tests {
row_range: 0..10,
text: "low priority content\n".into(),
order: 5,
+ context_source: ContextSource::Lsp,
}],
},
RelatedFile {
@@ -5049,6 +5069,7 @@ mod tests {
row_range: 0..10,
text: "high priority content\n".into(),
order: 1,
+ context_source: ContextSource::Lsp,
}],
},
],
@@ -5113,16 +5134,19 @@ mod tests {
row_range: 0..5,
text: "mod header\n".into(),
order: 1,
+ context_source: ContextSource::Lsp,
},
RelatedExcerpt {
row_range: 5..15,
text: "important fn\n".into(),
order: 1,
+ context_source: ContextSource::Lsp,
},
RelatedExcerpt {
row_range: 15..30,
text: "less important fn\n".into(),
order: 3,
+ context_source: ContextSource::Lsp,
},
],
}],
@@ -5522,6 +5546,7 @@ mod tests {
row_range: 0..5,
text: "low prio\n".into(),
order: 10,
+ context_source: ContextSource::Lsp,
}],
},
RelatedFile {
@@ -5532,6 +5557,7 @@ mod tests {
row_range: 0..5,
text: "high prio\n".into(),
order: 1,
+ context_source: ContextSource::Lsp,
}],
},
],