Skip to content

armgit / zed

Public Imported from GitHub
0

Code at the speed of thought – Zed is a high-performance, multiplayer code editor from the creators of Atom and Tree-sitter.

ep: Add accepted and explicit reject fetching (#58434)

Ben Kunkleb713cac496ac
diff --git a/crates/edit_prediction_cli/src/main.rs b/crates/edit_prediction_cli/src/main.rs
index 63f701999d0..8076d2280f6 100644
--- a/crates/edit_prediction_cli/src/main.rs
+++ b/crates/edit_prediction_cli/src/main.rs
@@ -619,6 +619,7 @@ impl EpArgs {
/// This version introduced the current request schema with predicted edits in the edit
/// history, and open source repos distinguished.
const MIN_CAPTURE_VERSION: pull_examples::MinCaptureVersion = pull_examples::MinCaptureVersion {
+ major: 0,
minor: 224,
patch: 1,
};
@@ -765,16 +766,21 @@ async fn load_examples(
let mut settled_after_timestamps = Vec::new();
let mut rated_after_inputs: Vec<(String, Option<telemetry_events::EditPredictionRating>)> =
Vec::new();
+ let mut accepted_after_timestamps = Vec::new();
let mut file_inputs = Vec::new();
for input in &args.inputs {
let input_string = input.to_string_lossy();
if let Some(timestamp) = pull_examples::parse_captured_after_input(input_string.as_ref()) {
captured_after_timestamps.push(timestamp.to_string());
- } else if let Some(timestamp) =
+ } else if let Some((explicit, timestamp)) =
pull_examples::parse_rejected_after_input(input_string.as_ref())
{
- rejected_after_timestamps.push(timestamp.to_string());
+ rejected_after_timestamps.push((explicit, timestamp.to_string()));
+ } else if let Some(timestamp) =
+ pull_examples::parse_accepted_after_input(input_string.as_ref())
+ {
+ accepted_after_timestamps.push(timestamp.to_string());
} else if let Some(timestamp) =
pull_examples::parse_requested_after_input(input_string.as_ref())
{
@@ -833,6 +839,21 @@ async fn load_examples(
examples.append(&mut rejected_examples);
}
+ if !accepted_after_timestamps.is_empty() {
+ accepted_after_timestamps.sort();
+
+ let mut accepted_examples = pull_examples::fetch_accepted_examples_after(
+ http_client.clone(),
+ &accepted_after_timestamps,
+ max_rows_per_timestamp,
+ remaining_offset,
+ background_executor.clone(),
+ Some(MIN_CAPTURE_VERSION),
+ )
+ .await?;
+ examples.append(&mut accepted_examples);
+ }
+
if !requested_after_timestamps.is_empty() {
requested_after_timestamps.sort();
diff --git a/crates/edit_prediction_cli/src/pull_examples.rs b/crates/edit_prediction_cli/src/pull_examples.rs
index 88de3053d85..73778ef5484 100644
--- a/crates/edit_prediction_cli/src/pull_examples.rs
+++ b/crates/edit_prediction_cli/src/pull_examples.rs
@@ -15,7 +15,7 @@ use telemetry_events::EditPredictionRating;
use zeta_prompt::{ZetaFormat, ZetaPromptInput, excerpt_range_for_format};
use crate::PredictionProvider;
-use crate::example::{Example, ExamplePrompt};
+use crate::example::{Example, ExamplePrediction, ExamplePrompt};
use crate::progress::{InfoStyle, Progress, Step};
use edit_prediction::example_spec::{ExampleSpec, TelemetrySource};
@@ -24,10 +24,13 @@ pub(crate) const SNOWFLAKE_ASYNC_IN_PROGRESS_CODE: &str = "333334";
const SNOWFLAKE_TIMEOUT_CODE: &str = "000630";
/// Minimum Zed version for filtering captured examples.
-/// For example, `MinCaptureVersion { minor: 224, patch: 1 }` means only pull examples
-/// where `zed_version >= 0.224.1`.
+/// For example, `MinCaptureVersion { major: 0, minor: 224, patch: 1 }` means only pull
+/// examples where `zed_version >= 0.224.1`. The `major` component is required because Zed
+/// moved from the `0.<minor>.<patch>` scheme to `1.<minor>.<patch>`; comparing on `minor`
+/// alone would exclude all `1.*` versions (whose `minor` resets to small values).
#[derive(Clone, Copy, Debug)]
pub struct MinCaptureVersion {
+ pub major: u32,
pub minor: u32,
pub patch: u32,
}
@@ -45,9 +48,20 @@ pub fn parse_captured_after_input(input: &str) -> Option<&str> {
input.strip_prefix("captured-after:")
}
+/// Parse an input token of the form `accepted-after:{timestamp}`.
+pub fn parse_accepted_after_input(input: &str) -> Option<&str> {
+ input.strip_prefix("accepted-after:")
+}
+
/// Parse an input token of the form `rejected-after:{timestamp}`.
-pub fn parse_rejected_after_input(input: &str) -> Option<&str> {
- input.strip_prefix("rejected-after:")
+pub fn parse_rejected_after_input(input: &str) -> Option<(bool, &str)> {
+ if let Some(timestamp) = input.strip_prefix("rejected-after:") {
+ Some((false, timestamp))
+ } else if let Some(timestamp) = input.strip_prefix("explicitly-rejected-after:") {
+ Some((true, timestamp))
+ } else {
+ None
+ }
}
/// Parse an input token of the form `requested-after:{timestamp}`.
@@ -543,7 +557,7 @@ pub(crate) async fn run_sql(
pub async fn fetch_rejected_examples_after(
http_client: Arc<dyn HttpClient>,
- after_timestamps: &[String],
+ after_timestamps: &[(bool, String)],
max_rows_per_timestamp: Option<usize>,
offset: usize,
background_executor: BackgroundExecutor,
@@ -557,15 +571,16 @@ pub async fn fetch_rejected_examples_after(
let mut all_examples = Vec::new();
- for after_date in after_timestamps.iter() {
+ for (explicit, after_date) in after_timestamps.iter() {
let step_progress_name = format!("rejected>{after_date}");
let step_progress = progress.start(Step::PullExamples, &step_progress_name);
step_progress.set_substatus("querying");
- let min_minor_str = min_capture_version.map(|version| version.minor.to_string());
- let min_patch_str = min_capture_version.map(|version| version.patch.to_string());
- let min_minor_str_ref = min_minor_str.as_deref();
- let min_patch_str_ref = min_patch_str.as_deref();
+ let min_version_str = min_capture_version.map(|version| {
+ (version.major as u64 * 1_000_000 + version.minor as u64 * 1_000 + version.patch as u64)
+ .to_string()
+ });
+ let min_version_ref = min_version_str.as_deref();
let statement = indoc! {r#"
SELECT
@@ -581,16 +596,14 @@ pub async fn fetch_rejected_examples_after(
ep_rejected_reason AS reason,
zed_version AS zed_version
FROM ZED_DBT.DBT_PROD.fct_edit_prediction_examples
- WHERE ep_outcome LIKE 'Rejected%'
+ WHERE ep_outcome LIKE ?
AND is_ep_shown_before_rejected = true
AND requested_at > TRY_TO_TIMESTAMP_NTZ(?)
AND (? IS NULL OR (
- TRY_CAST(SPLIT_PART(zed_version, '.', 2) AS INTEGER) > ?
- OR (
- TRY_CAST(SPLIT_PART(zed_version, '.', 2) AS INTEGER) = ?
- AND TRY_CAST(SPLIT_PART(SPLIT_PART(zed_version, '.', 3), '+', 1) AS INTEGER) >= ?
- )
- ))
+ COALESCE(TRY_CAST(SPLIT_PART(zed_version, '.', 1) AS INTEGER), 0) * 1000000
+ + COALESCE(TRY_CAST(SPLIT_PART(zed_version, '.', 2) AS INTEGER), 0) * 1000
+ + COALESCE(TRY_CAST(SPLIT_PART(SPLIT_PART(zed_version, '.', 3), '+', 1) AS INTEGER), 0)
+ ) >= ?)
ORDER BY requested_at ASC
LIMIT ?
OFFSET ?
@@ -608,13 +621,12 @@ pub async fn fetch_rejected_examples_after(
},
|retry_state| {
json!({
- "1": { "type": "TEXT", "value": retry_state.resume_after },
- "2": { "type": "FIXED", "value": min_minor_str_ref },
- "3": { "type": "FIXED", "value": min_minor_str_ref },
- "4": { "type": "FIXED", "value": min_minor_str_ref },
- "5": { "type": "FIXED", "value": min_patch_str_ref },
- "6": { "type": "FIXED", "value": format_limit(retry_state.remaining_limit) },
- "7": { "type": "FIXED", "value": retry_state.offset.to_string() }
+ "1": { "type": "TEXT", "value": if *explicit { "Rejected (Explicit)" } else { "Rejected%" } },
+ "2": { "type": "TEXT", "value": retry_state.resume_after },
+ "3": { "type": "FIXED", "value": min_version_ref },
+ "4": { "type": "FIXED", "value": min_version_ref },
+ "5": { "type": "FIXED", "value": format_limit(retry_state.remaining_limit) },
+ "6": { "type": "FIXED", "value": retry_state.offset.to_string() }
})
},
&[
@@ -639,6 +651,96 @@ pub async fn fetch_rejected_examples_after(
Ok(all_examples)
}
+pub async fn fetch_accepted_examples_after(
+ http_client: Arc<dyn HttpClient>,
+ after_timestamps: &[String],
+ max_rows_per_timestamp: Option<usize>,
+ offset: usize,
+ background_executor: BackgroundExecutor,
+ min_capture_version: Option<MinCaptureVersion>,
+) -> Result<Vec<Example>> {
+ if after_timestamps.is_empty() {
+ return Ok(Vec::new());
+ }
+
+ let progress = Progress::global();
+
+ let mut all_examples = Vec::new();
+
+ for after_date in after_timestamps.iter() {
+ let step_progress_name = format!("accepted>{after_date}");
+ let step_progress = progress.start(Step::PullExamples, &step_progress_name);
+ step_progress.set_substatus("querying");
+
+ let min_version_str = min_capture_version.map(|version| {
+ (version.major as u64 * 1_000_000 + version.minor as u64 * 1_000 + version.patch as u64)
+ .to_string()
+ });
+ let min_version_ref = min_version_str.as_deref();
+
+ let statement = indoc! {r#"
+ SELECT
+ ep_request_id AS request_id,
+ device_id AS device_id,
+ requested_at::string AS continuation_time,
+ requested_at::string AS time,
+ input_payload AS input,
+ prompt AS prompt,
+ requested_output AS output,
+ settled_editable_region AS settled_editable_region,
+ zed_version AS zed_version
+ FROM ZED_DBT.DBT_PROD.fct_edit_prediction_examples
+ WHERE ep_outcome = 'Accepted'
+ AND requested_at > TRY_TO_TIMESTAMP_NTZ(?)
+ AND (? IS NULL OR (
+ COALESCE(TRY_CAST(SPLIT_PART(zed_version, '.', 1) AS INTEGER), 0) * 1000000
+ + COALESCE(TRY_CAST(SPLIT_PART(zed_version, '.', 2) AS INTEGER), 0) * 1000
+ + COALESCE(TRY_CAST(SPLIT_PART(SPLIT_PART(zed_version, '.', 3), '+', 1) AS INTEGER), 0)
+ ) >= ?)
+ ORDER BY requested_at ASC
+ LIMIT ?
+ OFFSET ?
+ "#};
+
+ let examples = fetch_examples_with_query(
+ http_client.clone(),
+ &step_progress,
+ background_executor.clone(),
+ statement,
+ QueryRetryState {
+ resume_after: after_date.clone(),
+ remaining_limit: max_rows_per_timestamp,
+ offset,
+ },
+ |retry_state| {
+ json!({
+ "1": { "type": "TEXT", "value": retry_state.resume_after },
+ "2": { "type": "FIXED", "value": min_version_ref },
+ "3": { "type": "FIXED", "value": min_version_ref },
+ "4": { "type": "FIXED", "value": format_limit(retry_state.remaining_limit) },
+ "5": { "type": "FIXED", "value": retry_state.offset.to_string() }
+ })
+ },
+ &[
+ "request_id",
+ "device_id",
+ "time",
+ "input",
+ "prompt",
+ "output",
+ "settled_editable_region",
+ "zed_version",
+ ],
+ accepted_examples_from_response,
+ )
+ .await?;
+
+ all_examples.extend(examples);
+ }
+
+ Ok(all_examples)
+}
+
fn format_limit(limit: Option<usize>) -> String {
return limit.map(|l| l.to_string()).unwrap_or("NULL".to_string());
}
@@ -664,10 +766,11 @@ pub async fn fetch_requested_examples_after(
let step_progress = progress.start(Step::PullExamples, &step_progress_name);
step_progress.set_substatus("querying");
- let min_minor_str = min_capture_version.map(|version| version.minor.to_string());
- let min_patch_str = min_capture_version.map(|version| version.patch.to_string());
- let min_minor_str_ref = min_minor_str.as_deref();
- let min_patch_str_ref = min_patch_str.as_deref();
+ let min_version_str = min_capture_version.map(|version| {
+ (version.major as u64 * 1_000_000 + version.minor as u64 * 1_000 + version.patch as u64)
+ .to_string()
+ });
+ let min_version_ref = min_version_str.as_deref();
let statement = indoc! {r#"
SELECT
@@ -680,12 +783,10 @@ pub async fn fetch_requested_examples_after(
FROM ZED_DBT.DBT_PROD.fct_edit_prediction_examples
WHERE requested_at > TRY_TO_TIMESTAMP_NTZ(?)
AND (? IS NULL OR (
- TRY_CAST(SPLIT_PART(zed_version, '.', 2) AS INTEGER) > ?
- OR (
- TRY_CAST(SPLIT_PART(zed_version, '.', 2) AS INTEGER) = ?
- AND TRY_CAST(SPLIT_PART(SPLIT_PART(zed_version, '.', 3), '+', 1) AS INTEGER) >= ?
- )
- ))
+ COALESCE(TRY_CAST(SPLIT_PART(zed_version, '.', 1) AS INTEGER), 0) * 1000000
+ + COALESCE(TRY_CAST(SPLIT_PART(zed_version, '.', 2) AS INTEGER), 0) * 1000
+ + COALESCE(TRY_CAST(SPLIT_PART(SPLIT_PART(zed_version, '.', 3), '+', 1) AS INTEGER), 0)
+ ) >= ?)
ORDER BY requested_at ASC
LIMIT ?
OFFSET ?
@@ -704,12 +805,10 @@ pub async fn fetch_requested_examples_after(
|retry_state| {
json!({
"1": { "type": "TEXT", "value": retry_state.resume_after },
- "2": { "type": "FIXED", "value": min_minor_str_ref },
- "3": { "type": "FIXED", "value": min_minor_str_ref },
- "4": { "type": "FIXED", "value": min_minor_str_ref },
- "5": { "type": "FIXED", "value": min_patch_str_ref },
- "6": { "type": "FIXED", "value": format_limit(retry_state.remaining_limit) },
- "7": { "type": "FIXED", "value": retry_state.offset.to_string() }
+ "2": { "type": "FIXED", "value": min_version_ref },
+ "3": { "type": "FIXED", "value": min_version_ref },
+ "4": { "type": "FIXED", "value": format_limit(retry_state.remaining_limit) },
+ "5": { "type": "FIXED", "value": retry_state.offset.to_string() }
})
},
&["request_id", "device_id", "time", "input", "zed_version"],
@@ -744,10 +843,11 @@ pub async fn fetch_captured_examples_after(
let step_progress = progress.start(Step::PullExamples, &step_progress_name);
step_progress.set_substatus("querying");
- let min_minor_str = min_capture_version.map(|version| version.minor.to_string());
- let min_patch_str = min_capture_version.map(|version| version.patch.to_string());
- let min_minor_str_ref = min_minor_str.as_deref();
- let min_patch_str_ref = min_patch_str.as_deref();
+ let min_version_str = min_capture_version.map(|version| {
+ (version.major as u64 * 1_000_000 + version.minor as u64 * 1_000 + version.patch as u64)
+ .to_string()
+ });
+ let min_version_ref = min_version_str.as_deref();
let statement = indoc! {r#"
SELECT
@@ -764,12 +864,10 @@ pub async fn fetch_captured_examples_after(
AND example_payload IS NOT NULL
AND requested_at > TRY_TO_TIMESTAMP_NTZ(?)
AND (? IS NULL OR (
- TRY_CAST(SPLIT_PART(zed_version, '.', 2) AS INTEGER) > ?
- OR (
- TRY_CAST(SPLIT_PART(zed_version, '.', 2) AS INTEGER) = ?
- AND TRY_CAST(SPLIT_PART(SPLIT_PART(zed_version, '.', 3), '+', 1) AS INTEGER) >= ?
- )
- ))
+ COALESCE(TRY_CAST(SPLIT_PART(zed_version, '.', 1) AS INTEGER), 0) * 1000000
+ + COALESCE(TRY_CAST(SPLIT_PART(zed_version, '.', 2) AS INTEGER), 0) * 1000
+ + COALESCE(TRY_CAST(SPLIT_PART(SPLIT_PART(zed_version, '.', 3), '+', 1) AS INTEGER), 0)
+ ) >= ?)
ORDER BY requested_at ASC
LIMIT ?
OFFSET ?
@@ -788,12 +886,10 @@ pub async fn fetch_captured_examples_after(
|retry_state| {
json!({
"1": { "type": "TEXT", "value": retry_state.resume_after },
- "2": { "type": "FIXED", "value": min_minor_str_ref },
- "3": { "type": "FIXED", "value": min_minor_str_ref },
- "4": { "type": "FIXED", "value": min_minor_str_ref },
- "5": { "type": "FIXED", "value": min_patch_str_ref },
- "6": { "type": "FIXED", "value": format_limit(retry_state.remaining_limit) },
- "7": { "type": "FIXED", "value": retry_state.offset.to_string() }
+ "2": { "type": "FIXED", "value": min_version_ref },
+ "3": { "type": "FIXED", "value": min_version_ref },
+ "4": { "type": "FIXED", "value": format_limit(retry_state.remaining_limit) },
+ "5": { "type": "FIXED", "value": retry_state.offset.to_string() }
})
},
&[
@@ -1724,6 +1820,143 @@ struct RejectionInfo {
was_shown: bool,
}
+fn accepted_examples_from_response<'a>(
+ response: &'a SnowflakeStatementResponse,
+ column_indices: &'a std::collections::HashMap<String, usize>,
+) -> Result<Box<dyn Iterator<Item = Example> + 'a>> {
+ if let Some(code) = &response.code {
+ if code != SNOWFLAKE_SUCCESS_CODE {
+ anyhow::bail!(
+ "snowflake sql api returned error code={code} message={}",
+ response.message.as_deref().unwrap_or("<no message>")
+ );
+ }
+ }
+
+ let iter = response
+ .data
+ .iter()
+ .enumerate()
+ .filter_map(move |(row_index, data_row)| {
+ let get_string = |name: &str| -> Option<String> {
+ let index = column_indices.get(name).copied()?;
+ match data_row.get(index)? {
+ JsonValue::String(s) => Some(s.clone()),
+ JsonValue::Null => None,
+ other => Some(other.to_string()),
+ }
+ };
+
+ let get_json = |name: &str| -> Option<JsonValue> {
+ let index = column_indices.get(name).copied()?;
+ let value = data_row.get(index)?;
+ if value.is_null() {
+ return None;
+ }
+ match value {
+ JsonValue::String(s) => serde_json::from_str(s).ok(),
+ other => Some(other.clone()),
+ }
+ };
+
+ let request_id_str = get_string("request_id");
+ let device_id = get_string("device_id");
+ let time = get_string("time");
+ let input_json = get_json("input");
+ let input: Option<ZetaPromptInput> =
+ input_json.clone().and_then(|v| serde_json::from_value(v).ok());
+ let prompt = get_string("prompt");
+ let output = get_string("output");
+ let settled_editable_region = get_string("settled_editable_region");
+ let zed_version = get_string("zed_version");
+
+ match (request_id_str.clone(), device_id.clone(), time.clone(), input, output.clone()) {
+ (Some(request_id), Some(device_id), Some(time), Some(input), Some(output)) => {
+ Some(build_accepted_example(
+ request_id,
+ device_id,
+ time,
+ input,
+ prompt,
+ output,
+ settled_editable_region,
+ zed_version,
+ ))
+ }
+ _ => {
+ log::warn!(
+ "skipping row {row_index}: missing fields - request_id={:?} device_id={:?} time={:?} input={:?} output={:?}",
+ request_id_str.is_some(),
+ device_id.is_some(),
+ time.is_some(),
+ input_json.is_some(),
+ output.is_some(),
+ );
+ None
+ }
+ }
+ });
+
+ Ok(Box::new(iter))
+}
+
+fn build_accepted_example(
+ request_id: String,
+ device_id: String,
+ time: String,
+ input: ZetaPromptInput,
+ prompt: Option<String>,
+ output: String,
+ settled_editable_region: Option<String>,
+ zed_version: Option<String>,
+) -> Example {
+ let accepted_patch = build_output_patch(
+ &input.cursor_path,
+ input.cursor_excerpt.as_ref(),
+ &input.excerpt_ranges.editable_350,
+ &output,
+ );
+ let expected_patch = settled_editable_region
+ .as_ref()
+ .map(|settled_editable_region| {
+ build_output_patch(
+ &input.cursor_path,
+ input.cursor_excerpt.as_ref(),
+ &input.excerpt_ranges.editable_350,
+ settled_editable_region,
+ )
+ });
+ let mut example = build_example_from_snowflake(
+ request_id,
+ device_id,
+ time,
+ input,
+ vec!["accepted".to_string()],
+ None,
+ zed_version,
+ );
+ if let Some(expected_patch) = expected_patch {
+ example.spec.expected_patches = vec![expected_patch];
+ }
+ example.predictions.push(ExamplePrediction {
+ provider: PredictionProvider::default(),
+ actual_output: output.clone(),
+ actual_patch: Some(accepted_patch),
+ actual_cursor: None, // todo: why no cursor?
+ error: None,
+ cumulative_logprob: None,
+ avg_logprob: None,
+ });
+ example.prompt = prompt.map(|prompt| ExamplePrompt {
+ input: prompt,
+ expected_output: Some(output),
+ rejected_output: None,
+ prefill: None,
+ provider: PredictionProvider::default(),
+ });
+ example
+}
+
fn build_example_from_snowflake(
request_id: String,
device_id: String,
diff --git a/crates/zeta_prompt/src/zeta_prompt.rs b/crates/zeta_prompt/src/zeta_prompt.rs
index d513a9e66a4..6e46789937e 100644
--- a/crates/zeta_prompt/src/zeta_prompt.rs
+++ b/crates/zeta_prompt/src/zeta_prompt.rs
@@ -416,6 +416,61 @@ pub fn stop_tokens_for_format(format: ZetaFormat) -> &'static [&'static str] {
}
}
+/// Delimiters used by response-only SFT (e.g. Unsloth `train_on_responses_only`)
+/// to mask the prompt and train only on the model's completion.
+///
+/// Both strings must appear verbatim in the prompt produced by
+/// [`format_zeta_prompt`] for the same format: `instruction_part` marks the
+/// start of an example, and `response_part` is the final marker before the
+/// completion begins.
+pub struct TrainingDelimiters {
+ pub instruction_part: &'static str,
+ pub response_part: &'static str,
+}
+
+/// Return the response-only training delimiters for a format.
+///
+/// This match is intentionally exhaustive with no wildcard arm so that adding a
+/// new [`ZetaFormat`] fails to compile until its delimiters are specified.
+pub fn training_delimiters_for_format(format: ZetaFormat) -> TrainingDelimiters {
+ match format {
+ ZetaFormat::V0211SeedCoder
+ | ZetaFormat::V0331SeedCoderModelPy
+ | ZetaFormat::V0304SeedNoEdits
+ | ZetaFormat::V0306SeedMultiRegions
+ | ZetaFormat::V0316SeedMultiRegions
+ | ZetaFormat::V0317SeedMultiRegions
+ | ZetaFormat::V0318SeedMultiRegions
+ | ZetaFormat::V0327SingleFile
+ | ZetaFormat::V0420Diagnostics => TrainingDelimiters {
+ instruction_part: seed_coder::FIM_SUFFIX,
+ response_part: seed_coder::FIM_MIDDLE,
+ },
+ ZetaFormat::V0112MiddleAtEnd
+ | ZetaFormat::V0113Ordered
+ | ZetaFormat::V0114180EditableRegion => TrainingDelimiters {
+ instruction_part: "<|file_sep|>",
+ response_part: "<|fim_middle|>updated\n",
+ },
+ ZetaFormat::V0120GitMergeMarkers => TrainingDelimiters {
+ instruction_part: "<|file_sep|>",
+ response_part: v0120_git_merge_markers::SEPARATOR,
+ },
+ ZetaFormat::V0131GitMergeMarkersPrefix | ZetaFormat::V0211Prefill => TrainingDelimiters {
+ instruction_part: "<|file_sep|>",
+ response_part: "<|fim_middle|>",
+ },
+ ZetaFormat::v0226Hashline => TrainingDelimiters {
+ instruction_part: "<|file_sep|>",
+ response_part: hashline::END_MARKER,
+ },
+ ZetaFormat::V0304VariableEdit => TrainingDelimiters {
+ instruction_part: "<|file_sep|>",
+ response_part: "<|fim_prefix|>",
+ },
+ }
+}
+
/// Return (editable_range, context_range) for the prompt format
pub fn excerpt_ranges_for_format(
format: ZetaFormat,