Skip to content

armgit / zed

Public Imported from GitHub
0

Code at the speed of thought – Zed is a high-performance, multiplayer code editor from the creators of Atom and Tree-sitter.

edit_prediction: Send only format-specific FIM stop sequences (#65360)

Michael Kühnelefed1f245a2e
diff --git a/crates/edit_prediction/src/fim.rs b/crates/edit_prediction/src/fim.rs
index 5482fc1fb2a..0957298c3c0 100644
--- a/crates/edit_prediction/src/fim.rs
+++ b/crates/edit_prediction/src/fim.rs
@@ -98,7 +98,10 @@ pub fn request_prediction(
let prefix = editable_text[..cursor_in_editable].to_string();
let suffix = editable_text[cursor_in_editable..].to_string();
let prompt = format_fim_prompt(prompt_format, &prefix, &suffix);
- let stop_tokens = get_fim_stop_tokens();
+ let stop_tokens = fim_stop_tokens(prompt_format)
+ .iter()
+ .map(|token| token.to_string())
+ .collect();
let max_tokens = settings.max_output_tokens;
@@ -219,23 +222,40 @@ fn format_fim_prompt(
}
}
-fn get_fim_stop_tokens() -> Vec<String> {
- vec![
- "<|endoftext|>".to_string(),
- "<|file_separator|>".to_string(),
- "<|fim_pad|>".to_string(),
- "<|fim_prefix|>".to_string(),
- "<|fim_middle|>".to_string(),
- "<|fim_suffix|>".to_string(),
- "<fim_prefix>".to_string(),
- "<fim_middle>".to_string(),
- "<fim_suffix>".to_string(),
- "<PRE>".to_string(),
- "<SUF>".to_string(),
- "<MID>".to_string(),
- "[PREFIX]".to_string(),
- "[SUFFIX]".to_string(),
- ]
+// OpenAI-compatible servers such as vLLM reject requests with more than 4 stop
+// sequences, so each format may only list up to 4 tokens.
+fn fim_stop_tokens(prompt_format: EditPredictionPromptFormat) -> &'static [&'static str] {
+ match prompt_format {
+ EditPredictionPromptFormat::CodeLlama => &["<PRE>", "<SUF>", "<MID>"],
+ EditPredictionPromptFormat::DeepseekCoder => {
+ &["<|fim▁begin|>", "<|fim▁hole|>", "<|fim▁end|>"]
+ }
+ EditPredictionPromptFormat::Qwen => &[
+ "<|endoftext|>",
+ "<|fim_prefix|>",
+ "<|fim_middle|>",
+ "<|fim_suffix|>",
+ ],
+ EditPredictionPromptFormat::CodeGemma => &[
+ "<|file_separator|>",
+ "<|fim_prefix|>",
+ "<|fim_middle|>",
+ "<|fim_suffix|>",
+ ],
+ EditPredictionPromptFormat::Codestral => &["[PREFIX]", "[SUFFIX]"],
+ EditPredictionPromptFormat::Glm => &[
+ "<|endoftext|>",
+ "<|code_prefix|>",
+ "<|code_suffix|>",
+ "<|code_middle|>",
+ ],
+ _ => &[
+ "<|endoftext|>",
+ "<fim_prefix>",
+ "<fim_middle>",
+ "<fim_suffix>",
+ ],
+ }
}
fn clean_fim_completion(response: &str) -> String {
@@ -315,6 +335,46 @@ mod tests {
);
}
+ const FIM_PROMPT_FORMATS: [EditPredictionPromptFormat; 7] = [
+ EditPredictionPromptFormat::CodeLlama,
+ EditPredictionPromptFormat::StarCoder,
+ EditPredictionPromptFormat::DeepseekCoder,
+ EditPredictionPromptFormat::Qwen,
+ EditPredictionPromptFormat::CodeGemma,
+ EditPredictionPromptFormat::Codestral,
+ EditPredictionPromptFormat::Glm,
+ ];
+
+ #[test]
+ fn fim_stop_tokens_respect_openai_stop_limit() {
+ for format in FIM_PROMPT_FORMATS {
+ let stop_tokens = fim_stop_tokens(format);
+ assert!(!stop_tokens.is_empty(), "{format:?} has no stop tokens");
+ assert!(
+ stop_tokens.len() <= 4,
+ "{format:?} has {} stop tokens, but OpenAI allows at most 4",
+ stop_tokens.len()
+ );
+ }
+ }
+
+ #[test]
+ fn fim_stop_tokens_match_prompt_markers() {
+ let end_of_text_tokens = ["<|endoftext|>", "<|file_separator|>"];
+ for format in FIM_PROMPT_FORMATS {
+ let prompt = format_fim_prompt(format, "a", "b");
+ for token in fim_stop_tokens(format) {
+ if end_of_text_tokens.contains(token) {
+ continue;
+ }
+ assert!(
+ prompt.contains(token),
+ "stop token {token:?} for {format:?} does not appear in prompt {prompt:?}"
+ );
+ }
+ }
+ }
+
#[test]
fn infer_prompt_format_returns_none_for_unsupported_models() {
assert_eq!(infer_prompt_format("llama3:70b"), None);