Code at the speed of thought – Zed is a high-performance, multiplayer code editor from the creators of Atom and Tree-sitter.
edit_prediction: Send only format-specific FIM stop sequences (#65360)
Michael Kühnelefed1f245a2e
diff --git a/crates/edit_prediction/src/fim.rs b/crates/edit_prediction/src/fim.rsindex 5482fc1fb2a..0957298c3c0 100644--- a/crates/edit_prediction/src/fim.rs+++ b/crates/edit_prediction/src/fim.rs@@ -98,7 +98,10 @@ pub fn request_prediction(let prefix = editable_text[..cursor_in_editable].to_string();let suffix = editable_text[cursor_in_editable..].to_string();let prompt = format_fim_prompt(prompt_format, &prefix, &suffix);- let stop_tokens = get_fim_stop_tokens();+ let stop_tokens = fim_stop_tokens(prompt_format)+ .iter()+ .map(|token| token.to_string())+ .collect();let max_tokens = settings.max_output_tokens;@@ -219,23 +222,40 @@ fn format_fim_prompt(}}-fn get_fim_stop_tokens() -> Vec<String> {- vec![- "<|endoftext|>".to_string(),- "<|file_separator|>".to_string(),- "<|fim_pad|>".to_string(),- "<|fim_prefix|>".to_string(),- "<|fim_middle|>".to_string(),- "<|fim_suffix|>".to_string(),- "<fim_prefix>".to_string(),- "<fim_middle>".to_string(),- "<fim_suffix>".to_string(),- "<PRE>".to_string(),- "<SUF>".to_string(),- "<MID>".to_string(),- "[PREFIX]".to_string(),- "[SUFFIX]".to_string(),- ]+// OpenAI-compatible servers such as vLLM reject requests with more than 4 stop+// sequences, so each format may only list up to 4 tokens.+fn fim_stop_tokens(prompt_format: EditPredictionPromptFormat) -> &'static [&'static str] {+ match prompt_format {+ EditPredictionPromptFormat::CodeLlama => &["<PRE>", "<SUF>", "<MID>"],+ EditPredictionPromptFormat::DeepseekCoder => {+ &["<|fim▁begin|>", "<|fim▁hole|>", "<|fim▁end|>"]+ }+ EditPredictionPromptFormat::Qwen => &[+ "<|endoftext|>",+ "<|fim_prefix|>",+ "<|fim_middle|>",+ "<|fim_suffix|>",+ ],+ EditPredictionPromptFormat::CodeGemma => &[+ "<|file_separator|>",+ "<|fim_prefix|>",+ "<|fim_middle|>",+ "<|fim_suffix|>",+ ],+ EditPredictionPromptFormat::Codestral => &["[PREFIX]", "[SUFFIX]"],+ EditPredictionPromptFormat::Glm => &[+ "<|endoftext|>",+ "<|code_prefix|>",+ "<|code_suffix|>",+ "<|code_middle|>",+ ],+ _ => &[+ "<|endoftext|>",+ "<fim_prefix>",+ "<fim_middle>",+ "<fim_suffix>",+ ],+ }}fn clean_fim_completion(response: &str) -> String {@@ -315,6 +335,46 @@ mod tests {);}+ const FIM_PROMPT_FORMATS: [EditPredictionPromptFormat; 7] = [+ EditPredictionPromptFormat::CodeLlama,+ EditPredictionPromptFormat::StarCoder,+ EditPredictionPromptFormat::DeepseekCoder,+ EditPredictionPromptFormat::Qwen,+ EditPredictionPromptFormat::CodeGemma,+ EditPredictionPromptFormat::Codestral,+ EditPredictionPromptFormat::Glm,+ ];++ #[test]+ fn fim_stop_tokens_respect_openai_stop_limit() {+ for format in FIM_PROMPT_FORMATS {+ let stop_tokens = fim_stop_tokens(format);+ assert!(!stop_tokens.is_empty(), "{format:?} has no stop tokens");+ assert!(+ stop_tokens.len() <= 4,+ "{format:?} has {} stop tokens, but OpenAI allows at most 4",+ stop_tokens.len()+ );+ }+ }++ #[test]+ fn fim_stop_tokens_match_prompt_markers() {+ let end_of_text_tokens = ["<|endoftext|>", "<|file_separator|>"];+ for format in FIM_PROMPT_FORMATS {+ let prompt = format_fim_prompt(format, "a", "b");+ for token in fim_stop_tokens(format) {+ if end_of_text_tokens.contains(token) {+ continue;+ }+ assert!(+ prompt.contains(token),+ "stop token {token:?} for {format:?} does not appear in prompt {prompt:?}"+ );+ }+ }+ }+#[test]fn infer_prompt_format_returns_none_for_unsupported_models() {assert_eq!(infer_prompt_format("llama3:70b"), None);