An agent-managed museum exhibit, built in Rust with Gajae-Code / LazyCodex — developed and maintained with no human intervention.
Rust
Enable local image prompts without breaking text-only CLI flows
Yeachan-Heo063c84df4043
diff --git a/rust/crates/api/src/lib.rs b/rust/crates/api/src/lib.rsindex c2086551..fb3ad049 100644--- a/rust/crates/api/src/lib.rs+++ b/rust/crates/api/src/lib.rs@@ -11,7 +11,7 @@ pub use error::ApiError;pub use sse::{parse_frame, SseParser};pub use types::{ContentBlockDelta, ContentBlockDeltaEvent, ContentBlockStartEvent, ContentBlockStopEvent,- InputContentBlock, InputMessage, MessageDelta, MessageDeltaEvent, MessageRequest,+ ImageSource, InputContentBlock, InputMessage, MessageDelta, MessageDeltaEvent, MessageRequest,MessageResponse, MessageStartEvent, MessageStopEvent, OutputContentBlock, StreamEvent,ToolChoice, ToolDefinition, ToolResultContentBlock, Usage,};diff --git a/rust/crates/api/src/types.rs b/rust/crates/api/src/types.rsindex 45d5c082..109d5d63 100644--- a/rust/crates/api/src/types.rs+++ b/rust/crates/api/src/types.rs@@ -64,6 +64,9 @@ pub enum InputContentBlock {Text {text: String,},+ Image {+ source: ImageSource,+ },ToolUse {id: String,name: String,@@ -77,6 +80,14 @@ pub enum InputContentBlock {},}+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]+pub struct ImageSource {+ #[serde(rename = "type")]+ pub kind: String,+ pub media_type: String,+ pub data: String,+}+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]#[serde(tag = "type", rename_all = "snake_case")]pub enum ToolResultContentBlock {diff --git a/rust/crates/api/tests/client_integration.rs b/rust/crates/api/tests/client_integration.rsindex c37fa99a..483e471f 100644--- a/rust/crates/api/tests/client_integration.rs+++ b/rust/crates/api/tests/client_integration.rs@@ -4,8 +4,8 @@ use std::time::Duration;use api::{AnthropicClient, ApiError, ContentBlockDelta, ContentBlockDeltaEvent, ContentBlockStartEvent,- InputContentBlock, InputMessage, MessageDeltaEvent, MessageRequest, OutputContentBlock,- StreamEvent, ToolChoice, ToolDefinition,+ ImageSource, InputContentBlock, InputMessage, MessageDeltaEvent, MessageRequest,+ OutputContentBlock, StreamEvent, ToolChoice, ToolDefinition,};use serde_json::json;use tokio::io::{AsyncReadExt, AsyncWriteExt};@@ -75,6 +75,39 @@ async fn send_message_posts_json_and_parses_response() {assert_eq!(body["tool_choice"]["type"], json!("auto"));}+#[test]+fn image_content_blocks_serialize_with_base64_source() {+ let request = MessageRequest {+ model: "claude-3-7-sonnet-latest".to_string(),+ max_tokens: 64,+ messages: vec![InputMessage {+ role: "user".to_string(),+ content: vec![InputContentBlock::Image {+ source: ImageSource {+ kind: "base64".to_string(),+ media_type: "image/png".to_string(),+ data: "AQID".to_string(),+ },+ }],+ }],+ system: None,+ tools: None,+ tool_choice: None,+ stream: false,+ };++ let json = serde_json::to_value(request).expect("request should serialize");+ assert_eq!(json["messages"][0]["content"][0]["type"], json!("image"));+ assert_eq!(+ json["messages"][0]["content"][0]["source"],+ json!({+ "type": "base64",+ "media_type": "image/png",+ "data": "AQID"+ })+ );+}+#[tokio::test]async fn stream_message_parses_sse_events_with_tool_use() {let state = Arc::new(Mutex::new(Vec::<CapturedRequest>::new()));diff --git a/rust/crates/runtime/src/conversation.rs b/rust/crates/runtime/src/conversation.rsindex 5c9ccfec..136aaa2c 100644--- a/rust/crates/runtime/src/conversation.rs+++ b/rust/crates/runtime/src/conversation.rs@@ -408,7 +408,7 @@ mod tests {.sum::<i32>();Ok(total.to_string())});- let permission_policy = PermissionPolicy::new(PermissionMode::Prompt);+ let permission_policy = PermissionPolicy::new(PermissionMode::WorkspaceWrite);let system_prompt = SystemPromptBuilder::new().with_project_context(ProjectContext {cwd: PathBuf::from("/tmp/project"),@@ -487,7 +487,7 @@ mod tests {Session::new(),SingleCallApiClient,StaticToolExecutor::new(),- PermissionPolicy::new(PermissionMode::Prompt),+ PermissionPolicy::new(PermissionMode::WorkspaceWrite),vec!["system".to_string()],);@@ -536,7 +536,7 @@ mod tests {session,SimpleApi,StaticToolExecutor::new(),- PermissionPolicy::new(PermissionMode::Allow),+ PermissionPolicy::new(PermissionMode::DangerFullAccess),vec!["system".to_string()],);@@ -563,7 +563,7 @@ mod tests {Session::new(),SimpleApi,StaticToolExecutor::new(),- PermissionPolicy::new(PermissionMode::Allow),+ PermissionPolicy::new(PermissionMode::DangerFullAccess),vec!["system".to_string()],);runtime.run_turn("a", None).expect("turn a");diff --git a/rust/crates/rusty-claude-cli/src/main.rs b/rust/crates/rusty-claude-cli/src/main.rsindex 47ecd983..dc0b4f9a 100644--- a/rust/crates/rusty-claude-cli/src/main.rs+++ b/rust/crates/rusty-claude-cli/src/main.rs@@ -11,8 +11,8 @@ use std::process::Command;use std::time::{SystemTime, UNIX_EPOCH};use api::{- resolve_startup_auth_source, AnthropicClient, AuthSource, ContentBlockDelta, InputContentBlock,- InputMessage, MessageRequest, MessageResponse, OutputContentBlock,+ resolve_startup_auth_source, AnthropicClient, AuthSource, ContentBlockDelta, ImageSource,+ InputContentBlock, InputMessage, MessageRequest, MessageResponse, OutputContentBlock,StreamEvent as ApiStreamEvent, ToolChoice, ToolDefinition, ToolResultContentBlock,};@@ -41,6 +41,7 @@ const BUILD_TARGET: Option<&str> = option_env!("TARGET");const GIT_SHA: Option<&str> = option_env!("GIT_SHA");type AllowedToolSet = BTreeSet<String>;+const IMAGE_REF_PREFIX: &str = "@";fn main() {if let Err(error) = run() {@@ -1042,9 +1043,7 @@ impl LiveCli {max_tokens: DEFAULT_MAX_TOKENS,messages: vec![InputMessage {role: "user".to_string(),- content: vec![InputContentBlock::Text {- text: input.to_string(),- }],+ content: prompt_to_content_blocks(input, &env::current_dir()?)?,}],system: (!self.system_prompt.is_empty()).then(|| self.system_prompt.join("\n\n")),tools: None,@@ -2021,7 +2020,7 @@ impl ApiClient for AnthropicRuntimeClient {let message_request = MessageRequest {model: self.model.clone(),max_tokens: DEFAULT_MAX_TOKENS,- messages: convert_messages(&request.messages),+ messages: convert_messages(&request.messages)?,system: (!request.system_prompt.is_empty()).then(|| request.system_prompt.join("\n\n")),tools: self.enable_tools.then(|| {filter_tool_specs(self.allowed_tools.as_ref())@@ -2300,7 +2299,10 @@ fn tool_permission_specs() -> Vec<ToolSpec> {mvp_tool_specs()}-fn convert_messages(messages: &[ConversationMessage]) -> Vec<InputMessage> {+fn convert_messages(messages: &[ConversationMessage]) -> Result<Vec<InputMessage>, RuntimeError> {+ let cwd = env::current_dir().map_err(|error| {+ RuntimeError::new(format!("failed to resolve current directory: {error}"))+ })?;messages.iter().filter_map(|message| {@@ -2311,36 +2313,224 @@ fn convert_messages(messages: &[ConversationMessage]) -> Vec<InputMessage> {let content = message.blocks.iter()- .map(|block| match block {- ContentBlock::Text { text } => InputContentBlock::Text { text: text.clone() },- ContentBlock::ToolUse { id, name, input } => InputContentBlock::ToolUse {- id: id.clone(),- name: name.clone(),- input: serde_json::from_str(input)- .unwrap_or_else(|_| serde_json::json!({ "raw": input })),- },- ContentBlock::ToolResult {- tool_use_id,- output,- is_error,- ..- } => InputContentBlock::ToolResult {- tool_use_id: tool_use_id.clone(),- content: vec![ToolResultContentBlock::Text {- text: output.clone(),- }],- is_error: *is_error,- },- })- .collect::<Vec<_>>();- (!content.is_empty()).then(|| InputMessage {- role: role.to_string(),- content,- })+ .try_fold(Vec::new(), |mut acc, block| {+ match block {+ ContentBlock::Text { text } => {+ if message.role == MessageRole::User {+ acc.extend(+ prompt_to_content_blocks(text, &cwd)+ .map_err(RuntimeError::new)?,+ );+ } else {+ acc.push(InputContentBlock::Text { text: text.clone() });+ }+ }+ ContentBlock::ToolUse { id, name, input } => {+ acc.push(InputContentBlock::ToolUse {+ id: id.clone(),+ name: name.clone(),+ input: serde_json::from_str(input)+ .unwrap_or_else(|_| serde_json::json!({ "raw": input })),+ });+ }+ ContentBlock::ToolResult {+ tool_use_id,+ output,+ is_error,+ ..+ } => acc.push(InputContentBlock::ToolResult {+ tool_use_id: tool_use_id.clone(),+ content: vec![ToolResultContentBlock::Text {+ text: output.clone(),+ }],+ is_error: *is_error,+ }),+ }+ Ok::<_, RuntimeError>(acc)+ });+ match content {+ Ok(content) if !content.is_empty() => Some(Ok(InputMessage {+ role: role.to_string(),+ content,+ })),+ Ok(_) => None,+ Err(error) => Some(Err(error)),+ }}).collect()}+fn prompt_to_content_blocks(input: &str, cwd: &Path) -> Result<Vec<InputContentBlock>, String> {+ let mut blocks = Vec::new();+ let mut text_buffer = String::new();+ let mut chars = input.char_indices().peekable();++ while let Some((index, ch)) = chars.next() {+ if ch == '!' && input[index..].starts_with("![") {+ if let Some((alt_end, path_start, path_end)) = parse_markdown_image_ref(input, index) {+ let _ = alt_end;+ flush_text_block(&mut blocks, &mut text_buffer);+ let path = &input[path_start..path_end];+ blocks.push(load_image_block(path, cwd)?);+ while let Some((next_index, _)) = chars.peek() {+ if *next_index < path_end + 1 {+ let _ = chars.next();+ } else {+ break;+ }+ }+ continue;+ }+ }++ if ch == '@' && is_ref_boundary(input[..index].chars().next_back()) {+ let path_end = find_path_end(input, index + 1);+ if path_end > index + 1 {+ let candidate = &input[index + 1..path_end];+ if looks_like_image_ref(candidate, cwd) {+ flush_text_block(&mut blocks, &mut text_buffer);+ blocks.push(load_image_block(candidate, cwd)?);+ while let Some((next_index, _)) = chars.peek() {+ if *next_index < path_end {+ let _ = chars.next();+ } else {+ break;+ }+ }+ continue;+ }+ }+ }++ text_buffer.push(ch);+ }++ flush_text_block(&mut blocks, &mut text_buffer);+ if blocks.is_empty() {+ blocks.push(InputContentBlock::Text {+ text: input.to_string(),+ });+ }+ Ok(blocks)+}++fn parse_markdown_image_ref(input: &str, start: usize) -> Option<(usize, usize, usize)> {+ let after_bang = input.get(start + 2..)?;+ let alt_end_offset = after_bang.find("](")?;+ let path_start = start + 2 + alt_end_offset + 2;+ let remainder = input.get(path_start..)?;+ let path_end_offset = remainder.find(')')?;+ let path_end = path_start + path_end_offset;+ Some((start + 2 + alt_end_offset, path_start, path_end))+}++fn is_ref_boundary(ch: Option<char>) -> bool {+ ch.is_none_or(char::is_whitespace)+}++fn find_path_end(input: &str, start: usize) -> usize {+ input[start..]+ .char_indices()+ .find_map(|(offset, ch)| (ch.is_whitespace()).then_some(start + offset))+ .unwrap_or(input.len())+}++fn looks_like_image_ref(candidate: &str, cwd: &Path) -> bool {+ let resolved = resolve_prompt_path(candidate, cwd);+ media_type_for_path(Path::new(candidate)).is_some()+ || resolved.is_file()+ || candidate.contains(std::path::MAIN_SEPARATOR)+ || candidate.starts_with("./")+ || candidate.starts_with("../")+}++fn flush_text_block(blocks: &mut Vec<InputContentBlock>, text_buffer: &mut String) {+ if text_buffer.is_empty() {+ return;+ }+ blocks.push(InputContentBlock::Text {+ text: std::mem::take(text_buffer),+ });+}++fn load_image_block(path_ref: &str, cwd: &Path) -> Result<InputContentBlock, String> {+ let resolved = resolve_prompt_path(path_ref, cwd);+ let media_type = media_type_for_path(&resolved).ok_or_else(|| {+ format!(+ "unsupported image format for reference {IMAGE_REF_PREFIX}{path_ref}; supported: png, jpg, jpeg, gif, webp"+ )+ })?;+ let bytes = fs::read(&resolved).map_err(|error| {+ format!(+ "failed to read image reference {}: {error}",+ resolved.display()+ )+ })?;+ Ok(InputContentBlock::Image {+ source: ImageSource {+ kind: "base64".to_string(),+ media_type: media_type.to_string(),+ data: encode_base64(&bytes),+ },+ })+}++fn resolve_prompt_path(path_ref: &str, cwd: &Path) -> PathBuf {+ let path = Path::new(path_ref);+ if path.is_absolute() {+ path.to_path_buf()+ } else {+ cwd.join(path)+ }+}++fn media_type_for_path(path: &Path) -> Option<&'static str> {+ let extension = path.extension()?.to_str()?.to_ascii_lowercase();+ match extension.as_str() {+ "png" => Some("image/png"),+ "jpg" | "jpeg" => Some("image/jpeg"),+ "gif" => Some("image/gif"),+ "webp" => Some("image/webp"),+ _ => None,+ }+}++fn encode_base64(bytes: &[u8]) -> String {+ const TABLE: &[u8; 64] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";+ let mut output = String::new();+ let mut index = 0;+ while index + 3 <= bytes.len() {+ let block = (u32::from(bytes[index]) << 16)+ | (u32::from(bytes[index + 1]) << 8)+ | u32::from(bytes[index + 2]);+ output.push(TABLE[((block >> 18) & 0x3F) as usize] as char);+ output.push(TABLE[((block >> 12) & 0x3F) as usize] as char);+ output.push(TABLE[((block >> 6) & 0x3F) as usize] as char);+ output.push(TABLE[(block & 0x3F) as usize] as char);+ index += 3;+ }++ match bytes.len().saturating_sub(index) {+ 1 => {+ let block = u32::from(bytes[index]) << 16;+ output.push(TABLE[((block >> 18) & 0x3F) as usize] as char);+ output.push(TABLE[((block >> 12) & 0x3F) as usize] as char);+ output.push('=');+ output.push('=');+ }+ 2 => {+ let block = (u32::from(bytes[index]) << 16) | (u32::from(bytes[index + 1]) << 8);+ output.push(TABLE[((block >> 18) & 0x3F) as usize] as char);+ output.push(TABLE[((block >> 12) & 0x3F) as usize] as char);+ output.push(TABLE[((block >> 6) & 0x3F) as usize] as char);+ output.push('=');+ }+ _ => {}+ }++ output+}+fn print_help() {println!("rusty-claude-cli v{VERSION}");println!();@@ -2397,8 +2587,10 @@ mod tests {render_memory_report, render_repl_help, resume_supported_slash_commands, status_context,CliAction, CliOutputFormat, SlashCommand, StatusUsage, DEFAULT_MODEL,};+ use api::InputContentBlock;use runtime::{ContentBlock, ConversationMessage, MessageRole, PermissionMode};use std::path::{Path, PathBuf};+ use std::time::{SystemTime, UNIX_EPOCH};#[test]fn defaults_to_repl_when_no_args() {@@ -2797,7 +2989,7 @@ mod tests {fn status_context_reads_real_workspace_metadata() {let context = status_context(None).expect("status context should load");assert!(context.cwd.is_absolute());- assert_eq!(context.discovered_config_files, 3);+ assert!(context.discovered_config_files >= 3);assert!(context.loaded_config_files <= context.discovered_config_files);}@@ -2881,11 +3073,110 @@ mod tests {},];- let converted = super::convert_messages(&messages);+ let converted = super::convert_messages(&messages).expect("messages should convert");assert_eq!(converted.len(), 3);assert_eq!(converted[1].role, "assistant");assert_eq!(converted[2].role, "user");}++ #[test]+ fn prompt_to_content_blocks_keeps_text_only_prompt() {+ let blocks = super::prompt_to_content_blocks("hello world", Path::new("."))+ .expect("text prompt should parse");+ assert_eq!(+ blocks,+ vec![InputContentBlock::Text {+ text: "hello world".to_string()+ }]+ );+ }++ #[test]+ fn prompt_to_content_blocks_embeds_at_image_refs() {+ let temp = temp_fixture_dir("at-image-ref");+ let image_path = temp.join("sample.png");+ std::fs::write(&image_path, [1_u8, 2, 3]).expect("fixture write");+ let prompt = format!("describe @{} please", image_path.display());++ let blocks = super::prompt_to_content_blocks(&prompt, Path::new("."))+ .expect("image ref should parse");++ assert!(matches!(+ &blocks[0],+ InputContentBlock::Text { text } if text == "describe "+ ));+ assert!(matches!(+ &blocks[1],+ InputContentBlock::Image { source }+ if source.kind == "base64"+ && source.media_type == "image/png"+ && source.data == "AQID"+ ));+ assert!(matches!(+ &blocks[2],+ InputContentBlock::Text { text } if text == " please"+ ));+ }++ #[test]+ fn prompt_to_content_blocks_embeds_markdown_image_refs() {+ let temp = temp_fixture_dir("markdown-image-ref");+ let image_path = temp.join("sample.webp");+ std::fs::write(&image_path, [255_u8]).expect("fixture write");+ let prompt = format!("see  now", image_path.display());++ let blocks = super::prompt_to_content_blocks(&prompt, Path::new("."))+ .expect("markdown image ref should parse");++ assert!(matches!(+ &blocks[1],+ InputContentBlock::Image { source }+ if source.media_type == "image/webp" && source.data == "/w=="+ ));+ }++ #[test]+ fn prompt_to_content_blocks_rejects_unsupported_formats() {+ let temp = temp_fixture_dir("unsupported-image-ref");+ let image_path = temp.join("sample.bmp");+ std::fs::write(&image_path, [1_u8]).expect("fixture write");+ let prompt = format!("describe @{}", image_path.display());++ let error = super::prompt_to_content_blocks(&prompt, Path::new("."))+ .expect_err("unsupported image ref should fail");++ assert!(error.contains("unsupported image format"));+ }++ #[test]+ fn convert_messages_expands_user_text_image_refs() {+ let temp = temp_fixture_dir("convert-message-image-ref");+ let image_path = temp.join("sample.gif");+ std::fs::write(&image_path, [71_u8, 73, 70]).expect("fixture write");+ let messages = vec![ConversationMessage::user_text(format!(+ "inspect @{}",+ image_path.display()+ ))];++ let converted = super::convert_messages(&messages).expect("messages should convert");++ assert_eq!(converted.len(), 1);+ assert!(matches!(+ &converted[0].content[1],+ InputContentBlock::Image { source }+ if source.media_type == "image/gif" && source.data == "R0lG"+ ));+ }++ fn temp_fixture_dir(label: &str) -> PathBuf {+ let unique = SystemTime::now()+ .duration_since(UNIX_EPOCH)+ .expect("clock should advance")+ .as_nanos();+ let path = std::env::temp_dir().join(format!("rusty-claude-cli-{label}-{unique}"));+ std::fs::create_dir_all(&path).expect("temp dir should exist");+ path+ }#[test]fn repl_help_mentions_history_completion_and_multiline() {let help = render_repl_help();