Skip to content

Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors

Rust

test: complete Codex provider validation surfaces

Gabriel Scharbcfb215094ea3
diff --git a/crates/ai-memory-cli/src/cli.rs b/crates/ai-memory-cli/src/cli.rs
index 4142ad04..a44d0a9d 100644
--- a/crates/ai-memory-cli/src/cli.rs
+++ b/crates/ai-memory-cli/src/cli.rs
@@ -1909,6 +1909,9 @@ pub struct LlmTestArgs {
/// Prompt to send.
#[arg(long)]
pub prompt: String,
+ /// Request a small JSON-schema response instead of plain text.
+ #[arg(long)]
+ pub structured: bool,
/// Base URL override (required for openai-compat).
#[arg(long)]
pub base_url: Option<String>,
@@ -3348,6 +3351,28 @@ mod tests {
assert_eq!(args.prompt, "ping");
}
+ #[test]
+ fn llm_test_codex_structured_parses() {
+ let cli = Cli::try_parse_from([
+ "ai-memory",
+ "llm-test",
+ "--provider",
+ "codex",
+ "--model",
+ "gpt-5.6-luna",
+ "--prompt",
+ "ping",
+ "--structured",
+ ])
+ .unwrap();
+
+ let Command::LlmTest(args) = cli.command else {
+ panic!("expected llm-test command");
+ };
+ assert!(matches!(args.provider, LlmProviderChoice::Codex));
+ assert!(args.structured);
+ }
+
#[test]
fn completions_parses_every_supported_shell() {
for (arg, expected) in [
diff --git a/crates/ai-memory-cli/src/commands/llm_test.rs b/crates/ai-memory-cli/src/commands/llm_test.rs
index aaba3aa6..b43330c8 100644
--- a/crates/ai-memory-cli/src/commands/llm_test.rs
+++ b/crates/ai-memory-cli/src/commands/llm_test.rs
@@ -36,10 +36,24 @@ pub async fn run(config: &Config, args: LlmTestArgs) -> Result<()> {
model = client.model(),
"sending prompt",
);
- let resp = client
- .complete(representative_request(args.prompt))
- .await
- .context("calling provider")?;
+ let request = representative_request(args.prompt);
+ if args.structured {
+ let value = client
+ .complete_structured_raw(
+ request,
+ serde_json::json!({
+ "type": "object",
+ "properties": {"answer": {"type": "string"}},
+ "required": ["answer"]
+ }),
+ )
+ .await
+ .context("calling provider for structured output")?;
+ println!("--- model: {} ---", client.model());
+ println!("{}", serde_json::to_string_pretty(&value)?);
+ return Ok(());
+ }
+ let resp = client.complete(request).await.context("calling provider")?;
println!("--- model: {} ---", resp.model);
if let Some(u) = resp.usage {
diff --git a/docs/install.md b/docs/install.md
index 862e42a5..ae99cdb8 100644
--- a/docs/install.md
+++ b/docs/install.md
@@ -1730,6 +1730,7 @@ export AI_MEMORY_LLM_PROVIDER=codex
export AI_MEMORY_LLM_MODEL=gpt-5.6-luna
export AI_MEMORY_LLM_REASONING_EFFORT=medium
ai-memory llm-test --provider codex --model gpt-5.6-luna --prompt "Reply with OK"
+ai-memory llm-test --provider codex --model gpt-5.6-luna --structured --prompt "Return a short answer"
```
`AI_MEMORY_CODEX_EXECUTABLE` optionally selects another Codex binary. File
diff --git a/docs/llm-providers.md b/docs/llm-providers.md
index f8a8f9ec..800115e7 100644
--- a/docs/llm-providers.md
+++ b/docs/llm-providers.md
@@ -70,6 +70,7 @@ export AI_MEMORY_LLM_PROVIDER=codex
export AI_MEMORY_LLM_MODEL=gpt-5.6-luna
export AI_MEMORY_LLM_REASONING_EFFORT=medium
ai-memory llm-test --provider codex --model gpt-5.6-luna --prompt "Reply with OK"
+ai-memory llm-test --provider codex --model gpt-5.6-luna --structured --prompt "Return a short answer"
```
Codex credential storage mode `file` is supported. `auto` works only when its
diff --git a/evals/src/ab.rs b/evals/src/ab.rs
index 15511aa8..f2ee5cd1 100644
--- a/evals/src/ab.rs
+++ b/evals/src/ab.rs
@@ -63,7 +63,8 @@ pub struct AbArgs {
#[arg(long, default_value = "evals/runs")]
out: PathBuf,
- /// Baseline provider (`anthropic` / `openai` / `openai-compat` / `openai-oauth` / `copilot`).
+ /// Baseline provider (`anthropic` / `openai` / `openai-compat` /
+ /// `openai-oauth` / `codex` / `copilot`).
#[arg(long)]
baseline_provider: String,
@@ -83,7 +84,7 @@ pub struct AbArgs {
#[arg(long)]
baseline_api_key_env: Option<String>,
- /// Baseline OpenAI OAuth token file for `--baseline-provider openai-oauth`.
+ /// Baseline auth file for `openai-oauth` or `codex`.
#[arg(long)]
baseline_token_file: Option<PathBuf>,
@@ -107,7 +108,7 @@ pub struct AbArgs {
#[arg(long)]
candidate_api_key_env: Option<String>,
- /// Candidate OpenAI OAuth token file for `--candidate-provider openai-oauth`.
+ /// Candidate auth file for `openai-oauth` or `codex`.
#[arg(long)]
candidate_token_file: Option<PathBuf>,
}
@@ -282,10 +283,11 @@ fn resolve_provider(side: &str, args: &AbArgs) -> Result<ResolvedConfig> {
"openai" => ProviderChoice::OpenAi,
"openai-compat" | "openai_compat" => ProviderChoice::OpenAiCompat,
"openai-oauth" | "openai_oauth" => ProviderChoice::OpenAiOAuth,
+ "codex" => ProviderChoice::Codex,
"copilot" | "github-copilot" | "github_copilot" => ProviderChoice::Copilot,
other => {
bail!(
- "{side}: provider {other} not one of anthropic|openai|openai-compat|openai-oauth|copilot"
+ "{side}: provider {other} not one of anthropic|openai|openai-compat|openai-oauth|codex|copilot"
)
}
};
@@ -311,6 +313,12 @@ fn resolve_provider(side: &str, args: &AbArgs) -> Result<ResolvedConfig> {
anyhow::anyhow!("{side}: --{side}-token-file is required for openai-oauth")
})?)
}
+ AuthRequirement::CodexAuthFile => ProviderAuth::codex(
+ token_file.ok_or_else(|| {
+ anyhow::anyhow!("{side}: --{side}-token-file is required for codex")
+ })?,
+ "codex",
+ ),
AuthRequirement::CopilotToken => ProviderAuth::copilot(
token_file.ok_or_else(|| {
anyhow::anyhow!("{side}: --{side}-token-file is required for copilot")