Skip to content

Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors

Rust

M6: LLM provider trait + Anthropic + OpenAI + OpenAI-compat clients

AkitaOnRails67e1da71a3ed
diff --git a/Cargo.lock b/Cargo.lock
index d3db3e9b..7ffee8df 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -35,6 +35,7 @@ version = "0.1.0"
dependencies = [
"ai-memory-core",
"ai-memory-hooks",
+ "ai-memory-llm",
"ai-memory-mcp",
"ai-memory-store",
"ai-memory-wiki",
@@ -45,6 +46,7 @@ dependencies = [
"figment",
"flate2",
"rmcp",
+ "secrecy",
"serde",
"serde_json",
"sysinfo",
@@ -100,6 +102,16 @@ name = "ai-memory-llm"
version = "0.1.0"
dependencies = [
"ai-memory-core",
+ "anyhow",
+ "async-trait",
+ "reqwest",
+ "schemars",
+ "secrecy",
+ "serde",
+ "serde_json",
+ "thiserror 2.0.18",
+ "tokio",
+ "tracing",
]
[[package]]
@@ -366,6 +378,12 @@ version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801"
+[[package]]
+name = "cfg_aliases"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724"
+
[[package]]
name = "chacha20"
version = "0.10.0"
@@ -374,7 +392,7 @@ checksum = "6f8d983286843e49675a4b7a2d174efe136dc93a18d69130dd18198a6c167601"
dependencies = [
"cfg-if",
"cpufeatures 0.3.0",
- "rand_core",
+ "rand_core 0.10.1",
]
[[package]]
@@ -821,8 +839,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0"
dependencies = [
"cfg-if",
+ "js-sys",
"libc",
"wasi",
+ "wasm-bindgen",
]
[[package]]
@@ -832,9 +852,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd"
dependencies = [
"cfg-if",
+ "js-sys",
"libc",
"r-efi 5.3.0",
"wasip2",
+ "wasm-bindgen",
]
[[package]]
@@ -846,7 +868,7 @@ dependencies = [
"cfg-if",
"libc",
"r-efi 6.0.0",
- "rand_core",
+ "rand_core 0.10.1",
"wasip2",
"wasip3",
]
@@ -966,6 +988,23 @@ dependencies = [
"pin-project-lite",
"smallvec",
"tokio",
+ "want",
+]
+
+[[package]]
+name = "hyper-rustls"
+version = "0.27.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f"
+dependencies = [
+ "http",
+ "hyper",
+ "hyper-util",
+ "rustls",
+ "tokio",
+ "tokio-rustls",
+ "tower-service",
+ "webpki-roots",
]
[[package]]
@@ -974,13 +1013,21 @@ version = "0.1.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0"
dependencies = [
+ "base64",
"bytes",
+ "futures-channel",
+ "futures-util",
"http",
"http-body",
"hyper",
+ "ipnet",
+ "libc",
+ "percent-encoding",
"pin-project-lite",
+ "socket2",
"tokio",
"tower-service",
+ "tracing",
]
[[package]]
@@ -1160,6 +1207,12 @@ dependencies = [
"libc",
]
+[[package]]
+name = "ipnet"
+version = "2.12.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2"
+
[[package]]
name = "is_terminal_polyfill"
version = "1.70.2"
@@ -1344,6 +1397,12 @@ version = "0.4.29"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897"
+[[package]]
+name = "lru-slab"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154"
+
[[package]]
name = "matchers"
version = "0.2.0"
@@ -1584,6 +1643,15 @@ version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391"
+[[package]]
+name = "ppv-lite86"
+version = "0.2.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9"
+dependencies = [
+ "zerocopy",
+]
+
[[package]]
name = "prettyplease"
version = "0.2.37"
@@ -1616,6 +1684,61 @@ dependencies = [
"yansi",
]
+[[package]]
+name = "quinn"
+version = "0.11.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b9e20a958963c291dc322d98411f541009df2ced7b5a4f2bd52337638cfccf20"
+dependencies = [
+ "bytes",
+ "cfg_aliases",
+ "pin-project-lite",
+ "quinn-proto",
+ "quinn-udp",
+ "rustc-hash",
+ "rustls",
+ "socket2",
+ "thiserror 2.0.18",
+ "tokio",
+ "tracing",
+ "web-time",
+]
+
+[[package]]
+name = "quinn-proto"
+version = "0.11.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098"
+dependencies = [
+ "bytes",
+ "getrandom 0.3.4",
+ "lru-slab",
+ "rand 0.9.4",
+ "ring",
+ "rustc-hash",
+ "rustls",
+ "rustls-pki-types",
+ "slab",
+ "thiserror 2.0.18",
+ "tinyvec",
+ "tracing",
+ "web-time",
+]
+
+[[package]]
+name = "quinn-udp"
+version = "0.5.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "addec6a0dcad8a8d96a771f815f0eaf55f9d1805756410b39f5fa81332574cbd"
+dependencies = [
+ "cfg_aliases",
+ "libc",
+ "once_cell",
+ "socket2",
+ "tracing",
+ "windows-sys 0.60.2",
+]
+
[[package]]
name = "quote"
version = "1.0.45"
@@ -1637,6 +1760,16 @@ version = "6.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf"
+[[package]]
+name = "rand"
+version = "0.9.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea"
+dependencies = [
+ "rand_chacha",
+ "rand_core 0.9.5",
+]
+
[[package]]
name = "rand"
version = "0.10.1"
@@ -1645,7 +1778,26 @@ checksum = "d2e8e8bcc7961af1fdac401278c6a831614941f6164ee3bf4ce61b7edb162207"
dependencies = [
"chacha20",
"getrandom 0.4.2",
- "rand_core",
+ "rand_core 0.10.1",
+]
+
+[[package]]
+name = "rand_chacha"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb"
+dependencies = [
+ "ppv-lite86",
+ "rand_core 0.9.5",
+]
+
+[[package]]
+name = "rand_core"
+version = "0.9.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c"
+dependencies = [
+ "getrandom 0.3.4",
]
[[package]]
@@ -1787,6 +1939,58 @@ version = "0.8.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a"
+[[package]]
+name = "reqwest"
+version = "0.12.28"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147"
+dependencies = [
+ "base64",
+ "bytes",
+ "futures-core",
+ "http",
+ "http-body",
+ "http-body-util",
+ "hyper",
+ "hyper-rustls",
+ "hyper-util",
+ "js-sys",
+ "log",
+ "percent-encoding",
+ "pin-project-lite",
+ "quinn",
+ "rustls",
+ "rustls-pki-types",
+ "serde",
+ "serde_json",
+ "serde_urlencoded",
+ "sync_wrapper",
+ "tokio",
+ "tokio-rustls",
+ "tower",
+ "tower-http",
+ "tower-service",
+ "url",
+ "wasm-bindgen",
+ "wasm-bindgen-futures",
+ "web-sys",
+ "webpki-roots",
+]
+
+[[package]]
+name = "ring"
+version = "0.17.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7"
+dependencies = [
+ "cc",
+ "cfg-if",
+ "getrandom 0.2.17",
+ "libc",
+ "untrusted",
+ "windows-sys 0.52.0",
+]
+
[[package]]
name = "rmcp"
version = "1.7.0"
@@ -1803,7 +2007,7 @@ dependencies = [
"http-body-util",
"pastey",
"pin-project-lite",
- "rand",
+ "rand 0.10.1",
"rmcp-macros",
"schemars",
"serde",
@@ -1845,6 +2049,12 @@ dependencies = [
"smallvec",
]
+[[package]]
+name = "rustc-hash"
+version = "2.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe"
+
[[package]]
name = "rustix"
version = "1.1.4"
@@ -1858,6 +2068,41 @@ dependencies = [
"windows-sys 0.61.2",
]
+[[package]]
+name = "rustls"
+version = "0.23.40"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b"
+dependencies = [
+ "once_cell",
+ "ring",
+ "rustls-pki-types",
+ "rustls-webpki",
+ "subtle",
+ "zeroize",
+]
+
+[[package]]
+name = "rustls-pki-types"
+version = "1.14.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "30a7197ae7eb376e574fe940d068c30fe0462554a3ddbe4eca7838e049c937a9"
+dependencies = [
+ "web-time",
+ "zeroize",
+]
+
+[[package]]
+name = "rustls-webpki"
+version = "0.103.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e"
+dependencies = [
+ "ring",
+ "rustls-pki-types",
+ "untrusted",
+]
+
[[package]]
name = "rustversion"
version = "1.0.22"
@@ -1911,6 +2156,16 @@ version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
+[[package]]
+name = "secrecy"
+version = "0.10.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e891af845473308773346dc847b2c23ee78fe442e0472ac50e22a18a93d3ae5a"
+dependencies = [
+ "serde",
+ "zeroize",
+]
+
[[package]]
name = "semver"
version = "1.0.28"
@@ -2117,6 +2372,12 @@ version = "0.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
+[[package]]
+name = "subtle"
+version = "2.6.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292"
+
[[package]]
name = "symlink"
version = "0.1.0"
@@ -2139,6 +2400,9 @@ name = "sync_wrapper"
version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263"
+dependencies = [
+ "futures-core",
+]
[[package]]
name = "synstructure"
@@ -2279,6 +2543,21 @@ dependencies = [
"zerovec",
]
+[[package]]
+name = "tinyvec"
+version = "1.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3"
+dependencies = [
+ "tinyvec_macros",
+]
+
+[[package]]
+name = "tinyvec_macros"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
+
[[package]]
name = "tokio"
version = "1.52.3"
@@ -2307,6 +2586,16 @@ dependencies = [
"syn",
]
+[[package]]
+name = "tokio-rustls"
+version = "0.26.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61"
+dependencies = [
+ "rustls",
+ "tokio",
+]
+
[[package]]
name = "tokio-stream"
version = "0.1.18"
@@ -2389,6 +2678,24 @@ dependencies = [
"tracing",
]
+[[package]]
+name = "tower-http"
+version = "0.6.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840"
+dependencies = [
+ "bitflags",
+ "bytes",
+ "futures-util",
+ "http",
+ "http-body",
+ "pin-project-lite",
+ "tower",
+ "tower-layer",
+ "tower-service",
+ "url",
+]
+
[[package]]
name = "tower-layer"
version = "0.3.3"
@@ -2489,6 +2796,12 @@ dependencies = [
"tracing-serde",
]
+[[package]]
+name = "try-lock"
+version = "0.2.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b"
+
[[package]]
name = "typenum"
version = "1.20.0"
@@ -2522,6 +2835,12 @@ version = "0.2.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861"
+[[package]]
+name = "untrusted"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1"
+
[[package]]
name = "url"
version = "2.5.8"
@@ -2587,6 +2906,15 @@ dependencies = [
"winapi-util",
]
+[[package]]
+name = "want"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e"
+dependencies = [
+ "try-lock",
+]
+
[[package]]
name = "wasi"
version = "0.11.1+wasi-snapshot-preview1"
@@ -2624,6 +2952,16 @@ dependencies = [
"wasm-bindgen-shared",
]
+[[package]]
+name = "wasm-bindgen-futures"
+version = "0.4.71"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "96492d0d3ffba25305a7dc88720d250b1401d7edca02cc3bcd50633b424673b8"
+dependencies = [
+ "js-sys",
+ "wasm-bindgen",
+]
+
[[package]]
name = "wasm-bindgen-macro"
version = "0.2.121"
@@ -2690,6 +3028,35 @@ dependencies = [
"semver",
]
+[[package]]
+name = "web-sys"
+version = "0.3.98"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4b572dff8bcf38bad0fa19729c89bb5748b2b9b1d8be70cf90df697e3a8f32aa"
+dependencies = [
+ "js-sys",
+ "wasm-bindgen",
+]
+
+[[package]]
+name = "web-time"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb"
+dependencies = [
+ "js-sys",
+ "wasm-bindgen",
+]
+
+[[package]]
+name = "webpki-roots"
+version = "1.0.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "52f5ee44c96cf55f1b349600768e3ece3a8f26010c05265ab73f945bb1a2eb9d"
+dependencies = [
+ "rustls-pki-types",
+]
+
[[package]]
name = "winapi"
version = "0.3.9"
@@ -2789,6 +3156,15 @@ dependencies = [
"windows-targets 0.48.5",
]
+[[package]]
+name = "windows-sys"
+version = "0.52.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d"
+dependencies = [
+ "windows-targets 0.52.6",
+]
+
[[package]]
name = "windows-sys"
version = "0.60.2"
@@ -3182,6 +3558,12 @@ dependencies = [
"synstructure",
]
+[[package]]
+name = "zeroize"
+version = "1.8.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0"
+
[[package]]
name = "zerotrie"
version = "0.2.4"
diff --git a/Cargo.toml b/Cargo.toml
index ac7ea64a..27fd85c9 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -87,6 +87,10 @@ regex = "1"
# Wiki git versioning.
git2 = { version = "0.20", default-features = false, features = ["vendored-libgit2"] }
+# LLM provider HTTP.
+reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"] }
+async-trait = "0.1"
+
# Testing.
tempfile = "3"
diff --git a/crates/ai-memory-cli/Cargo.toml b/crates/ai-memory-cli/Cargo.toml
index d49398a1..d6aee2d9 100644
--- a/crates/ai-memory-cli/Cargo.toml
+++ b/crates/ai-memory-cli/Cargo.toml
@@ -15,9 +15,11 @@ path = "src/main.rs"
[dependencies]
ai-memory-core.workspace = true
ai-memory-hooks.workspace = true
+ai-memory-llm.workspace = true
ai-memory-mcp.workspace = true
ai-memory-store.workspace = true
ai-memory-wiki.workspace = true
+secrecy.workspace = true
anyhow.workspace = true
axum.workspace = true
clap.workspace = true
diff --git a/crates/ai-memory-cli/src/cli.rs b/crates/ai-memory-cli/src/cli.rs
index 8bd1bfce..45b7a0cc 100644
--- a/crates/ai-memory-cli/src/cli.rs
+++ b/crates/ai-memory-cli/src/cli.rs
@@ -49,6 +49,8 @@ pub enum Command {
InstallHooks(InstallHooksArgs),
/// Stage + commit the wiki tree under git.
Commit(CommitArgs),
+ /// Smoke-test an LLM provider by sending one prompt.
+ LlmTest(LlmTestArgs),
}
/// Arguments for `init`.
@@ -137,6 +139,37 @@ pub struct CommitArgs {
pub message: String,
}
+/// LLM provider for `llm-test`.
+#[derive(Debug, Clone, Copy, clap::ValueEnum)]
+pub enum LlmProviderChoice {
+ /// Anthropic Messages API.
+ Anthropic,
+ /// OpenAI Chat Completions.
+ Openai,
+ /// OpenAI-compatible local (Ollama, vLLM, LM Studio).
+ OpenaiCompat,
+}
+
+/// Arguments for `llm-test`.
+#[derive(Debug, Args)]
+pub struct LlmTestArgs {
+ /// Provider to test.
+ #[arg(long, value_enum)]
+ pub provider: LlmProviderChoice,
+ /// Model identifier (e.g. `claude-sonnet-4-7`, `gpt-4o-mini`, `llama3.1:8b`).
+ #[arg(long)]
+ pub model: String,
+ /// Prompt to send.
+ #[arg(long)]
+ pub prompt: String,
+ /// Base URL override (required for openai-compat).
+ #[arg(long, env = "LLM_BASE_URL")]
+ pub base_url: Option<String>,
+ /// Optional API key override (otherwise pulled from env).
+ #[arg(long, env = "LLM_API_KEY", hide_env_values = true)]
+ pub api_key: Option<String>,
+}
+
/// Arguments for `install-hooks`.
#[derive(Debug, Args)]
pub struct InstallHooksArgs {
diff --git a/crates/ai-memory-cli/src/commands/llm_test.rs b/crates/ai-memory-cli/src/commands/llm_test.rs
new file mode 100644
index 00000000..01669fd7
--- /dev/null
+++ b/crates/ai-memory-cli/src/commands/llm_test.rs
@@ -0,0 +1,69 @@
+//! `ai-memory llm-test` — smoke test an LLM provider end-to-end.
+
+use ai_memory_llm::{ChatRequest, ProviderChoice, ProviderConfig, build_provider};
+use anyhow::{Context, Result};
+use secrecy::SecretString;
+use tracing::info;
+
+use crate::cli::{LlmProviderChoice, LlmTestArgs};
+use crate::config::Config;
+
+/// Run the `llm-test` subcommand.
+///
+/// # Errors
+/// Returns an error if the provider cannot be constructed, the env
+/// lacks the required keys, or the HTTP call fails.
+pub async fn run(_config: &Config, args: LlmTestArgs) -> Result<()> {
+ let provider = ProviderChoice::from(args.provider);
+ let api_key = resolve_api_key(args.api_key, provider);
+ let config = ProviderConfig {
+ provider,
+ model: args.model,
+ api_key,
+ base_url: args.base_url,
+ };
+ let client = build_provider(config).context("building LLM provider")?;
+ info!(
+ provider = client.name(),
+ model = client.model(),
+ "sending prompt",
+ );
+ let resp = client
+ .complete(ChatRequest::user_prompt(args.prompt))
+ .await
+ .context("calling provider")?;
+
+ println!("--- model: {} ---", resp.model);
+ if let Some(u) = resp.usage {
+ println!(
+ "--- usage: in={} out={} ---",
+ u.input_tokens, u.output_tokens
+ );
+ }
+ println!("{}", resp.text);
+ Ok(())
+}
+
+fn resolve_api_key(explicit: Option<String>, provider: ProviderChoice) -> Option<SecretString> {
+ if let Some(s) = explicit
+ && !s.is_empty()
+ {
+ return Some(SecretString::from(s));
+ }
+ let env_name = match provider {
+ ProviderChoice::Anthropic => "ANTHROPIC_API_KEY",
+ ProviderChoice::OpenAi => "OPENAI_API_KEY",
+ ProviderChoice::OpenAiCompat => "LLM_API_KEY",
+ };
+ std::env::var(env_name).ok().map(SecretString::from)
+}
+
+impl From<LlmProviderChoice> for ProviderChoice {
+ fn from(value: LlmProviderChoice) -> Self {
+ match value {
+ LlmProviderChoice::Anthropic => Self::Anthropic,
+ LlmProviderChoice::Openai => Self::OpenAi,
+ LlmProviderChoice::OpenaiCompat => Self::OpenAiCompat,
+ }
+ }
+}
diff --git a/crates/ai-memory-cli/src/commands/mod.rs b/crates/ai-memory-cli/src/commands/mod.rs
index e1712cb9..8474ea69 100644
--- a/crates/ai-memory-cli/src/commands/mod.rs
+++ b/crates/ai-memory-cli/src/commands/mod.rs
@@ -4,6 +4,7 @@ pub mod backup;
pub mod commit;
pub mod init;
pub mod install_hooks;
+pub mod llm_test;
pub mod reset;
pub mod restore;
pub mod search;
diff --git a/crates/ai-memory-cli/src/main.rs b/crates/ai-memory-cli/src/main.rs
index 3eec040e..02d6d3ab 100644
--- a/crates/ai-memory-cli/src/main.rs
+++ b/crates/ai-memory-cli/src/main.rs
@@ -48,5 +48,6 @@ async fn main() -> Result<()> {
Command::Restore(args) => commands::restore::run(&config, args),
Command::InstallHooks(args) => commands::install_hooks::run(&config, args),
Command::Commit(args) => commands::commit::run(&config, args),
+ Command::LlmTest(args) => commands::llm_test::run(&config, args).await,
}
}
diff --git a/crates/ai-memory-llm/Cargo.toml b/crates/ai-memory-llm/Cargo.toml
index e11c2017..c9829897 100644
--- a/crates/ai-memory-llm/Cargo.toml
+++ b/crates/ai-memory-llm/Cargo.toml
@@ -10,6 +10,19 @@ description = "LLM provider trait with typed Anthropic, OpenAI and OpenAI-compat
[dependencies]
ai-memory-core.workspace = true
+anyhow.workspace = true
+thiserror.workspace = true
+serde.workspace = true
+serde_json.workspace = true
+schemars.workspace = true
+secrecy.workspace = true
+tokio.workspace = true
+tracing.workspace = true
+reqwest.workspace = true
+async-trait.workspace = true
+
+[dev-dependencies]
+tokio = { workspace = true }
[lints]
workspace = true
diff --git a/crates/ai-memory-llm/src/anthropic.rs b/crates/ai-memory-llm/src/anthropic.rs
new file mode 100644
index 00000000..b7f96257
--- /dev/null
+++ b/crates/ai-memory-llm/src/anthropic.rs
@@ -0,0 +1,243 @@
+//! Anthropic Messages API client.
+
+use std::time::Duration;
+
+use async_trait::async_trait;
+use secrecy::{ExposeSecret, SecretString};
+use serde::de::DeserializeOwned;
+use serde::{Deserialize, Serialize};
+use tracing::debug;
+
+use crate::error::{LlmError, LlmResult};
+use crate::provider::LlmProvider;
+use crate::types::{ChatRequest, ChatResponse, Role, Usage};
+
+/// Default Anthropic API base.
+pub const DEFAULT_BASE_URL: &str = "https://api.anthropic.com";
+/// Pinned Anthropic API version header.
+pub const ANTHROPIC_VERSION: &str = "2023-06-01";
+
+/// Anthropic Messages-API-backed provider.
+pub struct AnthropicProvider {
+ client: reqwest::Client,
+ api_key: SecretString,
+ base_url: String,
+ model: String,
+}
+
+impl AnthropicProvider {
+ /// Construct a provider given an API key and model id.
+ ///
+ /// # Errors
+ /// Returns a `reqwest::Error` if the underlying HTTP client cannot
+ /// be built.
+ pub fn new(api_key: SecretString, model: impl Into<String>) -> LlmResult<Self> {
+ let client = reqwest::Client::builder()
+ .timeout(Duration::from_secs(120))
+ .build()?;
+ Ok(Self {
+ client,
+ api_key,
+ base_url: DEFAULT_BASE_URL.to_string(),
+ model: model.into(),
+ })
+ }
+
+ /// Override the API base URL (mostly for tests against wiremock).
+ #[must_use]
+ pub fn with_base_url(mut self, url: impl Into<String>) -> Self {
+ self.base_url = url.into();
+ self
+ }
+}
+
+#[derive(Debug, Serialize)]
+struct AnthropicRequest<'a> {
+ model: &'a str,
+ max_tokens: u32,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ system: Option<&'a str>,
+ messages: Vec<AnthropicMsg<'a>>,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ temperature: Option<f32>,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ tools: Option<Vec<AnthropicTool>>,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ tool_choice: Option<AnthropicToolChoice>,
+}
+
+#[derive(Debug, Serialize)]
+struct AnthropicMsg<'a> {
+ role: &'a str,
+ content: &'a str,
+}
+
+#[derive(Debug, Serialize)]
+struct AnthropicTool {
+ name: String,
+ description: String,
+ input_schema: serde_json::Value,
+}
+
+#[derive(Debug, Serialize)]
+#[serde(tag = "type", rename_all = "snake_case")]
+enum AnthropicToolChoice {
+ Tool { name: String },
+}
+
+#[derive(Debug, Deserialize)]
+struct AnthropicResponse {
+ content: Vec<AnthropicContent>,
+ model: String,
+ #[serde(default)]
+ usage: Option<AnthropicUsage>,
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(tag = "type", rename_all = "snake_case")]
+enum AnthropicContent {
+ Text {
+ text: String,
+ },
+ ToolUse {
+ #[allow(dead_code)]
+ id: String,
+ #[allow(dead_code)]
+ name: String,
+ input: serde_json::Value,
+ },
+}
+
+#[derive(Debug, Deserialize)]
+struct AnthropicUsage {
+ input_tokens: u32,
+ output_tokens: u32,
+}
+
+#[async_trait]
+impl LlmProvider for AnthropicProvider {
+ fn name(&self) -> &'static str {
+ "anthropic"
+ }
+
+ fn model(&self) -> &str {
+ &self.model
+ }
+
+ async fn complete(&self, request: ChatRequest) -> LlmResult<ChatResponse> {
+ let messages: Vec<AnthropicMsg<'_>> = request
+ .messages
+ .iter()
+ .map(|m| AnthropicMsg {
+ role: match m.role {
+ Role::User => "user",
+ Role::Assistant => "assistant",
+ },
+ content: &m.content,
+ })
+ .collect();
+ let body = AnthropicRequest {
+ model: &self.model,
+ max_tokens: request.max_tokens,
+ system: request.system.as_deref(),
+ messages,
+ temperature: request.temperature,
+ tools: None,
+ tool_choice: None,
+ };
+ let response: AnthropicResponse = self.post(&body).await?;
+ let text = response
+ .content
+ .iter()
+ .filter_map(|c| match c {
+ AnthropicContent::Text { text } => Some(text.as_str()),
+ AnthropicContent::ToolUse { .. } => None,
+ })
+ .collect::<Vec<_>>()
+ .join("\n");
+ Ok(ChatResponse {
+ text,
+ usage: response.usage.map(|u| Usage {
+ input_tokens: u.input_tokens,
+ output_tokens: u.output_tokens,
+ }),
+ model: response.model,
+ })
+ }
+
+ async fn complete_structured_raw(
+ &self,
+ request: ChatRequest,
+ schema: serde_json::Value,
+ ) -> LlmResult<serde_json::Value> {
+ let tool = AnthropicTool {
+ name: "result".into(),
+ description: "Emit the structured result.".into(),
+ input_schema: schema,
+ };
+ let messages: Vec<AnthropicMsg<'_>> = request
+ .messages
+ .iter()
+ .map(|m| AnthropicMsg {
+ role: match m.role {
+ Role::User => "user",
+ Role::Assistant => "assistant",
+ },
+ content: &m.content,
+ })
+ .collect();
+ let body = AnthropicRequest {
+ model: &self.model,
+ max_tokens: request.max_tokens,
+ system: request.system.as_deref(),
+ messages,
+ temperature: request.temperature,
+ tools: Some(vec![tool]),
+ tool_choice: Some(AnthropicToolChoice::Tool {
+ name: "result".into(),
+ }),
+ };
+ let response: AnthropicResponse = self.post(&body).await?;
+ for c in response.content {
+ if let AnthropicContent::ToolUse { input, .. } = c {
+ return Ok(input);
+ }
+ }
+ Err(LlmError::UnexpectedShape(
+ "anthropic response had no tool_use block".into(),
+ ))
+ }
+}
+
+impl AnthropicProvider {
+ async fn post<B: Serialize, R: DeserializeOwned>(&self, body: &B) -> LlmResult<R> {
+ let url = format!("{}/v1/messages", self.base_url.trim_end_matches('/'));
+ debug!(url, "POST anthropic");
+ let resp = self
+ .client
+ .post(&url)
+ .header("x-api-key", self.api_key.expose_secret())
+ .header("anthropic-version", ANTHROPIC_VERSION)
+ .header("content-type", "application/json")
+ .json(body)
+ .send()
+ .await?;
+ let status = resp.status();
+ if !status.is_success() {
+ let body = resp.text().await.unwrap_or_default();
+ return Err(LlmError::Provider {
+ status: status.as_u16(),
+ body: truncate(&body, 1024),
+ });
+ }
+ resp.json::<R>().await.map_err(LlmError::from)
+ }
+}
+
+fn truncate(s: &str, n: usize) -> String {
+ if s.len() <= n {
+ s.to_string()
+ } else {
+ format!("{}…", &s[..n])
+ }
+}
diff --git a/crates/ai-memory-llm/src/error.rs b/crates/ai-memory-llm/src/error.rs
new file mode 100644
index 00000000..1780ced3
--- /dev/null
+++ b/crates/ai-memory-llm/src/error.rs
@@ -0,0 +1,51 @@
+//! LLM error type.
+
+use thiserror::Error;
+
+/// Result alias used throughout the LLM crate.
+pub type LlmResult<T> = Result<T, LlmError>;
+
+/// Errors raised by LLM providers.
+#[derive(Debug, Error)]
+#[non_exhaustive]
+pub enum LlmError {
+ /// Underlying HTTP failure.
+ #[error("http: {0}")]
+ Http(#[from] reqwest::Error),
+
+ /// Provider returned a non-2xx status.
+ #[error("provider error {status}: {body}")]
+ Provider {
+ /// HTTP status code.
+ status: u16,
+ /// Response body (truncated).
+ body: String,
+ },
+
+ /// JSON (de)serialization failure.
+ #[error("serde: {0}")]
+ Serde(String),
+
+ /// Provider gave a response with unexpected shape (e.g. no tool
+ /// use block where structured output was requested).
+ #[error("unexpected response shape: {0}")]
+ UnexpectedShape(String),
+
+ /// Configured provider lacks the env var we need.
+ #[error("provider not configured: {0}")]
+ NotConfigured(String),
+
+ /// JSON schema for structured output could not be derived.
+ #[error("schema: {0}")]
+ Schema(String),
+
+ /// Operation exhausted its retry budget.
+ #[error("retries exhausted: {0}")]
+ RetriesExhausted(String),
+}
+
+impl From<serde_json::Error> for LlmError {
+ fn from(value: serde_json::Error) -> Self {
+ Self::Serde(value.to_string())
+ }
+}
diff --git a/crates/ai-memory-llm/src/factory.rs b/crates/ai-memory-llm/src/factory.rs
new file mode 100644
index 00000000..480b27a6
--- /dev/null
+++ b/crates/ai-memory-llm/src/factory.rs
@@ -0,0 +1,70 @@
+//! Provider factory.
+//!
+//! Maps the user-visible `ProviderChoice` + env config into a
+//! concrete `Arc<dyn LlmProvider>`.
+
+use std::sync::Arc;
+
+use secrecy::SecretString;
+
+use crate::AnthropicProvider;
+use crate::OpenAiCompatProvider;
+use crate::OpenAiProvider;
+use crate::error::{LlmError, LlmResult};
+use crate::provider::LlmProvider;
+
+/// Three providers ship in v1.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum ProviderChoice {
+ /// Anthropic Messages API.
+ Anthropic,
+ /// OpenAI Chat Completions.
+ OpenAi,
+ /// OpenAI-compatible (Ollama / vLLM / LM Studio).
+ OpenAiCompat,
+}
+
+/// All settings needed to construct one of the three providers.
+#[derive(Debug, Clone)]
+pub struct ProviderConfig {
+ /// Provider selection.
+ pub provider: ProviderChoice,
+ /// Model id (`claude-opus-4-7`, `gpt-4o-mini`, `llama3.1:8b`, …).
+ pub model: String,
+ /// API key. Required for Anthropic + OpenAI; optional for compat.
+ pub api_key: Option<SecretString>,
+ /// Base URL override (required for OpenAI-compat).
+ pub base_url: Option<String>,
+}
+
+/// Construct an `Arc<dyn LlmProvider>` matching the config.
+///
+/// # Errors
+/// Returns [`LlmError::NotConfigured`] if a required env value (API
+/// key, base URL) is missing.
+pub fn build_provider(config: ProviderConfig) -> LlmResult<Arc<dyn LlmProvider>> {
+ match config.provider {
+ ProviderChoice::Anthropic => {
+ let key = config
+ .api_key
+ .ok_or_else(|| LlmError::NotConfigured("ANTHROPIC_API_KEY".into()))?;
+ Ok(Arc::new(AnthropicProvider::new(key, config.model)?))
+ }
+ ProviderChoice::OpenAi => {
+ let key = config
+ .api_key
+ .ok_or_else(|| LlmError::NotConfigured("OPENAI_API_KEY".into()))?;
+ Ok(Arc::new(OpenAiProvider::new(key, config.model)?))
+ }
+ ProviderChoice::OpenAiCompat => {
+ let base = config
+ .base_url
+ .ok_or_else(|| LlmError::NotConfigured("LLM_BASE_URL".into()))?;
+ Ok(Arc::new(OpenAiCompatProvider::new(
+ base,
+ config.api_key,
+ config.model,
+ )?))
+ }
+ }
+}
diff --git a/crates/ai-memory-llm/src/lib.rs b/crates/ai-memory-llm/src/lib.rs
index fabcbeb5..5f528ec6 100644
--- a/crates/ai-memory-llm/src/lib.rs
+++ b/crates/ai-memory-llm/src/lib.rs
@@ -1,6 +1,36 @@
-//! LLM provider abstraction.
+//! LLM provider abstraction for ai-memory.
//!
-//! Native typed HTTP clients per provider, never a generic gateway. Lesson
-//! from cognee #2840: `LiteLLM` + `instructor` silently drop unknown kwargs
-//! and the wrapper layer ends up papering over wire-protocol drift forever.
-//! Implementation lands in milestone M6.
+//! Three providers ship in v1, each with a *native, typed*
+//! `reqwest`-based client — never a generic gateway. The cognee
+//! issue tracker showed that LiteLLM + Instructor silently drop
+//! unknown kwargs, which makes the wrapper layer drift away from
+//! the provider's wire protocol over time (#2840, #2608, #2782).
+//! Our clients deserialise into named structs that `serde` rejects
+//! on unknown fields, surfacing breakage immediately.
+//!
+//! Three structured-output strategies:
+//!
+//! * **Anthropic**: `tools[0]` is set to a single tool whose input
+//! schema we want filled, with `tool_choice = "tool"`. The
+//! model's `tool_use` content block is the structured payload.
+//! * **OpenAI**: `response_format = { type: "json_schema", strict: true }`.
+//! * **OpenAI-compat** (Ollama, vLLM, LM Studio): we ask for
+//! `response_format: { type: "json_object" }` when supported,
+//! otherwise parse the first balanced `{…}` from the text body.
+//! No tenacity-style 8-128s backoff (cognee #2840 lesson).
+
+pub mod anthropic;
+pub mod error;
+pub mod factory;
+pub mod openai;
+pub mod openai_compat;
+pub mod provider;
+pub mod types;
+
+pub use anthropic::AnthropicProvider;
+pub use error::{LlmError, LlmResult};
+pub use factory::{ProviderChoice, ProviderConfig, build_provider};
+pub use openai::OpenAiProvider;
+pub use openai_compat::OpenAiCompatProvider;
+pub use provider::{LlmProvider, complete_structured};
+pub use types::{ChatMessage, ChatRequest, ChatResponse, Role, Usage};
diff --git a/crates/ai-memory-llm/src/openai.rs b/crates/ai-memory-llm/src/openai.rs
new file mode 100644
index 00000000..2c05569e
--- /dev/null
+++ b/crates/ai-memory-llm/src/openai.rs
@@ -0,0 +1,226 @@
+//! OpenAI Chat Completions client (with `response_format` JSON schema for
+//! structured output).
+
+use std::time::Duration;
+
+use async_trait::async_trait;
+use secrecy::{ExposeSecret, SecretString};
+use serde::{Deserialize, Serialize};
+use tracing::debug;
+
+use crate::error::{LlmError, LlmResult};
+use crate::provider::LlmProvider;
+use crate::types::{ChatRequest, ChatResponse, Role, Usage};
+
+/// Default OpenAI API base.
+pub const DEFAULT_BASE_URL: &str = "https://api.openai.com";
+
+/// OpenAI Chat Completions-backed provider.
+pub struct OpenAiProvider {
+ client: reqwest::Client,
+ api_key: SecretString,
+ base_url: String,
+ model: String,
+}
+
+impl OpenAiProvider {
+ /// Construct a provider given an API key + model id.
+ ///
+ /// # Errors
+ /// Returns a `reqwest::Error` if the HTTP client cannot be built.
+ pub fn new(api_key: SecretString, model: impl Into<String>) -> LlmResult<Self> {
+ let client = reqwest::Client::builder()
+ .timeout(Duration::from_secs(120))
+ .build()?;
+ Ok(Self {
+ client,
+ api_key,
+ base_url: DEFAULT_BASE_URL.to_string(),
+ model: model.into(),
+ })
+ }
+
+ /// Override the API base URL (tests; or pointing at an
+ /// OpenAI-compatible mirror).
+ #[must_use]
+ pub fn with_base_url(mut self, url: impl Into<String>) -> Self {
+ self.base_url = url.into();
+ self
+ }
+}
+
+#[derive(Debug, Serialize)]
+struct OpenAiRequest<'a> {
+ model: &'a str,
+ messages: Vec<OpenAiMsg<'a>>,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ max_tokens: Option<u32>,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ temperature: Option<f32>,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ response_format: Option<OpenAiResponseFormat>,
+}
+
+#[derive(Debug, Serialize)]
+struct OpenAiMsg<'a> {
+ role: &'a str,
+ content: &'a str,
+}
+
+#[derive(Debug, Serialize)]
+#[serde(tag = "type", rename_all = "snake_case")]
+enum OpenAiResponseFormat {
+ JsonSchema { json_schema: OpenAiJsonSchema },
+}
+
+#[derive(Debug, Serialize)]
+struct OpenAiJsonSchema {
+ name: String,
+ schema: serde_json::Value,
+ strict: bool,
+}
+
+#[derive(Debug, Deserialize)]
+struct OpenAiResponse {
+ choices: Vec<OpenAiChoice>,
+ model: String,
+ #[serde(default)]
+ usage: Option<OpenAiUsage>,
+}
+
+#[derive(Debug, Deserialize)]
+struct OpenAiChoice {
+ message: OpenAiMessageResponse,
+}
+
+#[derive(Debug, Deserialize)]
+struct OpenAiMessageResponse {
+ content: Option<String>,
+}
+
+#[derive(Debug, Deserialize)]
+struct OpenAiUsage {
+ prompt_tokens: u32,
+ completion_tokens: u32,
+}
+
+#[async_trait]
+impl LlmProvider for OpenAiProvider {
+ fn name(&self) -> &'static str {
+ "openai"
+ }
+
+ fn model(&self) -> &str {
+ &self.model
+ }
+
+ async fn complete(&self, request: ChatRequest) -> LlmResult<ChatResponse> {
+ let response = self.post(&self.build_request(&request, None)).await?;
+ Ok(self.to_chat_response(response))
+ }
+
+ async fn complete_structured_raw(
+ &self,
+ request: ChatRequest,
+ schema: serde_json::Value,
+ ) -> LlmResult<serde_json::Value> {
+ let response_format = OpenAiResponseFormat::JsonSchema {
+ json_schema: OpenAiJsonSchema {
+ name: "Result".into(),
+ schema,
+ strict: true,
+ },
+ };
+ let response = self
+ .post(&self.build_request(&request, Some(response_format)))
+ .await?;
+ let text = response
+ .choices
+ .first()
+ .and_then(|c| c.message.content.as_deref())
+ .unwrap_or("");
+ serde_json::from_str::<serde_json::Value>(text).map_err(LlmError::from)
+ }
+}
+
+impl OpenAiProvider {
+ fn build_request<'a>(
+ &'a self,
+ request: &'a ChatRequest,
+ response_format: Option<OpenAiResponseFormat>,
+ ) -> OpenAiRequest<'a> {
+ let mut messages: Vec<OpenAiMsg<'a>> = Vec::new();
+ if let Some(sys) = request.system.as_deref() {
+ messages.push(OpenAiMsg {
+ role: "system",
+ content: sys,
+ });
+ }
+ for m in &request.messages {
+ messages.push(OpenAiMsg {
+ role: match m.role {
+ Role::User => "user",
+ Role::Assistant => "assistant",
+ },
+ content: &m.content,
+ });
+ }
+ OpenAiRequest {
+ model: &self.model,
+ messages,
+ max_tokens: Some(request.max_tokens),
+ temperature: request.temperature,
+ response_format,
+ }
+ }
+
+ fn to_chat_response(&self, response: OpenAiResponse) -> ChatResponse {
+ let text = response
+ .choices
+ .into_iter()
+ .next()
+ .and_then(|c| c.message.content)
+ .unwrap_or_default();
+ ChatResponse {
+ text,
+ usage: response.usage.map(|u| Usage {
+ input_tokens: u.prompt_tokens,
+ output_tokens: u.completion_tokens,
+ }),
+ model: response.model,
+ }
+ }
+
+ async fn post<B: Serialize>(&self, body: &B) -> LlmResult<OpenAiResponse> {
+ let url = format!(
+ "{}/v1/chat/completions",
+ self.base_url.trim_end_matches('/')
+ );
+ debug!(url, "POST openai");
+ let resp = self
+ .client
+ .post(&url)
+ .bearer_auth(self.api_key.expose_secret())
+ .header("content-type", "application/json")
+ .json(body)
+ .send()
+ .await?;
+ let status = resp.status();
+ if !status.is_success() {
+ let body = resp.text().await.unwrap_or_default();
+ return Err(LlmError::Provider {
+ status: status.as_u16(),
+ body: truncate(&body, 1024),
+ });
+ }
+ resp.json::<OpenAiResponse>().await.map_err(LlmError::from)
+ }
+}
+
+fn truncate(s: &str, n: usize) -> String {
+ if s.len() <= n {
+ s.to_string()
+ } else {
+ format!("{}…", &s[..n])
+ }
+}
diff --git a/crates/ai-memory-llm/src/openai_compat.rs b/crates/ai-memory-llm/src/openai_compat.rs
new file mode 100644
index 00000000..379e6dfa
--- /dev/null
+++ b/crates/ai-memory-llm/src/openai_compat.rs
@@ -0,0 +1,123 @@
+//! OpenAI-compatible client (Ollama, vLLM, LM Studio, llama.cpp).
+//!
+//! Uses the same wire format as [`crate::openai::OpenAiProvider`] but
+//! with a configurable base URL (and no key required for most local
+//! deployments). Structured output falls back to "parse first JSON
+//! object out of the text" because most local engines lack reliable
+//! `response_format` honour.
+
+use async_trait::async_trait;
+use secrecy::SecretString;
+use tracing::debug;
+
+use crate::error::{LlmError, LlmResult};
+use crate::openai::OpenAiProvider;
+use crate::provider::LlmProvider;
+use crate::types::{ChatRequest, ChatResponse};
+
+/// OpenAI-compatible provider, parameterised by base URL.
+pub struct OpenAiCompatProvider {
+ inner: OpenAiProvider,
+ name_tag: &'static str,
+}
+
+impl OpenAiCompatProvider {
+ /// Construct a provider pointed at `base_url` (`LLM_BASE_URL` or
+ /// `OLLAMA_HOST`). API key is optional; many local engines accept
+ /// any non-empty string.
+ ///
+ /// # Errors
+ /// Returns a `reqwest::Error` if the HTTP client cannot be built.
+ pub fn new(
+ base_url: impl Into<String>,
+ api_key: Option<SecretString>,
+ model: impl Into<String>,
+ ) -> LlmResult<Self> {
+ let key = api_key.unwrap_or_else(|| SecretString::from("dummy"));
+ let inner = OpenAiProvider::new(key, model)?.with_base_url(base_url);
+ Ok(Self {
+ inner,
+ name_tag: "openai-compat",
+ })
+ }
+}
+
+#[async_trait]
+impl LlmProvider for OpenAiCompatProvider {
+ fn name(&self) -> &'static str {
+ self.name_tag
+ }
+
+ fn model(&self) -> &str {
+ self.inner.model()
+ }
+
+ async fn complete(&self, request: ChatRequest) -> LlmResult<ChatResponse> {
+ self.inner.complete(request).await
+ }
+
+ async fn complete_structured_raw(
+ &self,
+ request: ChatRequest,
+ _schema: serde_json::Value,
+ ) -> LlmResult<serde_json::Value> {
+ // Most local engines don't honour `response_format`. We
+ // ask the model to emit a JSON object and fall back to
+ // extracting the first balanced `{…}` from the text.
+ let res = self.inner.complete(request).await?;
+ match serde_json::from_str::<serde_json::Value>(&res.text) {
+ Ok(v) if v.is_object() => Ok(v),
+ _ => {
+ let Some(slice) = first_json_object(&res.text) else {
+ debug!(text = %truncate(&res.text, 200), "no balanced JSON object found");
+ return Err(LlmError::UnexpectedShape(
+ "openai-compat response did not contain a JSON object".into(),
+ ));
+ };
+ serde_json::from_str::<serde_json::Value>(slice).map_err(LlmError::from)
+ }
+ }
+ }
+}
+
+fn first_json_object(s: &str) -> Option<&str> {
+ let start = s.find('{')?;
+ let mut depth = 0_i32;
+ let bytes = s.as_bytes();
+ for (i, &b) in bytes[start..].iter().enumerate() {
+ match b {
+ b'{' => depth += 1,
+ b'}' => {
+ depth -= 1;
+ if depth == 0 {
+ return Some(&s[start..=start + i]);
+ }
+ }
+ _ => {}
+ }
+ }
+ None
+}
+
+fn truncate(s: &str, n: usize) -> String {
+ if s.len() <= n {
+ s.to_string()
+ } else {
+ format!("{}…", &s[..n])
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn first_json_object_finds_balanced_object() {
+ assert_eq!(first_json_object("noise {\"k\":1} more"), Some("{\"k\":1}"));
+ assert_eq!(
+ first_json_object("text {\"a\":{\"b\":2}} trailing"),
+ Some("{\"a\":{\"b\":2}}"),
+ );
+ assert_eq!(first_json_object("no json here"), None);
+ }
+}
diff --git a/crates/ai-memory-llm/src/provider.rs b/crates/ai-memory-llm/src/provider.rs
new file mode 100644
index 00000000..e8d5b38a
--- /dev/null
+++ b/crates/ai-memory-llm/src/provider.rs
@@ -0,0 +1,61 @@
+//! [`LlmProvider`] trait — the only abstraction the rest of the
+//! workspace depends on.
+
+use async_trait::async_trait;
+use schemars::JsonSchema;
+use serde::de::DeserializeOwned;
+
+use crate::error::LlmResult;
+use crate::types::{ChatRequest, ChatResponse};
+
+/// Provider-agnostic chat-completion + structured-output API.
+///
+/// Implementations must be `Send + Sync` so the MCP server / hook
+/// router can stash an `Arc<dyn LlmProvider>` and call it from any
+/// tokio task. Errors map onto [`LlmError`](crate::LlmError).
+///
+/// To keep the trait dyn-compatible, the structured-output API takes
+/// a raw `serde_json::Value` schema in and yields a raw value out.
+/// Use the [`complete_structured`] free function for typed payloads.
+#[async_trait]
+pub trait LlmProvider: Send + Sync {
+ /// Friendly provider name (e.g. `"anthropic"`).
+ fn name(&self) -> &'static str;
+
+ /// The model identifier this provider will hit.
+ fn model(&self) -> &str;
+
+ /// Plain text completion.
+ async fn complete(&self, request: ChatRequest) -> LlmResult<ChatResponse>;
+
+ /// JSON-schema-constrained completion. Returns the unparsed
+ /// JSON the provider produced.
+ ///
+ /// Implementations *must* reject responses with unknown fields at
+ /// deserialisation time (cognee #2840 lesson: silent kwarg-drop in
+ /// the wrapper is the #1 source of provider-drift bugs).
+ async fn complete_structured_raw(
+ &self,
+ request: ChatRequest,
+ schema: serde_json::Value,
+ ) -> LlmResult<serde_json::Value>;
+}
+
+/// Typed wrapper around [`LlmProvider::complete_structured_raw`].
+///
+/// Derives the JSON schema from `T` via `schemars`, calls the provider,
+/// then deserialises the response into `T`.
+///
+/// # Errors
+/// Propagates any HTTP, schema, or deserialisation error.
+pub async fn complete_structured<T>(
+ provider: &(dyn LlmProvider + 'static),
+ request: ChatRequest,
+) -> LlmResult<T>
+where
+ T: DeserializeOwned + JsonSchema + Send + 'static,
+{
+ let schema = serde_json::to_value(schemars::schema_for!(T))?;
+ let value = provider.complete_structured_raw(request, schema).await?;
+ serde_json::from_value::<T>(value).map_err(crate::LlmError::from)
+}
diff --git a/crates/ai-memory-llm/src/types.rs b/crates/ai-memory-llm/src/types.rs
new file mode 100644
index 00000000..e6576453
--- /dev/null
+++ b/crates/ai-memory-llm/src/types.rs
@@ -0,0 +1,102 @@
+//! Provider-neutral request/response types.
+
+use serde::{Deserialize, Serialize};
+
+/// Message role in a chat turn.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "lowercase")]
+pub enum Role {
+ /// Message from the user.
+ User,
+ /// Message from the assistant (the model).
+ Assistant,
+}
+
+/// One message in the chat history.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct ChatMessage {
+ /// Role.
+ pub role: Role,
+ /// Message text.
+ pub content: String,
+}
+
+impl ChatMessage {
+ /// Convenience constructor.
+ #[must_use]
+ pub fn user(content: impl Into<String>) -> Self {
+ Self {
+ role: Role::User,
+ content: content.into(),
+ }
+ }
+
+ /// Convenience constructor.
+ #[must_use]
+ pub fn assistant(content: impl Into<String>) -> Self {
+ Self {
+ role: Role::Assistant,
+ content: content.into(),
+ }
+ }
+}
+
+/// Provider-neutral chat completion request.
+#[derive(Debug, Clone)]
+pub struct ChatRequest {
+ /// Optional system prompt.
+ pub system: Option<String>,
+ /// User + assistant messages.
+ pub messages: Vec<ChatMessage>,
+ /// Maximum tokens to generate.
+ pub max_tokens: u32,
+ /// Sampling temperature (0.0..2.0). `None` to defer to provider default.
+ pub temperature: Option<f32>,
+}
+
+impl ChatRequest {
+ /// Build a request from a single user prompt.
+ #[must_use]
+ pub fn user_prompt(prompt: impl Into<String>) -> Self {
+ Self {
+ system: None,
+ messages: vec![ChatMessage::user(prompt)],
+ max_tokens: 1024,
+ temperature: None,
+ }
+ }
+
+ /// Attach a system prompt to the request.
+ #[must_use]
+ pub fn with_system(mut self, system: impl Into<String>) -> Self {
+ self.system = Some(system.into());
+ self
+ }
+
+ /// Override the max-tokens cap.
+ #[must_use]
+ pub const fn with_max_tokens(mut self, n: u32) -> Self {
+ self.max_tokens = n;
+ self
+ }
+}
+
+/// Token usage report. Providers return at least input/output counts.
+#[derive(Debug, Clone, Copy, Default, Serialize, Deserialize)]
+pub struct Usage {
+ /// Tokens consumed by the prompt.
+ pub input_tokens: u32,
+ /// Tokens generated.
+ pub output_tokens: u32,
+}
+
+/// Provider-neutral response.
+#[derive(Debug, Clone, Serialize)]
+pub struct ChatResponse {
+ /// Model-assistant text output.
+ pub text: String,
+ /// Token usage, if reported.
+ pub usage: Option<Usage>,
+ /// Model identifier echoed by the provider.
+ pub model: String,
+}