Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

Add Local AI Store category and Ollama, MLX-LM, vLLM harnesses (#139)

Deedf00dbc59bb9
diff --git a/desktop/assets/engine-icons/mlx.png b/desktop/assets/engine-icons/mlx.png
new file mode 100644
index 000000000..384b343dd
Binary files /dev/null and b/desktop/assets/engine-icons/mlx.png differ
diff --git a/desktop/assets/engine-icons/ollama.png b/desktop/assets/engine-icons/ollama.png
new file mode 100644
index 000000000..130cb9ba4
Binary files /dev/null and b/desktop/assets/engine-icons/ollama.png differ
diff --git a/desktop/assets/engine-icons/vllm.png b/desktop/assets/engine-icons/vllm.png
new file mode 100644
index 000000000..7aaf17487
Binary files /dev/null and b/desktop/assets/engine-icons/vllm.png differ
diff --git a/desktop/lib/store/store_editorial.dart b/desktop/lib/store/store_editorial.dart
index 88f4f3bc8..e6e3c139f 100644
--- a/desktop/lib/store/store_editorial.dart
+++ b/desktop/lib/store/store_editorial.dart
@@ -7,11 +7,16 @@ const storeCategoryDomains = <String, Set<String>>{
'Media': {'Documents', 'Slides', 'Video', 'Math animation', 'Music'},
'Science': {'Chemistry', 'Notebooks', 'Simulation'},
'Games': {'Games'},
+ 'Local AI': {'Local AI'},
'Code': {'Code', 'Compute'},
};
String storeCategoryFor(DshEntry entry) {
if (entry.isEngine) return 'Code';
+ // Legacy Grid/Ollama packages used Compute before Local AI existed.
+ if (const {'autonomous/autonomous-grid', 'local/ollama'}.contains(entry.id)) {
+ return 'Local AI';
+ }
final domain = entry.category?.trim().toLowerCase();
for (final category in storeCategoryDomains.entries) {
if (category.key.toLowerCase() == domain ||
diff --git a/desktop/lib/store/store_screen.dart b/desktop/lib/store/store_screen.dart
index 45bec8098..0fff82fea 100644
--- a/desktop/lib/store/store_screen.dart
+++ b/desktop/lib/store/store_screen.dart
@@ -534,6 +534,7 @@ IconData _categoryIcon(String category) => switch (category) {
'Engineering' => LucideIcons.cpu300,
'Media' => LucideIcons.film300,
'Science' => LucideIcons.flaskConical300,
+ 'Local AI' => LucideIcons.brainCircuit300,
'Code' => LucideIcons.terminal300,
'Games' => LucideIcons.gamepad2300,
_ => LucideIcons.shapes300,
diff --git a/desktop/lib/widgets/engine_identity.dart b/desktop/lib/widgets/engine_identity.dart
index 11488d186..4125a7133 100644
--- a/desktop/lib/widgets/engine_identity.dart
+++ b/desktop/lib/widgets/engine_identity.dart
@@ -222,6 +222,75 @@ const _engines = <String, EngineIdentity>{
/// own. It is only a face. A harness absent here still draws — the daemon
/// sends its name, and [engineIdentity] falls back to an initial.
const _harnesses = <String, EngineIdentity>{
+ 'autonomous/vllm': EngineIdentity(
+ id: 'autonomous/vllm',
+ tagline:
+ 'High-performance LLM inference on Apple Silicon using MLX and vLLM',
+ label: 'vLLM',
+ category: 'Local AI',
+ creator: 'OpenHarness contributors',
+ color: Color(0xffffc743),
+ asset: 'assets/engine-icons/vllm.png',
+ homepage: 'https://github.com/vllm-project/vllm-metal',
+ blurb: 'Serve models with vLLM Metal and measure concurrent request performance.',
+ ),
+ 'autonomous/mlx-lm': EngineIdentity(
+ id: 'autonomous/mlx-lm',
+ tagline:
+ 'Generating text with large language models on Apple silicon with MLX',
+ label: 'MLX-LM',
+ category: 'Local AI',
+ creator: 'OpenHarness contributors',
+ color: Color(0xffa9caff),
+ asset: 'assets/engine-icons/mlx.png',
+ homepage: 'https://github.com/ml-explore/mlx-lm',
+ blurb:
+ 'Run and compare language models directly on Apple Silicon with MLX.',
+ ),
+ 'autonomous/ollama': EngineIdentity(
+ id: 'autonomous/ollama',
+ tagline: 'Start building with open models.',
+ label: 'Ollama',
+ category: 'Local AI',
+ creator: 'OpenHarness contributors',
+ color: Color(0xfff4f6ef),
+ asset: 'assets/engine-icons/ollama.png',
+ homepage: 'https://ollama.com',
+ blurb:
+ 'Run and benchmark Ollama models on your Mac through natural language.',
+ ),
+ 'local/vllm': EngineIdentity(
+ id: 'local/vllm',
+ label: 'vLLM',
+ category: 'Local AI',
+ creator: 'Local',
+ color: Color(0xffffc743),
+ asset: 'assets/engine-icons/vllm.png',
+ homepage: 'https://github.com/vllm-project/vllm-metal',
+ blurb: 'Serve models with vLLM Metal and measure concurrent request performance.',
+ ),
+ 'local/mlx-lm': EngineIdentity(
+ id: 'local/mlx-lm',
+ label: 'MLX-LM',
+ category: 'Local AI',
+ creator: 'Local',
+ color: Color(0xffa9caff),
+ asset: 'assets/engine-icons/mlx.png',
+ homepage: 'https://github.com/ml-explore/mlx-lm',
+ blurb:
+ 'Run and compare language models directly on Apple Silicon with MLX.',
+ ),
+ 'local/ollama': EngineIdentity(
+ id: 'local/ollama',
+ label: 'Ollama',
+ category: 'Local AI',
+ creator: 'Local',
+ color: Color(0xfff4f6ef),
+ asset: 'assets/engine-icons/ollama.png',
+ homepage: 'https://ollama.com',
+ blurb:
+ 'Run and benchmark Ollama models on your Mac through natural language.',
+ ),
// Original creative identities; source vectors ship in each package under brand/.
'autonomous/voxel-worlds': EngineIdentity(
id: 'autonomous/voxel-worlds',
@@ -236,7 +305,8 @@ const _harnesses = <String, EngineIdentity>{
id: 'autonomous/generative-art',
label: 'Generative Art',
category: 'Original art',
- tagline: 'Turn a visual brief into editable artwork and complete asset sets',
+ tagline:
+ 'Turn a visual brief into editable artwork and complete asset sets',
creator: 'Autonomous',
color: Color(0xffce7645),
asset: 'assets/engine-icons/generative-art.png',
@@ -325,7 +395,7 @@ const _harnesses = <String, EngineIdentity>{
'autonomous/autonomous-grid': EngineIdentity(
id: 'autonomous/autonomous-grid',
label: 'Grid',
- category: 'Compute',
+ category: 'Local AI',
tagline: 'Deploy open-weight models across your machines and watch the fleet live',
creator: 'Autonomous',
color: Color(0xfff5a623),
@@ -556,7 +626,7 @@ const _harnesses = <String, EngineIdentity>{
id: 'autonomous/openscad',
label: "OpenSCAD",
category: "CAD",
- tagline: "The Programmers Solid 3D CAD Modeller",
+ tagline: "Your measurements. A family of usable parts.",
creator: "OpenSCAD",
color: Color(0xffe2c63d),
asset: 'assets/engine-icons/openscad.png',
@@ -565,7 +635,7 @@ const _harnesses = <String, EngineIdentity>{
id: 'autonomous/freecad',
label: "FreeCAD",
category: "CAD",
- tagline: "Parametric 3D modeling with verified STEP exports",
+ tagline: "From your measurements to checked, editable custom parts",
creator: "OpenHarness contributors",
color: Color(0xff418fde),
asset: 'assets/engine-icons/freecad.png',
@@ -574,7 +644,7 @@ const _harnesses = <String, EngineIdentity>{
id: 'autonomous/orca-slicer',
label: "Orca Slicer",
category: "Fabrication",
- tagline: "An open-source slicer for FFF 3D printers",
+ tagline: "Your mesh. Compared plans. Editable native projects.",
creator: "OpenHarness contributors",
color: Color(0xff009789),
asset: 'assets/engine-icons/orca-slicer.png',
@@ -610,7 +680,7 @@ const _harnesses = <String, EngineIdentity>{
id: 'autonomous/score',
label: "Score",
category: "Music",
- tagline: "Music notation for everyone",
+ tagline: "Your musical idea, ready to hear and share",
creator: "OpenHarness contributors",
color: Color(0xffbd965f),
asset: 'assets/engine-icons/score.png',
@@ -630,6 +700,12 @@ const _harnesses = <String, EngineIdentity>{
/// say "Runs on Claude Code" — and send the right `engine` — before the machine
/// has answered `dsh_list`. The daemon's catalog is authoritative when present.
const knownHarnessBase = <String, String>{
+ 'autonomous/ollama': 'codex',
+ 'autonomous/mlx-lm': 'codex',
+ 'autonomous/vllm': 'codex',
+ 'local/ollama': 'codex',
+ 'local/mlx-lm': 'codex',
+ 'local/vllm': 'codex',
'autonomous/voxel-worlds': 'claude',
'autonomous/generative-art': 'claude',
'autonomous/music-studio': 'claude',
diff --git a/desktop/test/engine_identity_test.dart b/desktop/test/engine_identity_test.dart
index 533e7715d..d44320fea 100644
--- a/desktop/test/engine_identity_test.dart
+++ b/desktop/test/engine_identity_test.dart
@@ -23,6 +23,47 @@ class _NoPictures extends CachingAssetBundle {
}
void main() {
+ testWidgets('Local AI frameworks load their real logos without initials', (
+ tester,
+ ) async {
+ const ids = [
+ 'autonomous/ollama',
+ 'autonomous/mlx-lm',
+ 'autonomous/vllm',
+ 'local/ollama',
+ 'local/mlx-lm',
+ 'local/vllm',
+ ];
+ await tester.pumpWidget(
+ MaterialApp(
+ home: Row(
+ children: [for (final id in ids) EngineMark(engine: id, size: 32)],
+ ),
+ ),
+ );
+ final context = tester.element(find.byType(Row));
+ await tester.runAsync(() async {
+ for (final id in ids) {
+ final identity = engineIdentity(id);
+ expect(identity.asset, isNotNull, reason: '$id must have a logo');
+ await precacheImage(AssetImage(identity.asset!), context);
+ }
+ });
+ await tester.pump();
+ for (final id in ids) {
+ expect(find.byKey(ValueKey('engine-icon-$id')), findsOneWidget);
+ expect(find.byKey(ValueKey('engine-fallback-$id')), findsNothing);
+ expect(
+ find.descendant(
+ of: find.byKey(ValueKey('engine-icon-$id')),
+ matching: find.byType(RawImage),
+ ),
+ findsOneWidget,
+ );
+ }
+ expect(tester.takeException(), isNull);
+ });
+
test(
'every first-party harness has a face and a base engine this build knows',
() {
@@ -121,7 +162,11 @@ void main() {
expect(identity.category, package['category'], reason: '$id category');
checked++;
}
- expect(checked, knownHarnesses.length);
+ // Linked local harnesses are intentionally outside store/agents.
+ expect(
+ checked,
+ knownHarnesses.where((h) => !h.id.startsWith('local/')).length,
+ );
for (final engine in allEngines) {
expect(engine.tagline, isNotEmpty, reason: engine.id);
expect(engine.tagline!.length, lessThanOrEqualTo(80), reason: engine.id);
diff --git a/desktop/test/store_discover_test.dart b/desktop/test/store_discover_test.dart
index 5266c483c..137953295 100644
--- a/desktop/test/store_discover_test.dart
+++ b/desktop/test/store_discover_test.dart
@@ -37,6 +37,7 @@ final _catalog = [
('circuitjs', 'CircuitJS', 'Circuits'),
('rdkit', 'RDKit', 'Chemistry'),
('yosys', 'Yosys', 'Chips'),
+ ('ollama', 'Ollama', 'Local AI'),
])
DshEntry(
id: 'autonomous/$id',
@@ -196,6 +197,35 @@ Future<void> _capture(WidgetTester tester, GlobalKey key, String name) async {
}
void main() {
+ test('Local AI groups Grid and current and future local runtimes', () {
+ for (final (id, domain) in [
+ ('autonomous/autonomous-grid', 'Compute'),
+ ('local/ollama', 'Compute'),
+ ('local/mlx-lm', 'Local AI'),
+ ('local/vllm', 'Local AI'),
+ ('community/next-runtime', 'local ai'),
+ ]) {
+ expect(
+ storeCategoryFor(
+ DshEntry(id: id, name: id, engine: 'codex', category: domain),
+ ),
+ 'Local AI',
+ );
+ }
+ expect(
+ storeCategoryFor(
+ const DshEntry(
+ id: 'codex',
+ name: 'Codex',
+ engine: 'codex',
+ kind: 'engine',
+ category: 'Code',
+ ),
+ ),
+ 'Code',
+ );
+ });
+
setUpAll(() async {
await loadRealFonts();
await (FontLoader(
@@ -244,6 +274,7 @@ void main() {
'Media',
'Science',
'Games',
+ 'Local AI',
'Code',
]) {
expect(
diff --git a/docs/research/local-ai-harnesses/README.md b/docs/research/local-ai-harnesses/README.md
new file mode 100644
index 000000000..0a81fa5c2
--- /dev/null
+++ b/docs/research/local-ai-harnesses/README.md
@@ -0,0 +1,35 @@
+# Local AI harness research and qualification
+
+Research began September 19, 2026 with language models on Apple Silicon. The first
+three implementations are [Ollama](../../../store/agents/ollama/README.md),
+[MLX-LM](../../../store/agents/mlx-lm/README.md), and
+[vLLM](../../../store/agents/vllm/README.md), using Metal on the Mac.
+
+- [Store packaging and clean-install verification](store-verification.json).
+- [Ecosystem research](recommendation.md) and [candidate list](shortlist.json).
+- [Product direction and framework/backend boundary](architecture.md).
+- [Ollama findings](pilot-findings.md), [checks](pilot-verification.json), and
+ [raw measurements](pilot-benchmarks.json).
+- [MLX-LM findings](mlx-lm/findings.md), [checks](mlx-lm/verification.json), and
+ [raw measurements](mlx-lm/benchmarks.json).
+- [vLLM Metal findings](vllm/findings.md), [checks](vllm/verification.json), and
+ [raw measurements](vllm/benchmarks.json).
+
+These notes preserve dated observations, including failures and decisions that
+later changed. Early `local/…` identifiers refer to linked development installs;
+published packages use `autonomous/ollama`, `autonomous/mlx-lm`, and
+`autonomous/vllm`. The desktop retains the old identities for existing workspaces.
+
+Measurements used an M2 Max with 64 GiB unified memory and small demonstration
+models. Framework timing and cache definitions differ; this is functional
+qualification, not a cross-framework speed or quality ranking. Native vLLM UI
+inspection was unavailable during its original handoff. The candidate list does
+not commit to building or claim compatibility for every researched framework.
+
+Private home/workspace prefixes in evidence are replaced with `<home>` and
+`<pilot-checkout>`. Model weights, runtime environments, endpoint control tokens,
+session state and copied upstream source snapshots are excluded. Source links
+and public repository/release metadata preserve the research provenance.
+
+The Store sidebar [capture](store-category.png) is rendered by the Flutter widget
+check using its fixture catalog; it is not a live-model dashboard capture.
diff --git a/docs/research/local-ai-harnesses/architecture.md b/docs/research/local-ai-harnesses/architecture.md
new file mode 100644
index 000000000..389d50f28
--- /dev/null
+++ b/docs/research/local-ai-harnesses/architecture.md
@@ -0,0 +1,80 @@
+# Product direction: an agent for the engines
+
+Decision updated September 20, 2026 following the user's clarification.
+
+Harness, its conversational agent, and the viewer provide the model-management experience.
+The most valuable integrations expose engine capabilities that otherwise require learning
+complex commands, settings, deployment procedures, and troubleshooting. The user's analogy
+is Blender operated through an AI agent: ask for an outcome and inspect the actual result.
+
+```mermaid
+flowchart TB
+ U[User asks for an outcome] --> A[Harness + Codex or Claude]
+ A --> C[Shared jobs, model inventory and measurements]
+ C --> M[MLX-LM adapter]
+ C --> L[llama.cpp adapter]
+ C --> S[vLLM / SGLang adapters]
+ M --> E[Native engine capabilities]
+ L --> E
+ S --> E
+ E --> V[Live viewer: state, outputs, performance]
+```
+
+This is the intended architecture. Currently Ollama, MLX-LM and vLLM (Metal backend) adapters are implemented;
+llama.cpp and SGLang remain proposed work. Codex is the operator in the current
+packages. Its configured provider is separate from the local models it operates.
+
+## Revised priorities
+
+| Priority | Engine / framework | Distinct value to expose through the agent | Status |
+| --- | --- | --- | --- |
+| 1 | [MLX-LM](https://github.com/ml-explore/mlx-lm) | Direct Apple Silicon execution; eventually quantization, cache/context controls, adapters and tuning | Initial inference harness implemented; advanced controls are future work |
+| 2 | [llama.cpp](https://github.com/ggml-org/llama.cpp) | GGUF selection, quantization, Metal offload, context/cache configuration and native serving | Next recommended harness |
+| 3 | [vLLM](https://github.com/vllm-project/vllm), using [vLLM Metal](https://github.com/vllm-project/vllm-metal) on this Mac | Scheduling, batching, concurrency, throughput/latency tradeoffs and serving configuration | Initial Mac harness implemented and qualified on Qwen3 0.6B; community-maintained Metal backend |
+| 4 | [SGLang](https://github.com/sgl-project/sglang) | Advanced serving, cache reuse and structured workloads | Qualify the MLX backend on M2 Max; do not infer CUDA feature parity |
+| Retain | Ollama | Existing users, packaged model downloads and a convenient baseline | Working pilot remains supported |
+| Defer | LM Studio / llmster and other overlapping managers | Integrate when existing installations or unique backend capabilities justify it | Lower priority under this product direction |
+
+The reason to add a framework is a concrete capability, not complexity or popularity alone.
+MLX-LM is a language-model library built on MLX; MLX itself is the underlying array framework.
+Ollama and LM Studio also provide useful packaging and runtime services, so they are not
+merely duplicate chat interfaces. Their everyday management interface is nevertheless less
+central to our product than direct access to engine controls.
+
+## Outcomes the deeper harnesses should support
+
+- “Choose a quantization that fits this Mac and measure the quality/speed tradeoff.”
+- “Make long-document prompts work within this memory budget.”
+- “Serve several clients and show throughput alongside first-token latency.”
+- “Try these cache or offload settings, keep the best result, and explain the difference.”
+- “Diagnose this failed load and repair the configuration.”
+
+Some describe future capabilities. The vLLM harness now supports serving configuration, prefix-cache controls and bounded concurrent tests.
+The current MLX-LM harness supports download, load, unload, chat, sequential benchmarks,
+cancel, recovery, and native measurements. It deliberately establishes a working direct
+engine integration before adding tuning, conversion, fine-tuning or multi-client serving.
+
+## One harness per framework
+
+The accepted structure is **vLLM** with a **Metal backend** on this Mac, not separate
+vLLM and vLLM Metal entries. Other vLLM backends can later extend the same framework
+identity. MLX-LM remains separate because its direct library workflow differs from
+vLLM serving. See [vLLM findings](vllm/findings.md) for qualification and measurements.
+
+## Implementation rules
+
+Use the official CLI, API, or library interface appropriate to each operation. There is no
+requirement to force structured library calls through a CLI. The agent should not depend on
+the vendor's GUI. Models' generated text remains data, never executable commands.
+
+Share the job queue, state contract, control runner and viewer logic. Preserve native
+capabilities and truthful measurement definitions in each adapter. Each framework gets its
+own name, logo and relevant copy; the embedded viewer remains the visual companion to the
+agent. Avoid adding a second conversational agent to that viewer.
+
+Record cache conditions, quantization, revisions, runtime versions and hardware. A shared
+chart does not make different benchmarks directly comparable. In particular, MLX-LM's
+current fresh prompt cache and Ollama's possible prefix reuse are different conditions.
+
+The broad [original research](recommendation.md) remains useful as a dated ecosystem map.
+This decision supersedes its original implementation order.
diff --git a/docs/research/local-ai-harnesses/github-snapshot.json b/docs/research/local-ai-harnesses/github-snapshot.json
new file mode 100644
index 000000000..263b3c978
--- /dev/null
+++ b/docs/research/local-ai-harnesses/github-snapshot.json
@@ -0,0 +1,395 @@
+{
+ "retrieved_at": "2026-09-20T01:30:33.007932+00:00",
+ "source": "GitHub REST API; stars are an interest proxy, not active user counts",
+ "repositories": [
+ {
+ "full_name": "ollama/ollama",
+ "html_url": "https://github.com/ollama/ollama",
+ "description": "Get up and running with Kimi, GLM, MiniMax, DeepSeek, gpt-oss, Qwen, Gemma and other models.",
+ "stargazers_count": 181281,
+ "forks_count": 17939,
+ "created_at": "2023-06-26T19:39:32Z",
+ "pushed_at": "2026-09-19T20:41:40Z",
+ "updated_at": "2026-09-20T01:15:15Z",
+ "archived": false,
+ "default_branch": "main",
+ "license": "MIT"
+ },
+ {
+ "full_name": "ggml-org/llama.cpp",
+ "html_url": "https://github.com/ggml-org/llama.cpp",
+ "description": "LLM inference in C/C++",
+ "stargazers_count": 128863,
+ "forks_count": 23461,
+ "created_at": "2023-03-10T18:58:00Z",
+ "pushed_at": "2026-09-20T00:31:42Z",
+ "updated_at": "2026-09-20T01:16:48Z",
+ "archived": false,
+ "default_branch": "master",
+ "license": "MIT"
+ },
+ {
+ "full_name": "vllm-project/vllm",
+ "html_url": "https://github.com/vllm-project/vllm",
+ "description": "A high-throughput and memory-efficient inference and serving engine for LLMs",
+ "stargazers_count": 92191,
+ "forks_count": 22423,
+ "created_at": "2023-02-09T11:23:20Z",
+ "pushed_at": "2026-09-20T00:16:45Z",
+ "updated_at": "2026-09-20T01:11:41Z",
+ "archived": false,
+ "default_branch": "main",
+ "license": "Apache-2.0"
+ },
+ {
+ "full_name": "sgl-project/sglang",
+ "html_url": "https://github.com/sgl-project/sglang",
+ "description": "SGLang is a high-performance serving framework for large language models and multimodal models.",
+ "stargazers_count": 36178,
+ "forks_count": 8993,
+ "created_at": "2024-01-08T04:15:52Z",
+ "pushed_at": "2026-09-20T01:30:25Z",
+ "updated_at": "2026-09-20T01:17:33Z",
+ "archived": false,
+ "default_branch": "main",
+ "license": "Apache-2.0"
+ },
+ {
+ "full_name": "ml-explore/mlx",
+ "html_url": "https://github.com/ml-explore/mlx",
+ "description": "MLX: An array framework for Apple silicon",
+ "stargazers_count": 28484,
+ "forks_count": 2267,
+ "created_at": "2023-11-28T23:33:45Z",
+ "pushed_at": "2026-09-17T21:11:19Z",
+ "updated_at": "2026-09-20T00:41:54Z",
+ "archived": false,
+ "default_branch": "main",
+ "license": "MIT"
+ },
+ {
+ "full_name": "ml-explore/mlx-lm",
+ "html_url": "https://github.com/ml-explore/mlx-lm",
+ "description": "Run LLMs with MLX",
+ "stargazers_count": 7069,
+ "forks_count": 1059,
+ "created_at": "2025-03-11T16:38:30Z",
+ "pushed_at": "2026-09-18T15:27:36Z",
+ "updated_at": "2026-09-19T23:04:04Z",
+ "archived": false,
+ "default_branch": "main",
+ "license": "MIT"
+ },
+ {
+ "full_name": "jundot/omlx",
+ "html_url": "https://github.com/jundot/omlx",
+ "description": "LLM inference server with continuous batching & SSD caching for Apple Silicon \u2014 managed from the macOS menu bar",
+ "stargazers_count": 21930,
+ "forks_count": 1899,
+ "created_at": "2026-02-13T14:13:27Z",
+ "pushed_at": "2026-09-19T13:50:34Z",
+ "updated_at": "2026-09-20T00:47:03Z",
+ "archived": false,
+ "default_branch": "main",
+ "license": "Apache-2.0"
+ },
+ {
+ "full_name": "exo-explore/exo",
+ "html_url": "https://github.com/exo-explore/exo",
+ "description": "Run frontier AI locally.",
+ "stargazers_count": 47531,
+ "forks_count": 3527,
+ "created_at": "2024-06-24T18:36:22Z",
+ "pushed_at": "2026-08-25T18:59:53Z",
+ "updated_at": "2026-09-20T00:54:13Z",
+ "archived": false,
+ "default_branch": "main",
+ "license": "Apache-2.0"
+ },
+ {
+ "full_name": "waybarrios/vllm-mlx",
+ "html_url": "https://github.com/waybarrios/vllm-mlx",
+ "description": "High-performance OpenAI and Anthropic compatible LLM inference server for Apple Silicon. Native MLX, continuous batching, multimodal models, MCP tool calling, and Claude Code support.",
+ "stargazers_count": 1587,
+ "forks_count": 222,
+ "created_at": "2025-12-06T18:26:08Z",
+ "pushed_at": "2026-09-19T18:11:07Z",
+ "updated_at": "2026-09-19T18:11:11Z",
+ "archived": false,
+ "default_branch": "main",
+ "license": "Apache-2.0"
+ },
+ {
+ "full_name": "lemonade-sdk/lemonade",
+ "html_url": "https://github.com/lemonade-sdk/lemonade",
+ "description": "Lemonade helps users discover and run local AI apps by serving optimized LLMs right from their own GPUs and NPUs. Join our discord: https://discord.gg/5xXzkMu8Zk",
+ "stargazers_count": 5748,
+ "forks_count": 505,
+ "created_at": "2025-05-15T19:17:39Z",
+ "pushed_at": "2026-09-19T10:27:47Z",
+ "updated_at": "2026-09-19T22:14:42Z",
+ "archived": false,
+ "default_branch": "main",
+ "license": "Apache-2.0"
+ },
+ {
+ "full_name": "EricLBuehler/mistral.rs",
+ "html_url": "https://github.com/EricLBuehler/mistral.rs",
+ "description": "Fast, flexible LLM inference",
+ "stargazers_count": 7709,
+ "forks_count": 710,
+ "created_at": "2024-02-26T22:30:06Z",
+ "pushed_at": "2026-09-08T03:18:32Z",
+ "updated_at": "2026-09-20T00:58:27Z",
+ "archived": false,
+ "default_branch": "master",
+ "license": "MIT"
+ },
+ {
+ "full_name": "qualcomm/GenieX",
+ "html_url": "https://github.com/qualcomm/GenieX",
+ "description": "Run frontier LLMs and VLMs locally on Qualcomm devices across NPU, GPU, and CPU with a few lines of code",
+ "stargazers_count": 8380,
+ "forks_count": 1054,
+ "created_at": "2024-08-16T20:13:07Z",
+ "pushed_at": "2026-09-18T18:27:43Z",
+ "updated_at": "2026-09-19T03:57:23Z",
+ "archived": false,
+ "default_branch": "main",
+ "license": "BSD-3-Clause"
+ },
+ {
+ "full_name": "mudler/LocalAI",
+ "html_url": "https://github.com/mudler/LocalAI",
+ "description": "LocalAI is the open-source AI engine. Run any model - LLMs, vision, voice, image, video - on any hardware. No GPU required.",
+ "stargazers_count": 49177,
+ "forks_count": 4458,
+ "created_at": "2023-03-18T22:58:02Z",
+ "pushed_at": "2026-09-20T00:05:21Z",
+ "updated_at": "2026-09-20T01:18:26Z",
+ "archived": false,
+ "default_branch": "master",
+ "license": "MIT"
+ },
+ {
+ "full_name": "janhq/jan",
+ "html_url": "https://github.com/janhq/jan",
+ "description": "Jan is an open source alternative to ChatGPT that runs 100% offline on your computer.",
+ "stargazers_count": 44564,
+ "forks_count": 3040,
+ "created_at": "2023-08-17T02:17:10Z",
+ "pushed_at": "2026-09-19T04:55:33Z",
+ "updated_at": "2026-09-20T00:37:40Z",
+ "archived": false,
+ "default_branch": "main",
+ "license": "NOASSERTION"
+ },
+ {
+ "full_name": "mozilla-ai/llamafile",
+ "html_url": "https://github.com/mozilla-ai/llamafile",
+ "description": "Distribute and run LLMs with a single file.",
+ "stargazers_count": 25997,
+ "forks_count": 1598,
+ "created_at": "2023-09-10T21:12:32Z",
+ "pushed_at": "2026-09-16T09:14:28Z",
+ "updated_at": "2026-09-19T21:45:35Z",
+ "archived": false,
+ "default_branch": "main",
+ "license": "NOASSERTION"
+ },
+ {
+ "full_name": "mlc-ai/mlc-llm",
+ "html_url": "https://github.com/mlc-ai/mlc-llm",
+ "description": "Universal LLM Deployment Engine with ML Compilation",
+ "stargazers_count": 23169,
+ "forks_count": 2138,
+ "created_at": "2023-04-29T01:59:25Z",
+ "pushed_at": "2026-08-17T20:50:06Z",
+ "updated_at": "2026-09-20T01:24:05Z",
+ "archived": false,
+ "default_branch": "main",
+ "license": "Apache-2.0"
+ },
+ {
+ "full_name": "mostlygeek/llama-swap",
+ "html_url": "https://github.com/mostlygeek/llama-swap",
+ "description": "Reliable model swapping for any local OpenAI/Anthropic compatible server - llama.cpp, vllm, etc",
+ "stargazers_count": 5700,
+ "forks_count": 472,
+ "created_at": "2024-10-04T04:06:01Z",
+ "pushed_at": "2026-09-19T22:01:33Z",
+ "updated_at": "2026-09-20T01:13:11Z",
+ "archived": false,
+ "default_branch": "main",
+ "license": "MIT"
+ },
+ {
+ "full_name": "LostRuins/koboldcpp",
+ "html_url": "https://github.com/LostRuins/koboldcpp",
+ "description": "Run GGUF models easily with a KoboldAI UI. One File. Zero Install.",
+ "stargazers_count": 11804,
+ "forks_count": 777,
+ "created_at": "2023-03-16T02:07:02Z",
+ "pushed_at": "2026-09-19T15:12:22Z",
+ "updated_at": "2026-09-19T22:47:22Z",
+ "archived": false,
+ "default_branch": "concedo",
+ "license": "AGPL-3.0"
+ },
+ {
+ "full_name": "turboderp-org/exllamav3",
+ "html_url": "https://github.com/turboderp-org/exllamav3",
+ "description": "An optimized quantization and inference library for running LLMs locally on modern consumer-class GPUs ",
+ "stargazers_count": 1464,
+ "forks_count": 181,
+ "created_at": "2025-04-06T12:36:08Z",
+ "pushed_at": "2026-09-19T13:39:07Z",
+ "updated_at": "2026-09-20T01:12:37Z",
+ "archived": false,
+ "default_branch": "master",
+ "license": "MIT"
+ },
+ {
+ "full_name": "kvcache-ai/ktransformers",
+ "html_url": "https://github.com/kvcache-ai/ktransformers",
+ "description": "A Flexible Framework for Experiencing Heterogeneous LLM Inference/Fine-tune Optimizations",
+ "stargazers_count": 19522,
+ "forks_count": 1576,
+ "created_at": "2024-07-26T07:18:28Z",
+ "pushed_at": "2026-09-19T03:06:55Z",
+ "updated_at": "2026-09-19T22:59:35Z",
+ "archived": false,
+ "default_branch": "main",
+ "license": "Apache-2.0"
+ },
+ {
+ "full_name": "NVIDIA/TensorRT-LLM",
+ "html_url": "https://github.com/NVIDIA/TensorRT-LLM",
+ "description": "TensorRT LLM provides users with an easy-to-use Python API to define Large Language Models (LLMs) and supports state-of-the-art optimizations to perform inference efficiently on NVIDIA GPUs. TensorRT LLM also contains components to create Python and C++ runtimes that orchestrate the inference execution in a performant way.",
+ "stargazers_count": 14673,
+ "forks_count": 2762,
+ "created_at": "2023-08-16T17:14:27Z",
+ "pushed_at": "2026-09-19T23:51:06Z",
+ "updated_at": "2026-09-20T00:44:05Z",
+ "archived": false,
+ "default_branch": "main",
+ "license": "NOASSERTION"
+ },
+ {
+ "full_name": "huggingface/text-generation-inference",
+ "html_url": "https://github.com/huggingface/text-generation-inference",
+ "description": "Large Language Model Text Generation Inference",
+ "stargazers_count": 10885,
+ "forks_count": 1288,
+ "created_at": "2022-10-08T10:26:28Z",
+ "pushed_at": "2026-03-21T11:34:22Z",
+ "updated_at": "2026-09-19T11:48:43Z",
+ "archived": true,
+ "default_branch": "main",
+ "license": "Apache-2.0"
+ },
+ {
+ "full_name": "lmstudio-ai/lmstudio-js",
+ "html_url": "https://github.com/lmstudio-ai/lmstudio-js",
+ "description": "LM Studio TypeScript SDK",
+ "stargazers_count": 1781,
+ "forks_count": 303,
+ "created_at": "2023-09-15T06:05:04Z",
+ "pushed_at": "2026-09-18T22:24:27Z",
+ "updated_at": "2026-09-19T04:08:58Z",
+ "archived": false,
+ "default_branch": "main",
+ "license": "MIT"
+ },
+ {
+ "full_name": "ggml-org/whisper.cpp",
+ "html_url": "https://github.com/ggml-org/whisper.cpp",
+ "description": "Port of OpenAI's Whisper model in C/C++",
+ "stargazers_count": 53782,
+ "forks_count": 6181,
+ "created_at": "2022-09-25T18:26:37Z",
+ "pushed_at": "2026-09-18T10:20:25Z",
+ "updated_at": "2026-09-20T01:27:44Z",
+ "archived": false,
+ "default_branch": "master",
+ "license": "MIT"
+ },
+ {
+ "full_name": "Comfy-Org/ComfyUI",
+ "html_url": "https://github.com/Comfy-Org/ComfyUI",
+ "description": "The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface.",
+ "stargazers_count": 133934,
+ "forks_count": 15874,
+ "created_at": "2023-01-17T03:15:56Z",
+ "pushed_at": "2026-09-20T00:29:57Z",
+ "updated_at": "2026-09-20T01:16:24Z",
+ "archived": false,
+ "default_branch": "master",
+ "license": "GPL-3.0"
+ },
+ {
+ "full_name": "vllm-project/vllm-metal",
+ "html_url": "https://github.com/vllm-project/vllm-metal",
+ "description": "Community maintained hardware plugin for vLLM on Apple Silicon",
+ "stargazers_count": 1749,
+ "forks_count": 260,
+ "created_at": "2025-12-12T20:37:25Z",
+ "pushed_at": "2026-09-19T23:52:06Z",
+ "updated_at": "2026-09-20T00:59:06Z",
+ "archived": false,
+ "default_branch": "main",
+ "license": "Apache-2.0"
+ },
+ {
+ "full_name": "containers/ramalama",
+ "html_url": "https://github.com/containers/ramalama",
+ "description": "RamaLama is an open-source developer tool that simplifies the local serving of AI models from any source and facilitates their use for inference in production, all through the familiar language of containers.",
+ "stargazers_count": 3053,
+ "forks_count": 367,
+ "created_at": "2024-07-24T19:09:58Z",
+ "pushed_at": "2026-09-19T00:03:50Z",
+ "updated_at": "2026-09-19T19:09:18Z",
+ "archived": false,
+ "default_branch": "main",
+ "license": "MIT"
+ },
+ {
+ "full_name": "GradientHQ/parallax",
+ "html_url": "https://github.com/GradientHQ/parallax",
+ "description": "Parallax is a distributed model serving framework that lets you build your own AI cluster anywhere",
+ "stargazers_count": 1381,
+ "forks_count": 147,
+ "created_at": "2025-09-22T07:00:51Z",
+ "pushed_at": "2026-07-01T07:35:43Z",
+ "updated_at": "2026-09-18T14:55:16Z",
+ "archived": false,
+ "default_branch": "main",
+ "license": "Apache-2.0"
+ },
+ {
+ "full_name": "danveloper/flash-moe",
+ "html_url": "https://github.com/danveloper/flash-moe",
+ "description": "Running a big model on a small laptop",
+ "stargazers_count": 4145,
+ "forks_count": 513,
+ "created_at": "2026-03-18T16:13:13Z",
+ "pushed_at": "2026-03-19T17:21:57Z",
+ "updated_at": "2026-09-19T22:36:29Z",
+ "archived": false,
+ "default_branch": "main",
+ "license": null
+ },
+ {
+ "repo": "ikawrakow/ik_llama.cpp",
+ "error": "HTTP Error 403: rate limit exceeded"
+ },
+ {
+ "repo": "google-ai-edge/LiteRT-LM",
+ "error": "HTTP Error 403: rate limit exceeded"
+ },
+ {
+ "repo": "lightseekorg/chitu",
+ "error": "HTTP Error 404: Not Found"
+ }
+ ]
+}
diff --git a/docs/research/local-ai-harnesses/mlx-lm/benchmarks.json b/docs/research/local-ai-harnesses/mlx-lm/benchmarks.json
new file mode 100644
index 000000000..3fec700ee
--- /dev/null
+++ b/docs/research/local-ai-harnesses/mlx-lm/benchmarks.json
@@ -0,0 +1,333 @@
+{
+ "at": "2026-09-20T04:38:50.328Z",
+ "benchmarks": [
+ {
+ "id": "3c5bc697-2bbc-4009-975a-cc4da6257e88",
+ "model": "mlx-community/Qwen2.5-0.5B-Instruct-4bit",
+ "digest": "a5339a4131f135d0fdc6a5c8b5bbed2753bbe0f3",
+ "at": "2026-09-20T04:38:48.596Z",
+ "runtime": {
+ "name": "MLX-LM",
+ "version": "0.31.3",
+ "mlxVersion": "0.32.2"
+ },
+ "hardware": {
+ "platform": "darwin",
+ "arch": "arm64",
+ "chip": "Apple M2 Max",
+ "totalMemory": 68719476736,
+ "cores": 12
+ },
+ "settings": {
+ "context": 2048,
+ "maxTokens": 128,
+ "seed": 42,
+ "temperature": 0,
+ "thinking": false
+ },
+ "prompt": "Explain how a bicycle works to a curious beginner. Discuss the pedals, chain, gears, wheels, brakes, and balance. Write a clear explanation of at least 250 words without a heading.",
+ "promptHash": "e3aff106a75867995dfbb42b73d60cb031d6d1d9511c155329d29033854e829a",
+ "mode": "warm model; fresh prompt cache for each sample",
+ "samples": [
+ {
+ "model": "mlx-community/Qwen2.5-0.5B-Instruct-4bit",
+ "tokens": 128,
+ "promptTokens": 71,
+ "tokensPerSecond": 309.064244643694,
+ "promptTokensPerSecond": 1619.0993148969428,
+ "nativeFirstTokenMs": 126.28733401652426,
+ "nativeFirstOutputMs": 126.28733401652426,
+ "peakMemoryBytes": 364835054,
+ "thinking": false,
+ "doneReason": "length",
+ "wallMs": 547.7812089957297,
+ "measurement": "MLX-LM native generation_tps; TTFT includes local IPC; fresh prompt cache",
+ "firstTokenMs": 127.83279100000073,
+ "firstOutputMs": 127.83866599999601
+ },
+ {
+ "model": "mlx-community/Qwen2.5-0.5B-Instruct-4bit",
+ "tokens": 128,
+ "promptTokens": 71,
+ "tokensPerSecond": 425.51576417741893,
+ "promptTokensPerSecond": 3517.899168105599,
+ "nativeFirstTokenMs": 96.07566590420902,
+ "nativeFirstOutputMs": 96.07566590420902,
+ "peakMemoryBytes": 339931318,
+ "thinking": false,
+ "doneReason": "length",
+ "wallMs": 403.96612498443574,
+ "measurement": "MLX-LM native generation_tps; TTFT includes local IPC; fresh prompt cache",
+ "firstTokenMs": 96.49270799999795,
+ "firstOutputMs": 96.49812499999825
+ },
+ {
+ "model": "mlx-community/Qwen2.5-0.5B-Instruct-4bit",
+ "tokens": 128,
+ "promptTokens": 71,
+ "tokensPerSecond": 425.9770567293837,
+ "promptTokensPerSecond": 3476.486737135455,
+ "nativeFirstTokenMs": 83.1738340202719,
+ "nativeFirstOutputMs": 83.1738340202719,
+ "peakMemoryBytes": 339931318,
+ "thinking": false,
+ "doneReason": "length",
+ "wallMs": 390.4475839808583,
+ "measurement": "MLX-LM native generation_tps; TTFT includes local IPC; fresh prompt cache",
+ "firstTokenMs": 83.62958300000173,
+ "firstOutputMs": 83.66079199999513
+ }
+ ],
+ "medianTokensPerSecond": 425.51576417741893,
+ "medianFirstTokenMs": 96.49270799999795,
+ "medianFirstOutputMs": 96.49812499999825,
+ "residentBytes": 281161510
+ },
+ {
+ "id": "d0b3884b-aeb7-415d-8751-d8debc189886",
+ "model": "mlx-community/Qwen3-0.6B-4bit",
+ "digest": "73e3e38d981303bc594367cd910ea6eb48349da8",
+ "at": "2026-09-20T04:38:46.622Z",
+ "runtime": {
+ "name": "MLX-LM",
+ "version": "0.31.3",
+ "mlxVersion": "0.32.2"
+ },
+ "hardware": {
+ "platform": "darwin",
+ "arch": "arm64",
+ "chip": "Apple M2 Max",
+ "totalMemory": 68719476736,
+ "cores": 12
+ },
+ "settings": {
+ "context": 2048,
+ "maxTokens": 128,
+ "seed": 42,
+ "temperature": 0,
+ "thinking": false
+ },
+ "prompt": "Explain how a bicycle works to a curious beginner. Discuss the pedals, chain, gears, wheels, brakes, and balance. Write a clear explanation of at least 250 words without a heading.",
+ "promptHash": "e3aff106a75867995dfbb42b73d60cb031d6d1d9511c155329d29033854e829a",
+ "mode": "warm model; fresh prompt cache for each sample",
+ "samples": [
+ {
+ "model": "mlx-community/Qwen3-0.6B-4bit",
+ "tokens": 128,
+ "promptTokens": 54,
+ "tokensPerSecond": 413.0146398735052,
+ "promptTokensPerSecond": 1623.9828344274504,
+ "nativeFirstTokenMs": 102.54729096777737,
+ "nativeFirstOutputMs": 102.54729096777737,
+ "peakMemoryBytes": 500007338,
+ "thinking": false,
+ "doneReason": "length",
+ "wallMs": 419.22220797277987,
+ "measurement": "MLX-LM native generation_tps; TTFT includes local IPC; fresh prompt cache",
+ "firstTokenMs": 103.16879200000403,
+ "firstOutputMs": 103.18395800000144
+ },
+ {
+ "model": "mlx-community/Qwen3-0.6B-4bit",
+ "tokens": 128,
+ "promptTokens": 54,
+ "tokensPerSecond": 411.88033434660207,
+ "promptTokensPerSecond": 1928.7408127582573,
+ "nativeFirstTokenMs": 88.01449998281896,
+ "nativeFirstOutputMs": 88.01449998281896,
+ "peakMemoryBytes": 471810454,
+ "thinking": false,
+ "doneReason": "length",
+ "wallMs": 405.6305829435587,
+ "measurement": "MLX-LM native generation_tps; TTFT includes local IPC; fresh prompt cache",
+ "firstTokenMs": 88.4462910000002,
+ "firstOutputMs": 88.45141600000352
+ },
+ {
+ "model": "mlx-community/Qwen3-0.6B-4bit",
+ "tokens": 128,
+ "promptTokens": 54,
+ "tokensPerSecond": 411.30251863784935,
+ "promptTokensPerSecond": 2098.52032094187,
+ "nativeFirstTokenMs": 89.04995792545378,
+ "nativeFirstOutputMs": 89.04995792545378,
+ "peakMemoryBytes": 460374402,
+ "thinking": false,
+ "doneReason": "length",
+ "wallMs": 407.01795800123364,
+ "measurement": "MLX-LM native generation_tps; TTFT includes local IPC; fresh prompt cache",
+ "firstTokenMs": 89.4753750000018,
+ "firstOutputMs": 89.48500000000058
+ }
+ ],
+ "medianTokensPerSecond": 411.88033434660207,
+ "medianFirstTokenMs": 89.4753750000018,
+ "medianFirstOutputMs": 89.48500000000058,
+ "residentBytes": 335380518
+ },
+ {
+ "id": "158a1c87-c420-418f-a212-16c43b3f0c07",
+ "model": "mlx-community/Qwen2.5-0.5B-Instruct-4bit",
+ "digest": "a5339a4131f135d0fdc6a5c8b5bbed2753bbe0f3",
+ "at": "2026-09-20T04:34:16.940Z",
+ "runtime": {
+ "name": "MLX-LM",
+ "version": "0.31.3",
+ "mlxVersion": "0.32.2"
+ },
+ "hardware": {
+ "platform": "darwin",
+ "arch": "arm64",
+ "chip": "Apple M2 Max",
+ "totalMemory": 68719476736,
+ "cores": 12
+ },
+ "settings": {
+ "context": 2048,
+ "maxTokens": 128,
+ "seed": 42,
+ "temperature": 0,
+ "thinking": false
+ },
+ "prompt": "Explain how a bicycle works to a curious beginner. Discuss the pedals, chain, gears, wheels, brakes, and balance. Write a clear explanation of at least 250 words without a heading.",
+ "promptHash": "e3aff106a75867995dfbb42b73d60cb031d6d1d9511c155329d29033854e829a",
+ "mode": "warm model; fresh prompt cache for each sample",
+ "samples": [
+ {
+ "model": "mlx-community/Qwen2.5-0.5B-Instruct-4bit",
+ "tokens": 128,
+ "promptTokens": 71,
+ "tokensPerSecond": 428.8513067082037,
+ "promptTokensPerSecond": 2620.478144601585,
+ "nativeFirstTokenMs": 91.2971660727635,
+ "nativeFirstOutputMs": 91.2971660727635,
+ "peakMemoryBytes": 364966126,
+ "thinking": false,
+ "doneReason": "length",
+ "wallMs": 396.4953750837594,
+ "measurement": "MLX-LM native generation_tps; TTFT includes local IPC; fresh prompt cache",
+ "firstTokenMs": 91.68441699999676,
+ "firstOutputMs": 91.6875
+ },
+ {
+ "model": "mlx-community/Qwen2.5-0.5B-Instruct-4bit",
+ "tokens": 128,
+ "promptTokens": 71,
+ "tokensPerSecond": 425.0283594713211,
+ "promptTokensPerSecond": 3067.738741595023,
+ "nativeFirstTokenMs": 85.24666703306139,
+ "nativeFirstOutputMs": 85.24666703306139,
+ "peakMemoryBytes": 355954910,
+ "thinking": false,
+ "doneReason": "length",
+ "wallMs": 393.28362501692027,
+ "measurement": "MLX-LM native generation_tps; TTFT includes local IPC; fresh prompt cache",
+ "firstTokenMs": 85.60129200000665,
+ "firstOutputMs": 85.60370800000965
+ },
+ {
+ "model": "mlx-community/Qwen2.5-0.5B-Instruct-4bit",
+ "tokens": 128,
+ "promptTokens": 71,
+ "tokensPerSecond": 423.8341013732513,
+ "promptTokensPerSecond": 3441.16590669372,
+ "nativeFirstTokenMs": 82.10400003008544,
+ "nativeFirstOutputMs": 82.10400003008544,
+ "peakMemoryBytes": 339931318,
+ "thinking": false,
+ "doneReason": "length",
+ "wallMs": 390.81266697030514,
+ "measurement": "MLX-LM native generation_tps; TTFT includes local IPC; fresh prompt cache",
+ "firstTokenMs": 82.58695900000748,
+ "firstOutputMs": 82.59008399999584
+ }
+ ],
+ "medianTokensPerSecond": 425.0283594713211,
+ "medianFirstTokenMs": 85.60129200000665,
+ "medianFirstOutputMs": 85.60370800000965,
+ "residentBytes": 281161510
+ },
+ {
+ "id": "faed2a4b-beb8-4f83-8c62-7ba8713fc901",
+ "model": "mlx-community/Qwen3-0.6B-4bit",
+ "digest": "73e3e38d981303bc594367cd910ea6eb48349da8",
+ "at": "2026-09-20T04:34:15.203Z",
+ "runtime": {
+ "name": "MLX-LM",
+ "version": "0.31.3",
+ "mlxVersion": "0.32.2"
+ },
+ "hardware": {
+ "platform": "darwin",
+ "arch": "arm64",
+ "chip": "Apple M2 Max",
+ "totalMemory": 68719476736,
+ "cores": 12
+ },
+ "settings": {
+ "context": 2048,
+ "maxTokens": 128,
+ "seed": 42,
+ "temperature": 0,
+ "thinking": false
+ },
+ "prompt": "Explain how a bicycle works to a curious beginner. Discuss the pedals, chain, gears, wheels, brakes, and balance. Write a clear explanation of at least 250 words without a heading.",
+ "promptHash": "e3aff106a75867995dfbb42b73d60cb031d6d1d9511c155329d29033854e829a",
+ "mode": "warm model; fresh prompt cache for each sample",
+ "samples": [
+ {
+ "model": "mlx-community/Qwen3-0.6B-4bit",
+ "tokens": 128,
+ "promptTokens": 54,
+ "tokensPerSecond": 414.26054115189083,
+ "promptTokensPerSecond": 1697.1658923725854,
+ "nativeFirstTokenMs": 96.4624589541927,
+ "nativeFirstOutputMs": 96.4624589541927,
+ "peakMemoryBytes": 500007338,
+ "thinking": false,
+ "doneReason": "length",
+ "wallMs": 412.626166944392,
+ "measurement": "MLX-LM native generation_tps; TTFT includes local IPC; fresh prompt cache",
+ "firstTokenMs": 97.11204100000032,
+ "firstOutputMs": 97.1181249999936
+ },
+ {
+ "model": "mlx-community/Qwen3-0.6B-4bit",
+ "tokens": 128,
+ "promptTokens": 54,
+ "tokensPerSecond": 414.8126472336596,
+ "promptTokensPerSecond": 1749.1743603673244,
+ "nativeFirstTokenMs": 98.89866597950459,
+ "nativeFirstOutputMs": 98.89866597950459,
+ "peakMemoryBytes": 484753814,
+ "thinking": false,
+ "doneReason": "length",
+ "wallMs": 414.61766604334116,
+ "measurement": "MLX-LM native generation_tps; TTFT includes local IPC; fresh prompt cache",
+ "firstTokenMs": 99.36183299998811,
+ "firstOutputMs": 99.36516599998868
+ },
+ {
+ "model": "mlx-community/Qwen3-0.6B-4bit",
+ "tokens": 128,
+ "promptTokens": 54,
+ "tokensPerSecond": 410.56548576829806,
+ "promptTokensPerSecond": 1537.6309974077894,
+ "nativeFirstTokenMs": 103.7760830949992,
+ "nativeFirstOutputMs": 103.7760830949992,
+ "peakMemoryBytes": 509837746,
+ "thinking": false,
+ "doneReason": "length",
+ "wallMs": 422.4835420027375,
+ "measurement": "MLX-LM native generation_tps; TTFT includes local IPC; fresh prompt cache",
+ "firstTokenMs": 104.33358299999963,
+ "firstOutputMs": 104.33679099999426
+ }
+ ],
+ "medianTokensPerSecond": 414.26054115189083,
+ "medianFirstTokenMs": 99.36183299998811,
+ "medianFirstOutputMs": 99.36516599998868,
+ "residentBytes": 335380518
+ }
+ ]
+}
\ No newline at end of file
diff --git a/docs/research/local-ai-harnesses/mlx-lm/findings.md b/docs/research/local-ai-harnesses/mlx-lm/findings.md
new file mode 100644
index 000000000..350be4766
--- /dev/null
+++ b/docs/research/local-ai-harnesses/mlx-lm/findings.md
@@ -0,0 +1,57 @@
+# MLX-LM pilot findings
+
+Verified 2026-09-20T04:38:50.327Z on Apple M2 Max, 64 GiB unified memory.
+MLX-LM 0.31.3, MLX 0.32.2, isolated Python 3.12 environment.
+
+## What is working
+
+The agent runner reaches MLX directly through a private Python worker. Native load and
+streaming APIs provide real inventory, text, native token rates and peak allocation.
+Download of Qwen3-0.6B-4bit succeeded into the existing Hugging Face cache, pinned to
+`73e3e38d981303bc594367cd910ea6eb48349da8`; selected files total 351,383,618 bytes.
+Cached Qwen2.5-0.5B was reused without downloading a second copy.
+
+The shared viewer is branded for MLX-LM with the official MLX wordmark. The native app
+registers `local/mlx-lm` under Local AI alongside Grid and Ollama. A dedicated native
+workspace was opened at `<home>/harnesses/mlx-lm-2026-09-20-00-44`.
+
+Verified operations: actual download, load, streamed chat, native measurements, two-model
+benchmarks, one-model-at-a-time switching, unload, active cancellation, worker restart and
+reload. Cancellation also releases the resident model. The package doctor and manifest
+checks pass. Seventeen shared regression tests and fifteen targeted native tests pass.
+
+## Measurements
+
+| Model | Median generation tokens/s | Median first-token time, ms |
+| --- | ---: | ---: |
+| Qwen2.5-0.5B-Instruct-4bit | 425.5 | 96 |
+| Qwen3-0.6B-4bit | 411.9 | 89 |
+
+These are the recorded standalone verification results: a discarded 32-token warm-up,
+three measured runs, fresh prompt cache each time, 2,048-token context, 128-token output
+budget, seed 42, zero temperature, thinking off. These are short warm generation samples,
+not quality rankings or isolated cross-framework comparisons. Native app demonstration
+runs may differ and retain their own timestamps and samples.
+
+Native generation speed is distinct from client-observed first-token time. Visible-output
+timing excludes hidden reasoning and leading whitespace. Memory is MLX allocation, not RSS.
+The same base model with different quantization formats does not establish identical weights.
+Ollama's possible prefix-cache reuse differs from this measurement protocol.
+
+## Decisions for later
+
+Use the [engine-first product direction](../architecture.md). This second adapter proved
+that we can reuse the queue, agent runner, inventory schema and viewer while preserving
+engine-specific metrics and behavior. The current bundle is deterministic: parent shared
+sources are copied by `scripts/build-mlx.mjs`, with MLX branding and measurement copy.
+
+One model per workspace keeps initial ownership and cancellation clear. Concurrent serving,
+quantization/conversion, LoRA, configurable context/KV cache, memory-fit tuning and structured
+output are possible later slices, not currently exposed features. Do not claim parity with
+MLX-LM's full CLI or every model supported by upstream.
+
+The tiny model produces real text but has limited reasoning and factual accuracy. This
+validation proves operation and measurement, not useful answer quality for difficult tasks.
+
+Evidence: [live checks](verification.json), [raw benchmarks](benchmarks.json),
+[versions](versions.json), [package guide](../../../../store/agents/mlx-lm/README.md).
diff --git a/docs/research/local-ai-harnesses/mlx-lm/verification.json b/docs/research/local-ai-harnesses/mlx-lm/verification.json
new file mode 100644
index 000000000..6896be3cb
--- /dev/null
+++ b/docs/research/local-ai-harnesses/mlx-lm/verification.json
@@ -0,0 +1,238 @@
+{
+ "at": "2026-09-20T04:38:50.327Z",
+ "health": {
+ "application": "local-ai-harnesses",
+ "protocol": 1,
+ "workspace": "<pilot-checkout>/mlx-lm",
+ "runtime": "mlx-lm",
+ "pid": 43539
+ },
+ "runtime": {
+ "id": "mlx-lm",
+ "name": "MLX-LM",
+ "online": true,
+ "installed": true,
+ "version": "0.31.3",
+ "mlxVersion": "0.32.2",
+ "endpoint": "Private local worker",
+ "cacheRoot": "<home>/.cache/huggingface/hub",
+ "memoryScope": "MLX active allocations; one model per workspace",
+ "models": [
+ {
+ "id": "mlx-community/Qwen2.5-0.5B-Instruct-4bit",
+ "name": "Qwen2.5-0.5B-Instruct-4bit",
+ "size": 289598797,
+ "digest": "a5339a4131f135d0fdc6a5c8b5bbed2753bbe0f3",
+ "family": "qwen2",
+ "parameters": null,
+ "quantization": "4-bit",
+ "running": false
+ },
+ {
+ "id": "mlx-community/Qwen2.5-1.5B-Instruct-4bit",
+ "name": "Qwen2.5-1.5B-Instruct-4bit",
+ "size": 880169797,
+ "digest": "8b403126fc14f14cfc99bb4cfa72ecbc129ea677",
+ "family": "qwen2",
+ "parameters": null,
+ "quantization": "4-bit",
+ "running": false
+ },
+ {
+ "id": "mlx-community/Qwen2.5-7B-Instruct-4bit",
+ "name": "Qwen2.5-7B-Instruct-4bit",
+ "size": 4295887610,
+ "digest": "c26a38f6a37d0a51b4e9a1eb3026530fa35d9fed",
+ "family": "qwen2",
+ "parameters": null,
+ "quantization": "4-bit",
+ "running": false
+ },
+ {
+ "id": "mlx-community/Qwen3-0.6B-4bit",
+ "name": "Qwen3-0.6B-4bit",
+ "size": 351383618,
+ "digest": "73e3e38d981303bc594367cd910ea6eb48349da8",
+ "family": "qwen3",
+ "parameters": null,
+ "quantization": "4-bit",
+ "running": true
+ },
+ {
+ "id": "mlx-community/SmolLM2-135M-Instruct",
+ "name": "SmolLM2-135M-Instruct",
+ "size": 273874876,
+ "digest": "422de227b90002f443a21a58b1087f6ee7632731",
+ "family": "llama",
+ "parameters": null,
+ "quantization": "bfloat16",
+ "running": false
+ }
+ ]
+ },
+ "system": {
+ "platform": "darwin",
+ "arch": "arm64",
+ "chip": "Apple M2 Max",
+ "totalMemory": 68719476736,
+ "cores": 12,
+ "freeMemory": 4732207104
+ },
+ "checks": [
+ "asset routes",
+ "deploy",
+ "real streamed chat",
+ "native metrics",
+ "3-sample benchmarks",
+ "single-model switching",
+ "unload",
+ "active cancellation releases model",
+ "worker recovery and reload"
+ ],
+ "jobs": [
+ {
+ "id": "7629e096-8d6e-48bb-9df5-39a54008be9e",
+ "type": "deploy",
+ "model": "mlx-community/Qwen3-0.6B-4bit",
+ "models": [],
+ "status": "succeeded",
+ "progress": 100,
+ "message": "mlx-community/Qwen3-0.6B-4bit is ready on this Mac. Ask a question, or say “benchmark it”. MLX-LM keeps one model loaded per workspace. Switching models releases the previous model; idle models unload after 15 minutes.",
+ "createdAt": "2026-09-20T04:38:43.919Z",
+ "output": "",
+ "result": {
+ "model": "mlx-community/Qwen3-0.6B-4bit",
+ "message": "mlx-community/Qwen3-0.6B-4bit is ready on this Mac. Ask a question, or say “benchmark it”. MLX-LM keeps one model loaded per workspace. Switching models releases the previous model; idle models unload after 15 minutes."
+ },
+ "startedAt": "2026-09-20T04:38:43.920Z",
+ "activeModel": "mlx-community/Qwen3-0.6B-4bit",
+ "completedAt": "2026-09-20T04:38:44.385Z"
+ },
+ {
+ "id": "5ff5f363-0000-418f-9bbf-5aaae0e51586",
+ "type": "chat",
+ "model": "mlx-community/Qwen3-0.6B-4bit",
+ "models": [],
+ "status": "succeeded",
+ "progress": 100,
+ "message": "Response from mlx-community/Qwen3-0.6B-4bit.",
+ "createdAt": "2026-09-20T04:38:44.400Z",
+ "output": "Apple Silicon Unified Memory is a memory structure where each core can access it independently, allowing for parallel processing in the same memory space.",
+ "result": {
+ "model": "mlx-community/Qwen3-0.6B-4bit",
+ "metrics": {
+ "model": "mlx-community/Qwen3-0.6B-4bit",
+ "tokens": 238,
+ "promptTokens": 19,
+ "tokensPerSecond": 398.5474512479057,
+ "promptTokensPerSecond": 294.026995000332,
+ "nativeFirstTokenMs": 132.54183297976851,
+ "nativeFirstOutputMs": 662.7784579759464,
+ "peakMemoryBytes": 445546784,
+ "thinking": true,
+ "doneReason": "stop",
+ "wallMs": 736.6918330080807,
+ "measurement": "MLX-LM native generation_tps; TTFT includes local IPC; fresh prompt cache",
+ "firstTokenMs": 142.81645800000115,
+ "firstOutputMs": 673.0794160000005
+ },
+ "message": "Response from mlx-community/Qwen3-0.6B-4bit."
+ },
+ "startedAt": "2026-09-20T04:38:44.401Z",
+ "activeModel": "mlx-community/Qwen3-0.6B-4bit",
+ "completedAt": "2026-09-20T04:38:45.169Z"
+ },
+ {
+ "id": "368f5338-14b8-4afd-91fa-9566ad05da61",
+ "type": "benchmark",
+ "model": null,
+ "models": [
+ "mlx-community/Qwen3-0.6B-4bit",
+ "mlx-community/Qwen2.5-0.5B-Instruct-4bit"
+ ],
+ "status": "succeeded",
+ "progress": 100,
+ "message": "mlx-community/Qwen3-0.6B-4bit: 411.9 tokens/s · 89 ms to first token (median of 3 warm runs).\nmlx-community/Qwen2.5-0.5B-Instruct-4bit: 425.5 tokens/s · 96 ms to first token (median of 3 warm runs).",
+ "createdAt": "2026-09-20T04:38:45.182Z",
+ "output": "",
+ "result": {
+ "model": "mlx-community/Qwen2.5-0.5B-Instruct-4bit",
+ "benchmarkIds": [
+ "d0b3884b-aeb7-415d-8751-d8debc189886",
+ "3c5bc697-2bbc-4009-975a-cc4da6257e88"
+ ],
+ "message": "mlx-community/Qwen3-0.6B-4bit: 411.9 tokens/s · 89 ms to first token (median of 3 warm runs).\nmlx-community/Qwen2.5-0.5B-Instruct-4bit: 425.5 tokens/s · 96 ms to first token (median of 3 warm runs)."
+ },
+ "startedAt": "2026-09-20T04:38:45.183Z",
+ "activeModel": "mlx-community/Qwen2.5-0.5B-Instruct-4bit",
+ "completedAt": "2026-09-20T04:38:48.597Z"
+ },
+ {
+ "id": "482e8956-c158-49c8-bba7-699f7e560b62",
+ "type": "unload_all",
+ "model": null,
+ "models": [],
+ "status": "succeeded",
+ "progress": 100,
+ "message": "mlx-community/Qwen2.5-0.5B-Instruct-4bit unloaded. Downloaded files are still on disk.",
+ "createdAt": "2026-09-20T04:38:48.623Z",
+ "output": "",
+ "result": {
+ "message": "mlx-community/Qwen2.5-0.5B-Instruct-4bit unloaded. Downloaded files are still on disk."
+ },
+ "startedAt": "2026-09-20T04:38:48.624Z",
+ "completedAt": "2026-09-20T04:38:48.661Z"
+ },
+ {
+ "id": "017f5134-08ad-4360-ae4b-f3cdfe50ebf6",
+ "type": "deploy",
+ "model": "mlx-community/Qwen3-0.6B-4bit",
+ "models": [],
+ "status": "succeeded",
+ "progress": 100,
+ "message": "mlx-community/Qwen3-0.6B-4bit is ready on this Mac. Ask a question, or say “benchmark it”. MLX-LM keeps one model loaded per workspace. Switching models releases the previous model; idle models unload after 15 minutes.",
+ "createdAt": "2026-09-20T04:38:48.690Z",
+ "output": "",
+ "result": {
+ "model": "mlx-community/Qwen3-0.6B-4bit",
+ "message": "mlx-community/Qwen3-0.6B-4bit is ready on this Mac. Ask a question, or say “benchmark it”. MLX-LM keeps one model loaded per workspace. Switching models releases the previous model; idle models unload after 15 minutes."
+ },
+ "startedAt": "2026-09-20T04:38:48.690Z",
+ "activeModel": "mlx-community/Qwen3-0.6B-4bit",
+ "completedAt": "2026-09-20T04:38:49.077Z"
+ },
+ {
+ "id": "9e50e9ce-fbed-4a0c-a40e-9478f2622c11",
+ "type": "chat",
+ "model": "mlx-community/Qwen3-0.6B-4bit",
+ "models": [],
+ "status": "cancelled",
+ "progress": 90,
+ "message": "Cancelled. Any downloaded files stay cached; retrying can resume the download.",
+ "createdAt": "2026-09-20T04:38:49.081Z",
+ "output": "",
+ "result": null,
+ "startedAt": "2026-09-20T04:38:49.082Z",
+ "activeModel": "mlx-community/Qwen3-0.6B-4bit",
+ "completedAt": "2026-09-20T04:38:49.114Z"
+ },
+ {
+ "id": "5d644f1a-fa03-4894-975e-2ef8413d2b51",
+ "type": "deploy",
+ "model": "mlx-community/Qwen3-0.6B-4bit",
+ "models": [],
+ "status": "succeeded",
+ "progress": 100,
+ "message": "mlx-community/Qwen3-0.6B-4bit is ready on this Mac. Ask a question, or say “benchmark it”. MLX-LM keeps one model loaded per workspace. Switching models releases the previous model; idle models unload after 15 minutes.",
+ "createdAt": "2026-09-20T04:38:49.930Z",
+ "output": "",
+ "result": {
+ "model": "mlx-community/Qwen3-0.6B-4bit",
+ "message": "mlx-community/Qwen3-0.6B-4bit is ready on this Mac. Ask a question, or say “benchmark it”. MLX-LM keeps one model loaded per workspace. Switching models releases the previous model; idle models unload after 15 minutes."
+ },
+ "startedAt": "2026-09-20T04:38:49.931Z",
+ "activeModel": "mlx-community/Qwen3-0.6B-4bit",
+ "completedAt": "2026-09-20T04:38:50.314Z"
+ }
+ ]
+}
\ No newline at end of file
diff --git a/docs/research/local-ai-harnesses/mlx-lm/versions.json b/docs/research/local-ai-harnesses/mlx-lm/versions.json
new file mode 100644
index 000000000..2e3d9bf6e
--- /dev/null
+++ b/docs/research/local-ai-harnesses/mlx-lm/versions.json
@@ -0,0 +1,4 @@
+{
+ "mlx-lm": "0.31.3",
+ "mlx": "0.32.2"
+}
diff --git a/docs/research/local-ai-harnesses/pilot-benchmarks.json b/docs/research/local-ai-harnesses/pilot-benchmarks.json
new file mode 100644
index 000000000..107e75402
--- /dev/null
+++ b/docs/research/local-ai-harnesses/pilot-benchmarks.json
@@ -0,0 +1,259 @@
+[
+ {
+ "id": "b7c69a02-084f-442c-8c5b-7a361d73fc07",
+ "model": "gemma3:1b",
+ "digest": "8648f39daa8fbf5b18c7b4e6a8fb4990c692751d49917417b8842ca5758e7ffc",
+ "at": "2026-09-20T02:39:24.639Z",
+ "runtime": {
+ "name": "Ollama",
+ "version": "0.34.0"
+ },
+ "hardware": {
+ "platform": "darwin",
+ "arch": "arm64",
+ "chip": "Apple M2 Max",
+ "totalMemory": 68719476736,
+ "cores": 12
+ },
+ "settings": {
+ "context": 2048,
+ "maxTokens": 128,
+ "seed": 42,
+ "temperature": 0,
+ "thinking": false
+ },
+ "prompt": "Explain how a bicycle works to a curious beginner. Discuss the pedals, chain, gears, wheels, brakes, and balance. Write a clear explanation of at least 250 words without a heading.",
+ "promptHash": "e3aff106a75867995dfbb42b73d60cb031d6d1d9511c155329d29033854e829a",
+ "mode": "warm; repeated prompt may use prefix cache",
+ "samples": [
+ {
+ "model": "gemma3:1b",
+ "tokens": 128,
+ "promptTokens": 51,
+ "tokensPerSecond": 152.5140390364449,
+ "promptTokensPerSecond": 2825.1717261245294,
+ "loadMs": 0.9075,
+ "evalMs": 839.267,
+ "doneReason": "length",
+ "firstTokenMs": 22.768708000001425,
+ "firstOutputMs": 22.76887500000157,
+ "elapsedMs": 861.9158330000009,
+ "context": 2048,
+ "maxTokens": 128,
+ "temperature": 0,
+ "seed": 42,
+ "thinking": null
+ },
+ {
+ "model": "gemma3:1b",
+ "tokens": 128,
+ "promptTokens": 51,
+ "tokensPerSecond": 154.88136450482128,
+ "promptTokensPerSecond": 2105.0024764735017,
+ "loadMs": 0.750458,
+ "evalMs": 826.439,
+ "doneReason": "length",
+ "firstTokenMs": 29.213208000001032,
+ "firstOutputMs": 29.213417000000845,
+ "elapsedMs": 855.3763330000002,
+ "context": 2048,
+ "maxTokens": 128,
+ "temperature": 0,
+ "seed": 42,
+ "thinking": null
+ },
+ {
+ "model": "gemma3:1b",
+ "tokens": 128,
+ "promptTokens": 51,
+ "tokensPerSecond": 142.72127093291766,
+ "promptTokensPerSecond": 3196.690485144791,
+ "loadMs": 0.945875,
+ "evalMs": 896.853,
+ "doneReason": "length",
+ "firstTokenMs": 20.11791599999924,
+ "firstOutputMs": 20.118124999999054,
+ "elapsedMs": 916.9944999999989,
+ "context": 2048,
+ "maxTokens": 128,
+ "temperature": 0,
+ "seed": 42,
+ "thinking": null
+ }
+ ],
+ "medianTokensPerSecond": 152.5140390364449,
+ "medianFirstTokenMs": 22.768708000001425,
+ "medianFirstOutputMs": 22.76887500000157,
+ "residentBytes": 883603537
+ },
+ {
+ "id": "5fecc978-ca9a-42b0-9134-dd146aaec770",
+ "model": "qwen3:0.6b",
+ "digest": "7df6b6e09427a769808717c0a93cadc4ae99ed4eb8bf5ca557c90846becea435",
+ "at": "2026-09-20T02:39:19.316Z",
+ "runtime": {
+ "name": "Ollama",
+ "version": "0.34.0"
+ },
+ "hardware": {
+ "platform": "darwin",
+ "arch": "arm64",
+ "chip": "Apple M2 Max",
+ "totalMemory": 68719476736,
+ "cores": 12
+ },
+ "settings": {
+ "context": 2048,
+ "maxTokens": 128,
+ "seed": 42,
+ "temperature": 0,
+ "thinking": false
+ },
+ "prompt": "Explain how a bicycle works to a curious beginner. Discuss the pedals, chain, gears, wheels, brakes, and balance. Write a clear explanation of at least 250 words without a heading.",
+ "promptHash": "e3aff106a75867995dfbb42b73d60cb031d6d1d9511c155329d29033854e829a",
+ "mode": "warm; repeated prompt may use prefix cache",
+ "samples": [
+ {
+ "model": "qwen3:0.6b",
+ "tokens": 128,
+ "promptTokens": 58,
+ "tokensPerSecond": 277.06887109587234,
+ "promptTokensPerSecond": 11924.342105263157,
+ "loadMs": 0.907042,
+ "evalMs": 461.979,
+ "doneReason": "length",
+ "firstTokenMs": 9.088958999999704,
+ "firstOutputMs": 9.089334000000235,
+ "elapsedMs": 471.1010839999999,
+ "context": 2048,
+ "maxTokens": 128,
+ "temperature": 0,
+ "seed": 42,
+ "thinking": false
+ },
+ {
+ "model": "qwen3:0.6b",
+ "tokens": 128,
+ "promptTokens": 58,
+ "tokensPerSecond": 290.56041840700254,
+ "promptTokensPerSecond": 13116.236996834012,
+ "loadMs": 0.752667,
+ "evalMs": 440.528,
+ "doneReason": "length",
+ "firstTokenMs": 10.607291999999688,
+ "firstOutputMs": 10.607417000000169,
+ "elapsedMs": 451.0408749999997,
+ "context": 2048,
+ "maxTokens": 128,
+ "temperature": 0,
+ "seed": 42,
+ "thinking": false
+ },
+ {
+ "model": "qwen3:0.6b",
+ "tokens": 128,
+ "promptTokens": 58,
+ "tokensPerSecond": 290.91305790533505,
+ "promptTokensPerSecond": 15705.3885729759,
+ "loadMs": 0.723917,
+ "evalMs": 439.994,
+ "doneReason": "length",
+ "firstTokenMs": 7.3243750000001455,
+ "firstOutputMs": 7.324542000000292,
+ "elapsedMs": 448.0642090000001,
+ "context": 2048,
+ "maxTokens": 128,
+ "temperature": 0,
+ "seed": 42,
+ "thinking": false
+ }
+ ],
+ "medianTokensPerSecond": 290.56041840700254,
+ "medianFirstTokenMs": 9.088958999999704,
+ "medianFirstOutputMs": 9.089334000000235,
+ "residentBytes": 793719603
+ },
+ {
+ "id": "62251a02-7b91-4291-9428-f5c3365605eb",
+ "model": "qwen3:0.6b",
+ "digest": "7df6b6e09427a769808717c0a93cadc4ae99ed4eb8bf5ca557c90846becea435",
+ "at": "2026-09-20T02:25:27.450Z",
+ "runtime": {
+ "name": "Ollama",
+ "version": "0.34.0"
+ },
+ "hardware": {
+ "platform": "darwin",
+ "arch": "arm64",
+ "chip": "Apple M2 Max",
+ "totalMemory": 68719476736,
+ "cores": 12
+ },
+ "settings": {
+ "context": 2048,
+ "maxTokens": 128,
+ "seed": 42,
+ "temperature": 0
+ },
+ "prompt": "Explain how a bicycle works to a curious beginner. Discuss the pedals, chain, gears, wheels, brakes, and balance. Write a clear explanation of at least 250 words without a heading.",
+ "promptHash": "e3aff106a75867995dfbb42b73d60cb031d6d1d9511c155329d29033854e829a",
+ "mode": "warm; repeated prompt may use prefix cache",
+ "samples": [
+ {
+ "model": "qwen3:0.6b",
+ "tokens": 128,
+ "promptTokens": 58,
+ "tokensPerSecond": 276.84474849303456,
+ "promptTokensPerSecond": 11473.788328387736,
+ "firstTokenMs": 9.06441600015387,
+ "firstOutputMs": 9.064750000135973,
+ "elapsedMs": 471.8668750000652,
+ "loadMs": 0.809583,
+ "evalMs": 462.353,
+ "context": 2048,
+ "maxTokens": 128,
+ "temperature": 0,
+ "seed": 42,
+ "doneReason": "length"
+ },
+ {
+ "model": "qwen3:0.6b",
+ "tokens": 128,
+ "promptTokens": 58,
+ "tokensPerSecond": 266.45294919699825,
+ "promptTokensPerSecond": 13027.85265049416,
+ "firstTokenMs": 10.354583000065759,
+ "firstOutputMs": 10.355125000001863,
+ "elapsedMs": 490.79725000006147,
+ "loadMs": 0.622792,
+ "evalMs": 480.385,
+ "context": 2048,
+ "maxTokens": 128,
+ "temperature": 0,
+ "seed": 42,
+ "doneReason": "length"
+ },
+ {
+ "model": "qwen3:0.6b",
+ "tokens": 128,
+ "promptTokens": 58,
+ "tokensPerSecond": 278.56486861749124,
+ "promptTokensPerSecond": 14602.21550855992,
+ "firstTokenMs": 8.22295799991116,
+ "firstOutputMs": 8.223041999852285,
+ "elapsedMs": 467.6280829999596,
+ "loadMs": 0.705416,
+ "evalMs": 459.498,
+ "context": 2048,
+ "maxTokens": 128,
+ "temperature": 0,
+ "seed": 42,
+ "doneReason": "length"
+ }
+ ],
+ "medianTokensPerSecond": 276.84474849303456,
+ "medianFirstTokenMs": 9.06441600015387,
+ "medianFirstOutputMs": 9.064750000135973,
+ "residentBytes": 793719603
+ }
+]
diff --git a/docs/research/local-ai-harnesses/pilot-findings.md b/docs/research/local-ai-harnesses/pilot-findings.md
new file mode 100644
index 000000000..57498603d
--- /dev/null
+++ b/docs/research/local-ai-harnesses/pilot-findings.md
@@ -0,0 +1,150 @@
+# Ollama pilot findings
+
+Date: September 19, 2026 (America/New_York; evidence timestamps are September 20 UTC).
+
+## Decision and scope
+
+The user chose **one working harness first**, then a decision about the rest. Ollama was
+selected because it was installed on this Mac and exposes native model lifecycle and
+inference APIs. No other framework adapter was implemented or installed. The original
+six core and two experimental candidates remain in [the research report](recommendation.md).
+
+The intended product is a domain-specific Harness agent: natural-language conversation
+with Codex and a model/performance viewer on the left. Each future adapter should use its
+framework's supported native CLI and/or API. A CLI-only requirement would discard useful
+structured streaming and timing data. For Ollama, the CLI starts the daemon; its local API
+handles downloads, residency, chat, and measurement.
+
+The pilot is registered locally as `local/ollama`. Its package passes the spec-1 checker
+and dependency doctor. It appears as **Ollama, by Local** in OpenHarness's agent picker.
+The same `harness.json`/viewer launch contract used by the existing Grid package is reused.
+Grid's telemetry collector is not a drop-in Ollama adapter, so its fleet-specific data
+schema and UI were not copied. The embedded dashboard has no duplicate chat pane. A second view of the same
+workspace reuses the existing controller through a loopback relay. Its HTTP page,
+shared controller, saved benchmarks and JSON export were verified.
+
+## What actually ran
+
+Host: Apple M2 Max, 64 GiB unified memory, macOS 26.6.2. Runtime: Ollama 0.34.0.
+
+- Started the installed Ollama service locally with cloud disabled.
+- Downloaded Qwen 3 0.6B and Gemma 3 1B through the harness, and verified their residency.
+- Unloaded and reloaded Qwen, confirming both states in Ollama's inventory.
+- Streamed local generation and captured native token counts/timing.
+- Verified that Qwen with reasoning enabled answered the arithmetic sanity prompt with `4`.
+- Ran both models through the same three-run warm benchmark; saved raw samples, model
+ digests, runtime version, settings, timestamps and hardware.
+- Checked cancellation, failed jobs, stream parsing, restart recovery, request boundaries,
+ and metric unit conversions with 14 automated tests.
+
+The sanity prompt is only an integration check. It is not a model-quality evaluation.
+
+## Measured baseline
+
+Latest matched-setting runs from 2026-09-20 02:39 UTC:
+
+| Model | Median generation speed | Median first token | Ollama allocation |
+| --- | ---: | ---: | ---: |
+| qwen3:0.6b | 290.6 tokens/s | 9.1 ms | 0.74 GiB |
+| gemma3:1b | 152.5 tokens/s | 22.8 ms | 0.82 GiB |
+
+Each used one discarded 32-token warm-up followed by three measured runs of one fixed
+bicycle-explanation prompt: 2,048-token context, maximum 128 generated tokens, temperature
+0, seed 42. Qwen thinking was disabled for this speed baseline. Gemma does not expose
+that reasoning mode. Prefix caches may be warm, so these first-token times must not be
+advertised as cold-start latency. Both demo models were resident during the comparison;
+other desktop workloads were not controlled. These results are observations on this Mac,
+not cross-framework or hardware rankings.
+
+Ollama's allocation is not physical dedicated VRAM, total process RSS, or peak memory.
+Missing performance/temperature/utilization data is not invented. The first Qwen baseline
+was 276.8 tokens/s; the later matched comparison was 290.6. The recorded variation is one
+reason to retain raw samples and timestamps rather than presenting a single universal rate.
+
+Evidence: [all benchmark samples](pilot-benchmarks.json),
+[deployment and chat check](pilot-verification.json).
+
+## Lessons from real failures
+
+1. **A failed download needs a retry path.** The first Qwen download hit an upstream TLS
+ handshake timeout. The next attempt succeeded using Ollama's cached partial download.
+ The harness showed failure rather than claiming deployment. User-facing errors now
+ omit signed URL query strings and offer a retry; raw daemon logs stay local.
+2. **On-disk does not mean usable.** The existing `gpt-oss:20b` model appeared in inventory
+ but failed to load with `tensor "blk.0.ffn_down_exps.weight" size overflow`. Its files
+ were left untouched. The cause was not established; this is not evidence that all
+ GPT-OSS models or all Ollama installations fail. That model was not benchmarked.
+3. **Fast settings can hurt answers.** On the prompt “What is 2 + 2? Reply with just the
+ number.” Qwen 3 0.6B returned `2` with reasoning disabled, including a direct native API
+ reproduction. With reasoning enabled, the native API and the final harness both
+ returned `4`. Chat now enables reasoning where available. Benchmarks explicitly record
+ their own mode. A fast rate must never be presented as a quality score.
+4. **First token and visible answer differ.** The verified reasoning chat produced its
+ first token around 33 ms and first visible output around 350 ms. Both are saved.
+ Generated-token counts can include reasoning even when only visible text is displayed.
+5. **The conversation should remain the agent.** The embedded viewer provides observations
+ and model controls beside Codex. The standalone browser command panel is optional and
+ uses a bounded phrase parser; it is not a general autonomous language planner.
+6. **Native packaging matters.** A browser dashboard alone does not provide the expected
+ Harness experience. The package now supplies a workspace template, agent instructions,
+ a workspace-aware runner, setup/doctor commands, a viewer launch command, and verdicts.
+ Dependencies can use Harness's managed Node for GUI launches without relying on shell
+ profiles. The pilot does not publish a hosted service or require a paid inference key.
+
+## Validation limits
+
+- The native package was installed and verified visible in the OpenHarness agent picker.
+ The initial browser preview rendered, but Chrome later blocked the preview URL with
+ `ERR_BLOCKED_BY_CLIENT`. No browser security setting was changed. The follow-up on
+ 2026-09-19 verified the updated viewer inside the native left pane, including the
+ official Ollama logo and model-specific error explanation.
+- No full browser interaction or responsive screenshot test suite was run. Backend actions
+ were exercised through the same API/runner used by the viewer.
+- WebMCP registration is feature-detected; no supported WebMCP execution context was
+ available. Its optional tools are implemented but not verified in a host.
+- HTTP behavior is local only. Multi-machine orchestration, shared queues across separate
+ workspaces, peak-RAM sampling, energy, temperatures, and cold-start benchmarks are deferred.
+- Starting a model service is supported; installing Ollama itself is outside this Mac's
+ pilot because the official runtime was already present. On a machine without it,
+ doctor/setup reports the missing runtime instead of pretending readiness.
+
+## What to keep before adding another adapter
+
+Keep the framework adapter separate from jobs, intent resolution, stored measurements,
+and presentation. Require each adapter to expose discovery, readiness, download, load,
+unload, generate, cancellation, and honest capability/missing-metric reporting. Native
+CLI/API choices belong inside the adapter. Preserve failure and restart states, and verify
+one real inference before declaring a first deployment successful.
+
+The user's next decision is whether this experience feels useful. If it does, **llama.cpp**
+adds a useful independent GGUF/Metal baseline; **LM Studio/llmster** adds a different model
+management flow; **MLX-LM** adds a direct Apple Silicon comparison. Revisit the original
+research before choosing. oMLX, vLLM Metal and the experimental engines are still candidates,
+not a locked release plan.
+
+## Follow-up: identity, category and reported error (2026-09-19)
+
+- The viewer remains Ollama-specific; it now uses the official Ollama llama mark,
+ Ollama copy, a favicon and app icon. Shared jobs and charts are reusable foundations,
+ not a claim that other runtimes are already implemented.
+- OpenHarness desktop adds a Local AI shelf with a brain/circuit icon. It recognizes
+ older `autonomous/autonomous-grid` and `local/ollama` catalog entries that used Compute,
+ and any new package tagged Local AI. Ollama and the Grid kit manifests use Local AI.
+- The reported red badge belongs to the materialized workspace
+ `ollama-2026-09-19-23-03`, whose GPT-OSS benchmark failed. The viewer now includes a
+ visible explanation, the model name and original error; the failed job remains intact.
+- The existing GPT-OSS weights match SHA-256
+ `b112e727c6f18875636c56a779790a590d705aec9e1c0eb5a97d51fc2a778583`
+ and the manifest size of 13,780,154,624 bytes. Ollama 0.34.0 rejects them with a
+ tensor-size overflow. An isolated test with the checksum-verified official 0.34.2
+ Darwin CLI also fails during `/api/show`; no inference is required to reproduce it.
+ The test server was stopped. The existing runtime and model cache were not replaced.
+- Next recommendation for this Mac: MLX-LM. Its Apple Silicon focus and native CLI/Python
+ interfaces make it a useful independent runtime comparison, before committing to
+ newer MLX server wrappers. No second harness has been started or implicitly selected.
+ Source: https://github.com/ml-explore/mlx-lm
+
+Native follow-up verification: the rebuilt app opened successfully. The Local AI shelf
+visibly lists both Grid and Ollama. Ollama's real logo appears in its store card, tab,
+viewer header and dashboard. The viewer displays live inventory and the preserved
+GPT-OSS failure. Fifteen targeted native tests and fourteen harness tests passed.
diff --git a/docs/research/local-ai-harnesses/pilot-verification.json b/docs/research/local-ai-harnesses/pilot-verification.json
new file mode 100644
index 000000000..dfa73dd8f
--- /dev/null
+++ b/docs/research/local-ai-harnesses/pilot-verification.json
@@ -0,0 +1,168 @@
+{
+ "checkedAt": "2026-09-20T02:39:27.424Z",
+ "runtime": "0.34.0",
+ "hardware": {
+ "platform": "darwin",
+ "arch": "arm64",
+ "chip": "Apple M2 Max",
+ "totalMemory": 68719476736,
+ "cores": 12,
+ "freeMemory": 2098790400
+ },
+ "model": {
+ "id": "qwen3:0.6b",
+ "name": "qwen3:0.6b",
+ "size": 522653767,
+ "digest": "7df6b6e09427a769808717c0a93cadc4ae99ed4eb8bf5ca557c90846becea435",
+ "modifiedAt": "2026-09-19T22:24:53.944608106-04:00",
+ "family": "qwen3",
+ "parameters": "751.63M",
+ "quantization": "Q4_K_M",
+ "running": true,
+ "resident": {
+ "name": "qwen3:0.6b",
+ "model": "qwen3:0.6b",
+ "size": 1039086387,
+ "digest": "7df6b6e09427a769808717c0a93cadc4ae99ed4eb8bf5ca557c90846becea435",
+ "details": {
+ "parent_model": "",
+ "format": "gguf",
+ "family": "qwen3",
+ "families": [
+ "qwen3"
+ ],
+ "parameter_size": "751.63M",
+ "quantization_level": "Q4_K_M"
+ },
+ "expires_at": "2026-09-19T22:54:27.286847-04:00",
+ "size_vram": 1039086387,
+ "context_length": 4096
+ }
+ },
+ "checks": [
+ "unload verified in runtime inventory",
+ "cached deployment verified resident",
+ "chat arithmetic sanity check",
+ "generation metrics present",
+ "three-run measured benchmark persisted"
+ ],
+ "chat": {
+ "prompt": "What is 2 + 2? Reply with just the number.",
+ "output": "4",
+ "metrics": {
+ "model": "qwen3:0.6b",
+ "tokens": 90,
+ "promptTokens": 24,
+ "tokensPerSecond": 272.4292516368457,
+ "promptTokensPerSecond": 1213.4695115785216,
+ "loadMs": 0.636167,
+ "evalMs": 330.361,
+ "doneReason": "stop",
+ "firstTokenMs": 33.03279199999997,
+ "firstOutputMs": 350.1375420000004,
+ "elapsedMs": 354.0685420000009,
+ "context": 4096,
+ "maxTokens": 512,
+ "temperature": 0.3,
+ "seed": 42,
+ "thinking": true
+ }
+ },
+ "benchmark": {
+ "id": "5fecc978-ca9a-42b0-9134-dd146aaec770",
+ "model": "qwen3:0.6b",
+ "digest": "7df6b6e09427a769808717c0a93cadc4ae99ed4eb8bf5ca557c90846becea435",
+ "at": "2026-09-20T02:39:19.316Z",
+ "runtime": {
+ "name": "Ollama",
+ "version": "0.34.0"
+ },
+ "hardware": {
+ "platform": "darwin",
+ "arch": "arm64",
+ "chip": "Apple M2 Max",
+ "totalMemory": 68719476736,
+ "cores": 12
+ },
+ "settings": {
+ "context": 2048,
+ "maxTokens": 128,
+ "seed": 42,
+ "temperature": 0,
+ "thinking": false
+ },
+ "prompt": "Explain how a bicycle works to a curious beginner. Discuss the pedals, chain, gears, wheels, brakes, and balance. Write a clear explanation of at least 250 words without a heading.",
+ "promptHash": "e3aff106a75867995dfbb42b73d60cb031d6d1d9511c155329d29033854e829a",
+ "mode": "warm; repeated prompt may use prefix cache",
+ "samples": [
+ {
+ "model": "qwen3:0.6b",
+ "tokens": 128,
+ "promptTokens": 58,
+ "tokensPerSecond": 277.06887109587234,
+ "promptTokensPerSecond": 11924.342105263157,
+ "loadMs": 0.907042,
+ "evalMs": 461.979,
+ "doneReason": "length",
+ "firstTokenMs": 9.088958999999704,
+ "firstOutputMs": 9.089334000000235,
+ "elapsedMs": 471.1010839999999,
+ "context": 2048,
+ "maxTokens": 128,
+ "temperature": 0,
+ "seed": 42,
+ "thinking": false
+ },
+ {
+ "model": "qwen3:0.6b",
+ "tokens": 128,
+ "promptTokens": 58,
+ "tokensPerSecond": 290.56041840700254,
+ "promptTokensPerSecond": 13116.236996834012,
+ "loadMs": 0.752667,
+ "evalMs": 440.528,
+ "doneReason": "length",
+ "firstTokenMs": 10.607291999999688,
+ "firstOutputMs": 10.607417000000169,
+ "elapsedMs": 451.0408749999997,
+ "context": 2048,
+ "maxTokens": 128,
+ "temperature": 0,
+ "seed": 42,
+ "thinking": false
+ },
+ {
+ "model": "qwen3:0.6b",
+ "tokens": 128,
+ "promptTokens": 58,
+ "tokensPerSecond": 290.91305790533505,
+ "promptTokensPerSecond": 15705.3885729759,
+ "loadMs": 0.723917,
+ "evalMs": 439.994,
+ "doneReason": "length",
+ "firstTokenMs": 7.3243750000001455,
+ "firstOutputMs": 7.324542000000292,
+ "elapsedMs": 448.0642090000001,
+ "context": 2048,
+ "maxTokens": 128,
+ "temperature": 0,
+ "seed": 42,
+ "thinking": false
+ }
+ ],
+ "medianTokensPerSecond": 290.56041840700254,
+ "medianFirstTokenMs": 9.088958999999704,
+ "medianFirstOutputMs": 9.089334000000235,
+ "residentBytes": 793719603
+ },
+ "packageValidation": {
+ "installedId": "local/ollama",
+ "spec1Conformance": "passed",
+ "doctor": "passed",
+ "automatedTestsPassed": 14,
+ "nativePicker": "Ollama entry observed as installed on M2",
+ "nativeViewerRelay": "HTTP page, shared controller token, same benchmarks and export verified",
+ "nativePaneRendering": "not verified; app UI changed during workspace selection",
+ "webMCP": "implemented with feature detection; no supported validation context available"
+ }
+}
diff --git a/docs/research/local-ai-harnesses/recommendation.md b/docs/research/local-ai-harnesses/recommendation.md
new file mode 100644
index 000000000..568822eb7
--- /dev/null
+++ b/docs/research/local-ai-harnesses/recommendation.md
@@ -0,0 +1,102 @@
+**Local AI harness shortlist — September 19, 2026**
+
+**Original pilot decision (superseded September 20): build one Ollama pilot, try the experience, and decide the rest afterward.** The user explicitly chose this smaller first step rather than locking the entire list. The pilot uses the installed Ollama CLI and local API, a natural-language agent, and a live Harness viewer. See [pilot findings](pilot-findings.md) for implementation and measurements.
+
+The original proposed list is preserved for later: **Ollama, llama.cpp, LM Studio / llmster, MLX-LM, oMLX, and vLLM Metal**, plus **vllm-mlx and SGLang's MLX backend** as experiments. Ollama and MLX-LM are now implemented. The others remain research candidates, not commitments or tested compatibility claims. The [updated architecture and priorities](architecture.md) supersede the original order below.
+
+The user confirmed that the first release should cover language models and target this Mac. Read-only system inspection identified an **Apple M2 Max with 64 GiB unified memory, running macOS 26.6.2**. Ollama, llama-server, and the LM Studio CLI were already on PATH; their presence does not establish that their servers are running or that their installed versions have the latest features.
+
+The research combines official repositories, project documentation, release notes, and a dated GitHub API snapshot. Stars indicate public interest, not installed users, reliability, or speed. Repository creation dates are not necessarily product launch dates. GitHub's `/releases/latest` identifies the release designated latest by GitHub, and can differ from package registries, nightlies, or current source. Some capabilities below are documented on the main branch and must be verified against the version selected for implementation. No comparative inference benchmarks were run for this research.
+
+**The proposed build order follows practical value on this Mac.**
+
+| Order | Harness | Interest signal | Why include it | Integration and qualification |
+| --- | --- | --- | --- | --- |
+| 1 | [Ollama](https://github.com/ollama/ollama) | 181,281 stars | Broad ecosystem; approachable model discovery and downloads; the most familiar starting point in this shortlist. | Native model-management API and streamed inference. Compute generation speed from reported token counts and timing; distinguish downloaded from resident models. |
+| 2 | [llama.cpp](https://github.com/ggml-org/llama.cpp) | 128,863 stars | Direct GGUF execution with Metal and detailed controls; valuable as a baseline beneath several higher-level products. | Current server supports model routing, loading/unloading, download events, request timings, and optional Prometheus metrics. Detect installed version because commands and management endpoints have evolved. |
+| 3 | [LM Studio / llmster](https://lmstudio.ai/docs/developer/core/headless) | Desktop product; SDK stars are not a valid product comparison | Strong model-management experience and access to both GGUF and MLX engines. Headless operation makes it suitable for automation. | Use llmster to run without the desktop UI. Native REST v1 supports download, load, unload, and status; chat responses include performance statistics. |
+| 4 | [MLX-LM](https://github.com/ml-explore/mlx-lm) | 7,069 stars; underlying [MLX](https://github.com/ml-explore/mlx) has 28,484 | Direct access to Apple's MLX ecosystem and a useful reference for measuring the overhead or benefit of other MLX servers. | Manage an isolated process/environment, model downloads, and local inference server. Use its generation/benchmark statistics and client observations. Server functionality is version-dependent; it already has batching support. |
+| 5 | [oMLX](https://github.com/jundot/omlx) | 21,930 stars; repository created February 2026 | The strongest newer Mac-specific candidate by the combination of public interest and relevant operations features. It offers persistent prompt caching, concurrent requests, and memory-aware model management. | CLI plus OpenAI-compatible inference; admin code exposes model load/unload, downloads, statistics, and benchmark routes. Treat admin routes as version-sensitive rather than assuming a stable external contract. |
+| 6 | [vLLM Metal](https://github.com/vllm-project/vllm-metal) | 1,749 stars; parent vLLM has 92,191 | Brings the upstream vLLM server and scheduler to Apple Silicon, creating a path toward later Linux/GPU support. | A community-maintained plugin in the vLLM organization. Current installation supports macOS 15+ and native arm64 Python 3.12. Qualify supported model families; several features remain experimental. |
+| 7 — experimental | [vllm-mlx](https://github.com/waybarrios/vllm-mlx) | 1,587 stars; repository created December 2025 | An independent Mac server with OpenAI/Anthropic compatibility, batching, prompt caching, metrics, and a built-in serving benchmark. Useful for comparing agent workloads. | It is a separate project from upstream vLLM and vLLM Metal. Keep a separate adapter and runtime identity; do not imply upstream compatibility from its name. |
+| 8 — experimental | [SGLang with MLX](https://github.com/sgl-project/sglang/releases/tag/v0.5.10) | 36,178 stars for SGLang overall | A major serving project whose newer Apple Silicon path makes it relevant to a Mac-first evaluation. | MLX support was included in v0.5.10. Installation, feature coverage, cache behavior, and telemetry must be verified on the Mac backend rather than inferred from CUDA support. |
+
+The first three targets already have command-line executables on this machine, which should shorten initial integration. MLX-LM then establishes a direct Apple Silicon reference. oMLX adds practical model residency and cache management. vLLM Metal connects the Mac experience to a major server ecosystem. The experimental pair gives the product room to compare newer approaches without presenting unverified combinations as fully supported.
+
+**Several recent developments materially affect the choice.**
+
+LM Studio introduced its standalone llmster daemon in the January 28, 2026 release of 0.4.0. Its current API reference recommends `/api/v1/*` and documents model download/load/unload operations. That changes its suitability for an automated harness: we can drive a service directly instead of depending on desktop interaction. [Announcement](https://lmstudio.ai/blog/0.4.0), [current native API](https://lmstudio.ai/docs/developer/rest).
+
+llama.cpp's current server includes a router with model lifecycle operations and server-sent events. A separate swapping proxy is therefore optional for our first release. Some installations still expose the older `llama-server` executable, while the current top-level README also documents `llama serve`; the adapter should probe capabilities. [Server documentation](https://github.com/ggml-org/llama.cpp/blob/master/tools/server/README.md), [current quick start](https://github.com/ggml-org/llama.cpp).
+
+oMLX's repository was created on February 13, 2026, and its latest designated release was 0.6.4 on August 29. Its cache moves reusable attention state between RAM and SSD; this is different from streaming the model's weights from disk. Current source also documents experimental multi-Mac inference, which should remain outside our initial single-Mac scope. [Project](https://github.com/jundot/omlx), [release](https://github.com/jundot/omlx/releases/tag/v0.6.4).
+
+vLLM Metal and vllm-mlx both appeared as repositories in December 2025, but use different integration approaches. vLLM Metal plugs into upstream vLLM; vllm-mlx is an independent MLX serving system. vllm-mlx released 0.5.0 on September 17, 2026, with conversation-cache and streaming improvements. Feature and throughput claims from either project need reproduction on this M2 Max. [vLLM Metal architecture](https://github.com/vllm-project/vllm-metal), [vllm-mlx release](https://github.com/waybarrios/vllm-mlx/releases/tag/v0.5.0).
+
+SGLang's published v0.5.10 release notes explicitly include native MLX execution on Apple Silicon. The latest designated release in the snapshot is v0.5.20, published September 18. It would be outdated to describe the project as categorically unavailable on Macs, but its overall maturity does not establish feature parity for the newer backend. [MLX introduction](https://github.com/sgl-project/sglang/releases/tag/v0.5.10), [recent release](https://github.com/sgl-project/sglang/releases/tag/v0.5.20).
+
+**The broader popularity comparison explains the candidates we would postpone.**
+
+| Candidate | GitHub stars in snapshot | Distinctive value | Recommendation for the agreed Mac/language-model scope |
+| --- | ---: | --- | --- |
+| [vLLM core](https://github.com/vllm-project/vllm) | 92,191 | Large serving ecosystem, scheduling, batching, and observability | Cover this family through vLLM Metal first; add Linux CUDA/ROCm deployment when that hardware is in scope. |
+| [LocalAI](https://github.com/mudler/LocalAI) | 49,177 | Broad local API platform for text, vision, audio, images, and other modalities | Strong later addition when breadth matters; much of its distinctive value is outside the first text-only release. |
+| [exo](https://github.com/exo-explore/exo) | 47,531 | Device discovery, model sharding, and local clusters | Highest-interest later experiment. A second suitable Mac is needed to validate the distinctive clustering experience. |
+| [Jan](https://github.com/janhq/jan) | 44,564 | Open-source desktop assistant with local model downloads and an OpenAI-compatible server | Later compatibility adapter. First-wave engine coverage overlaps, and its desktop experience duplicates some of our proposed surface. |
+| [llamafile](https://github.com/mozilla-ai/llamafile) | 25,997 | Portable execution and model distribution in a single file | Later portable-deployment target. Much of the inference functionality overlaps llama.cpp. |
+| [MLC-LLM](https://github.com/mlc-ai/mlc-llm) | 23,169 | Compilation-based deployment across devices and platforms | Revisit for mobile, browser, or embedded deployment. Adds compilation/package work to this desktop-first project. |
+| [KTransformers](https://github.com/kvcache-ai/ktransformers) | 19,522 | Heterogeneous CPU/GPU execution for large mixture-of-experts models | Revisit for compatible Linux hardware and larger-model experiments; not the default path for this Mac. |
+| [TensorRT-LLM](https://github.com/NVIDIA/TensorRT-LLM) | 14,673 | NVIDIA-specific serving optimizations | Defer until NVIDIA hardware is in scope. |
+| [KoboldCpp](https://github.com/LostRuins/koboldcpp) | 11,804 | Accessible GGUF runner and established interactive-fiction ecosystem | Later if that workflow matters; substantial overlap with the GGUF harness. |
+| [GenieX, formerly Nexa SDK](https://github.com/qualcomm/GenieX) | 8,380 | Current project targets Qualcomm devices across NPU/GPU/CPU | Hardware mismatch for the current scope. The old Nexa repository redirects here; older comparisons can be misleading. |
+| [mistral.rs](https://github.com/EricLBuehler/mistral.rs) | 7,709 | Independent Rust runtime, quantization, Metal support, dynamic model management | Best next independent single-Mac engine after the initial set. A credible alternative if we prefer another architecture over two MLX serving experiments. |
+| [LiteRT-LM](https://github.com/google-ai-edge/LiteRT-LM) | About 6.5k, GitHub page | Google's edge deployment framework | Later for edge/mobile deployment. Exact API count was unavailable after rate limiting. |
+| [Lemonade](https://github.com/lemonade-sdk/lemonade) | 5,748 | Local server with hardware-specific backends; substantial AMD GPU/NPU emphasis and current Mac support | Add when AMD/Windows machines or broader modalities become priorities. |
+| [llama-swap](https://github.com/mostlygeek/llama-swap) | 5,700 | Routes requests and starts/stops upstream model servers; exposes monitoring and MCP integration | Treat as an orchestration integration, not an independent inference engine. First evaluate what we need beyond native runtime lifecycle controls. |
+| [Flash-MoE](https://github.com/danveloper/flash-moe) | 4,145 | Streams large mixture-of-experts model weights from SSD on Apple Silicon | Research track only: narrow model support, substantial storage needs, and additional output-correctness verification. |
+| [ik_llama.cpp](https://github.com/ikawrakow/ik_llama.cpp) | About 3.2k, GitHub page | llama.cpp fork with additional quantizations and optimization work | Later benchmark variant; maintain a distinct engine identity and pinned build. |
+| [RamaLama](https://github.com/containers/ramalama) | 3,053 | Container-oriented model acquisition and deployment | Later for container/server workflows; overlaps the deployment layer we are building. |
+| [ExLlamaV3](https://github.com/turboderp-org/exllamav3) | 1,464 | Quantized inference for modern consumer GPUs | Its documented CUDA requirements make it a later NVIDIA target. |
+| [Parallax](https://github.com/GradientHQ/parallax) | 1,381 | Distributed inference across heterogeneous machines, using existing backend engines | Later multi-machine alternative; additional networking and scheduling scope. |
+| [Hugging Face TGI](https://github.com/huggingface/text-generation-inference) | 10,885 | Established serving system | Exclude from new development: repository archived; README describes maintenance mode and recommends alternatives. |
+
+Image and speech systems were considered during initial discovery, then removed from the first-release scope following the user's choice. ComfyUI and whisper.cpp remain sensible topics for a separate future research pass. Frontends and agent application frameworks should not be counted as distinct inference engines merely because they can call local servers.
+
+**The most interesting experiments have different prerequisites.**
+
+exo is the strongest candidate for a later visual cluster map: its documented behavior includes automatic discovery and splitting a model across machines. The current macOS app requires macOS 26.2+. Its Thunderbolt RDMA path depends on compatible Thunderbolt 5 hardware; this M2 Max MacBook Pro has Thunderbolt 4 and does not qualify for that specific path. A network-based setup is a separate configuration to test, not an equivalent speed promise. [exo setup and RDMA requirements](https://github.com/exo-explore/exo), [Apple hardware specifications](https://support.apple.com/en-us/111838).
+
+Flash-MoE demonstrates an unusual capacity tradeoff: the project reports running Qwen3.5-397B-A17B on a 48 GB M3 Max by streaming approximately 209 GB of weights from SSD. Its reported 4-bit result is about 4.36 tokens/second, and its README notes broken tool-call JSON in a faster 2-bit configuration. These are author-reported results, not independent measurements on this Mac. The original repository's latest push in the snapshot is March 19, despite continuing attention; novelty and current maintenance are separate signals. [Project experiment](https://github.com/danveloper/flash-moe).
+
+mistral.rs is more practical than either cluster or SSD-weight-streaming experiments when the goal is another engine on one Mac. Its documented server includes OpenAI-compatible requests, model status, unload/reload, and metrics. We can substitute it for one of the two experimental MLX adapters if independent engine diversity is more valuable than comparing Mac serving layers. [Project](https://github.com/EricLBuehler/mistral.rs), [model management](https://docs.mistralrs.dev/guides/serve/multiple-models/).
+
+**Every accepted harness should satisfy the same user-facing contract.**
+
+- Detect the runtime, its version, installed models, active server, and supported hardware.
+- Interpret requests such as “run a small coding model,” resolve a concrete model and compatible format, and explain the selection.
+- Download and start that model, expose progress, and report when inference is actually ready.
+- List, load, unload, stop, and restart supported model instances; state unavailable capabilities clearly.
+- Offer a common inference interface and a reproducible benchmark action.
+- Emit model state, errors, logs, memory observations, and performance measurements to the viewer.
+
+Natural-language interpretation should produce typed actions with validated arguments. The runtime adapters then perform those actions through documented APIs or explicit process arguments. A shared inference protocol helps, but OpenAI compatibility alone does not provide model installation, download, lifecycle control, or hardware measurements.
+
+The viewer should distinguish **machine → runtime/engine → model instance**. A model downloaded in two formats or served by two runtimes is two deployments of one model family, not two unrelated models. Useful first views are a live model map, memory occupancy, request activity, and comparable benchmark results.
+
+**Performance needs a common measurement protocol.**
+
+| Measurement | How to make the comparison useful |
+| --- | --- |
+| Time to first token | Record end-to-end client time and server-reported time separately. Identify cold loading, prompt processing, and cache reuse. |
+| Generation tokens/second | Prefer server token counts/timing; identify whether this is one request or aggregate throughput. Never equate stream chunks with tokens. |
+| Prompt-processing speed | Report separately from generation. Record prompt length and cached-token count where available. |
+| Memory | Distinguish process footprint, model allocation, cache allocation, and system pressure. Apple unified memory does not map cleanly to a discrete-GPU VRAM chart. |
+| Concurrency and latency | Benchmark one request and a small concurrent batch; retain errors and p50/p95 latency instead of reporting only the fastest run. |
+| Correctness | Check basic output coherence, JSON conformance, and tool-call validity alongside speed. A faster broken output is not a successful result. |
+| Reproducibility | Store model revision, format, quantization, context length, prompt, output budget, sampling parameters, runtime version, and hardware. |
+
+For the same-model comparisons, use the same base weights and tokenizer where possible. GGUF and MLX quantizations are not automatically numerically equivalent, even when both are described as 4-bit. Mark such results as matched model-family comparisons rather than identical-weight engine comparisons. Repeat cold and warm runs; avoid ranking frameworks from a vendor's best example on different hardware.
+
+Native metrics are available in different forms: Ollama reports token counts and nanosecond timings, LM Studio reports speed and first-token/load timings, llama.cpp exposes request timings and optional Prometheus metrics, and vLLM/SGLang have Prometheus interfaces. MLX-based runtime-specific statistics need adapter normalization. Missing telemetry should appear as unavailable, not as a fabricated zero. [Ollama timings](https://docs.ollama.com/api/generate), [LM Studio statistics](https://lmstudio.ai/docs/developer/rest/chat), [llama.cpp metrics](https://github.com/ggml-org/llama.cpp/blob/master/tools/server/README.md), [vLLM metrics](https://docs.vllm.ai/en/latest/design/metrics/), [SGLang metrics](https://docs.sglang.io/docs/references/production_metrics).
+
+The evidence snapshots are saved beside this report in `github-snapshot.json` and `releases-snapshot.json`. Retrieval timestamps use UTC, which falls on September 20 for this September 19 evening session in New York. Rounded counts cited from GitHub pages are explicitly labeled. The broad ecosystem research is preserved. Follow the [updated architecture and priorities](architecture.md) for current implementation decisions.
diff --git a/docs/research/local-ai-harnesses/releases-snapshot.json b/docs/research/local-ai-harnesses/releases-snapshot.json
new file mode 100644
index 000000000..38e9fb999
--- /dev/null
+++ b/docs/research/local-ai-harnesses/releases-snapshot.json
@@ -0,0 +1,180 @@
+{
+ "retrieved_at": "2026-09-20T01:31:45.703989+00:00",
+ "releases": [
+ {
+ "repo": "ollama/ollama",
+ "tag_name": "v0.34.2",
+ "name": "v0.34.2",
+ "published_at": "2026-09-15T21:21:07Z",
+ "html_url": "https://github.com/ollama/ollama/releases/tag/v0.34.2",
+ "prerelease": false,
+ "body": "## What's Changed\r\n\r\n- Added first-run setup when running `ollama`, with options to sign in or continue locally. Setup completion is shared with the desktop app on macOS and Windows.\r\n- Added `ollama://apps` to open the desktop app\u2019s Apps page directly on macOS and Windows.\r\n- Fixed excessive memory growth during long generations with MLX speculative decoding.\r\n- Updated llama.cpp.\r\n\r\n\r\n**Full Changelog**: https://github.com/ollama/ollama/compare/v0.34.1...v0.34.2"
+ },
+ {
+ "repo": "ggml-org/llama.cpp",
+ "tag_name": "v0.4.1",
+ "name": "v0.4.1",
+ "published_at": "2026-09-14T18:27:29Z",
+ "html_url": "https://github.com/ggml-org/llama.cpp/releases/tag/v0.4.1",
+ "prerelease": false,
+ "body": "## Overview\r\n\r\nllama.cpp 0.4.1 adds Maple 20B-A1B, Tencent Hy 4, and Spark2.5 support, improves JSON schema handling, chat parsing, logging, and server child-process management, and updates ggml to v0.24.0.\r\n\r\n### API changes\r\n\r\n- Changed `llama_sampler_chain_n()` to return `int32_t` instead of `int` ([#28631](https://github.com/ggml-org/llama.cpp/pull/28631)).\r\n- Added `server_subproc` and `waiter` for router child-process monitoring in `server-common.h` ([#28555](https://github.com/ggml-org/llama.cpp/pull/28555)).\r\n\r\n### New models\r\n\r\n- Added Maple 20B-A1B ternary MoE architecture (CPU) ([#27000](https://github.com/ggml-org/llama.cpp/pull/27000)).\r\n- Added Tencent Hy 4 (`hy_v4`) preview architecture support ([#28127](https://github.com/ggml-org/llama.cpp/pull/28127)).\r\n- Added Spark2.5 support ([#27868](https://github.com/ggml-org/llama.cpp/pull/27868)).\r\n\r\n### Core changes\r\n\r\n- Added Kimi-K3 recurrent-state rollback support ([#28466](https://github.com/ggml-org/llama.cpp/pull/28466)).\r\n- Fixed MTP context KV cache allocation for DeepSeek2, GLM-MoE, and related models ([#28630](https://github.com/ggml-org/llama.cpp/pull/28630)).\r\n- Fixed GDN normalization from `max` to `rsqrt` for affected Qwen/Kimi/GLM models ([#28068](https://github.com/ggml-org/llama.cpp/pull/28068)).\r\n- Fixed Granite parameter counts and Granite3 MoE unknown parameter counts ([#28643](https://github.com/ggml-org/llama.cpp/pull/28643), [#28632](https://github.com/ggml-org/llama.cpp/pull/28632)).\r\n- Fixed MIMO2 SWA pattern loading and incorrect `get_key_or_arr` uses ([#28865](https://github.com/ggml-org/llama.cpp/pull/28865), [#28868](https://github.com/ggml-org/llama.cpp/pull/28868)).\r\n- Added `--fuse-qkv` conversion flag to fuse Q/K/V tensors during HF-to-GGUF conversion ([#22780](https://github.com/ggml-org/llama.cpp/pull/22780)).\r\n- Wrote explicit `recurrent_layers` metadata for Qwen3-Next/Qwen3.5 conversion ([#28208](https://github.com/ggml-org/llama.cpp/pull/28208)).\r\n- Refactored JSON schema handling with a `common_schema` internal representation ([#28736](https://github.com/ggml-org/llama.cpp/pull/28736)).\r\n- Split specialized chat parsers into `common/parsers` ([#27764](https://github.com/ggml-org/llama.cpp/pull/27764)).\r\n- Added structured JSONL logging via `--log-jsonl` and `LOG_JSON` ([#28437](https://github.com/ggml-org/llama.cpp/pull/28437), [#28586](https://github.com/ggml-org/llama.cpp/pull/28586)).\r\n- Removed deprecated `--mmap`/`--mlock`/`--direct-io` args in favor of `--load-mode` ([#28334](https://github.com/ggml-org/llama.cpp/pull/28334)).\r\n- Made mmproj and draft devices default to the global `--device` selection ([#28390](https://github.com/ggml-org/llama.cpp/pull/28390)).\r\n- Disabled lazy tensor loading by default on iGPUs ([#28326](https://github.com/ggml-org/llama.cpp/pull/28326)).\r\n- Fixed speculative decoding after multimodal input and DFlash mtmd chunk decode ([#28715](https://github.com/ggml-org/llama.cpp/pull/28715), [#28587](https://github.com/ggml-org/llama.cpp/pull/28587)).\r\n- Fixed Jinja dot-property integer literals and null `in` membership ([#28817](https://github.com/ggml-org/llama.cpp/pull/28817), [#28620](https://github.com/ggml-org/llama.cpp/pull/28620)).\r\n- Improved grammar insert/move handling and max repetition threshold ([#26885](https://github.com/ggml-org/llama.cpp/pull/26885), [#28469](https://github.com/ggml-org/llama.cpp/pull/28469)).\r\n- Improved qwen3-coder complex-type parsing and typed content detection ([#28742](https://github.com/ggml-org/llama.cpp/pull/28742), [#28511](https://github.com/ggml-org/llama.cpp/pull/28511)).\r\n- Added PCH and unity build support to improve build times ([#28091](https://github.com/ggml-org/llama.cpp/pull/28091)).\r\n- Updated cpp-httplib to 0.56.0 ([#28787](https://github.com/ggml-org/llama.cpp/pull/28787)).\r\n\r\n### Multi-modality changes\r\n\r\n- Propagated video hash to bitmap IDs (`{hash}+{frame}`) for cache identification ([#28601](https://github.com/ggml-org/llama.cpp/pull/28601)).\r\n- Fixed DFlash speculative decoding failures on mtmd chunks ([#28587](https://github.com/ggml-org/llama.cpp/pull/28587)).\r\n\r\n### Server changes\r\n\r\n- Refactored server child-process handling into a single monitor thread ([#28555](https://github.com/ggml-org/llama.cpp/pull/28555)).\r\n- Fixed LRU hang on multiple requests for the same model ([#28539](https://github.com/ggml-org/llama.cpp/pull/28539)).\r\n- Allow model downloads when `--models-max` is already reached ([#28530](https://github.com/ggml-org/llama.cpp/pull/28530)).\r\n- Fixed context checkpoint eviction on prompts shorter than `checkpoint_min_step` ([#28302](https://github.com/ggml-org/llama.cpp/pull/28302)).\r\n- Fixed speculative decoding after image input ([#28715](https://github.com/ggml-org/llama.cpp/pull/28715)).\r\n- Frame router child state commands as whole lines ([#28747](https://github.com/ggml-org/llama.cpp/pull/28747)).\r\n- Enabled `--reasoning-preserve` by default ([#28437](https://github.com/ggml-org/llama.cpp/pull/28437)).\r\n\r\n### UI changes\r\n\r\n- Improved chat message rendering performance with lazy mounting and compositor-friendly animations ([#28460](https://github.com/ggml-org/llama.cpp/pull/28460)).\r\n- Added UI asset cache for faster builds ([#28802](https://github.com/ggml-org/llama.cpp/pull/28802)).\r\n- Embedded UI assets directly with CMake, removing the build-time helper ([#28445](https://github.com/ggml-org/llama.cpp/pull/28445)).\r\n- Fixed MCP image attachments not displayed in tool blocks ([#28089](https://github.com/ggml-org/llama.cpp/pull/28089)).\r\n- Export conversations from the database instead of the cached store ([#27432](https://github.com/ggml-org/llama.cpp/pull/27432)).\r\n\r\n### ggml changes\r\n\r\n- Bumped ggml from v0.23.0 to v0.24.0 ([release](https://github.com/ggml-org/ggml/releases/tag/v0.24.0)).\r\n\r\nVersion 0.24.0 focuses on expanding backend coverage and robustness, with a new precision-control API, major Vulkan/SYCL/Hexagon/OpenCL work, and numerous correctness and performance fixes across CPU, CUDA, Metal, and other backends.\r\n\r\n## Assets\r\n\r\n**Nightly build:** [b10964](https://github.com/ggml-org/llama.cpp/releases/tag/b10964)\r\n\r\n## More info\r\n\r\n- [Releases and versioning of `ggml-org` projects](https://github.com/ggml-org/ggml/discussions/1579)\r\n\r\n## Changelog since v0.4.0\r\n\r\nb29c606e2 llama.cpp : bump version to 0.4.1 (#28900)\r\nd9e03f107 sync : ggml\r\neeea73161 ggml : bump version to 0.24.0 (ggml/1627)\r\nbbdd9f246 tests : add fusion baseline README and broaden fusion CI triggers (#28893)\r\n97e4ca735 models : fix incorrect uses of get_key_or_arr (#28868)\r\n1aca1f9fc models : fix mimo2 swa pattern load (#28865)\r\nbe2c6d7d1 tests(s390x): add non-vxe build to tests (#28776)\r\n3d10bcd19 llama: add Maple 20B-A1B ternary MoE architecture (CPU) (#27000)\r\n21f6b0d22 sycl: rfc: Use radix select for top_k (#28670)\r\n2f539596c ggml-cpu : disable PCH and fix CACHE_LINE_SIZE ambiguity to fix heap corruption (#28882)\r\n89fe24240 ci : trigger self-hosted CI on changes to ci/run.sh (#28859)\r\n15d8f2d59 ci : remove gg_sum summary logic (#28857)\r\n661643e43 sycl : fix oneDNN scratchpad breaking the pool free order (#28704)\r\n093a2f86c common : move llama_n_rs_seq to before llama_decode (#28749)\r\nad6c66839 ggml-cuda: fallback to F32 on device without BF16 hardware acceleration (#28846)\r\n7a16a6ce3 grammar : coalesce find + insert into a single insert and adjust move/copy mechanics (#26885)\r\n5f436dddb tests : exclude HY_V4 from WebGPU test-llama-archs tests (#28855)\r\ne49d2c276 models : guard the expert FFN size fallback in nemotron-h against a zero divisor (#28779)\r\n697805298 ggml-cpu(s390x): guard VXE-only repack helpers (#28775)\r\n243a3082d tests : fix typo in test-quant-type-selection for nemotron 3 nano (#28835)\r\nb6b003d2c sycl : Fix get mem error (#28227)\r\nc95f8e47b ci : run editorconfig and code-style checks on ubuntu-slim (#28854)\r\nbc52a12b3 pi : prefer PI_MODEL_NAME env var for model disclosure (#28853)\r\n4a8993735 tests : reduce FA test sizes (#28842)\r\n37b3a9e0c ci : remove leftover command (#28839)\r\n002a12ad2 ci : cap test-backend-ops parallel jobs at 2 and add a 3600s timeout (#28833)\r\nf1e44dcc1 vulkan: workaround NV queuesubmit driver bug (#28830)\r\n56b9eb280 opencl: apply the noshuffle row-alignment rule to q4_K, q5_K and q8_0, not just q6_K (#28575)\r\n790cf51aa chat : improve parsing of complex types in qwen3-coder (#28742)\r\n8e330954a common: add LOG_JSON macro to log structured data (#28586)\r\nacecd5603 common : implement common_schema internal representation for JSON schemas (#28736)\r\nae9afff8d jinja : support dot property integer literals (#28817)\r\n737e0980f cmake: leave the timestamp out of precompiled headers on clang (#28816)\r\n3057bb66c ui : add cache (#28802)\r\n56381e407 server : allow model downloads at model limit fix issue #26809 (#28530)\r\nc8edceb06 ggml-cuda: hip add specific config table for AMD GCN (#27841)\r\ne192abb40 server : add missing headers (#28795)\r\n718f7b417 vendor : update cpp-httplib to 0.56.0 (#28787)\r\n2a3005c23 syscl : Handle (fail gracefully) unsupported tq1_0 quants (#28681)\r\nf3a33dff2 rpc : fix linking when compiling with BUILD_SHARED_LIBS=OFF (#28492)\r\nc069aa7f5 server: frame the router child state command as a whole line (#28747)\r\n8a56aedd6 opencl: fix several bugs where the backend aborts (#27630)\r\n07fc97716 opencl: add bin kernel `kernel_gemm_noshuffle_q4_k_f32_32b_trans_ila_a8_bin` (#28677)\r\n3f5e94d7c webgpu: align tensor bindings to the type block size (#28382)\r\neafe15a5e hexagon: support for multi-device model split (aka row-split) (#28589)\r\nd3146f2b5 ggml-webgpu: Update to a recent version of Dawn (#28683)\r\n82d6bb284 server: refactor subproc handling (#28555)\r\n8ea290247 cmake : skip PCH for llama-server PCH when using MSVC (#28763)\r\nb78a39a2f ci : run test-backend-ops as a dedicated ci/run.sh test (#28740)\r\n982937a33 tests: extend test-quantize-fns to test nrc=2 (i8mm) kernels (#16234)\r\n8172e6577 tests: tolerate a shared pool abort in test_completion_unified (#28759)\r\n43f3dda62 ggml: skip 0-sized ids tensor when offloading selected experts (#28739)\r\n5bda51bfb metal : skip the empty half of the mul_mm_id token tile (#28301)\r\n3bcfeb700 cmake : add PCH and unity build to improve build times (#28091)\r\n1dfe94e04 common : fix typo in speculative.cpp comment [no ci] (#28750)\r\na2878d30d metal : single-source fusion table + fusion debug rework (#28164)\r\naac810230 metal : fix idle threads in the remaining iq mul_mv kernels for ne00 < 1024 (#28692)\r\n5cdd3d1da model : fix MTP context kv cache allocation for deepseek2, glm4moe, \u2026 (#28630)\r\nb0dcb8192 server: fix speculation after an image (#28715)\r\n16378d93f CUDA/HIP: Flash Attention tuning (gfx1201) (#28102)\r\n451b89bae ci : key cache to sanitizer matrix (#28708)\r\n481c65f09 vulkan: fix data race and OOB access in argsort(large) (#28705)\r\ndf03399b8 opencl: add A8 Q4_0 mm binary kernel support (#28268)\r\n28ff09582 vulkan: use CPU writes in ggml_backend_vk_cpy_tensor_async if the context is idle (#28618)\r\n50182a53f vulkan: use add_alloc_dep to enable topk_moe fusion for prefill (#28422)\r\n6788edb4f vulkan: small M matrix optimizations for qwen (#28457)\r\n52d426865 ci : add self-hosted-gpu-cuda and server-sanitize to hf-jobs (#28693)\r\n18c17b4d6 ci : Update WoA CUDA 13.4 release to use 13.4.1 GA redistributables (#28687)\r\nfa6769818 spec: fix failed to decode mtmd chunk with DFlash (#28587)\r\n41fc7584f scripts : use sed instead of grep for version parsing [no ci] (#28700)\r\nd344123fe models: clean up some dead switch branches in old models (#28669)\r\nc32d1dabe tests : increase tolerance for Add fusion tests (#28691)\r\ne5a8d439c tests : drop SYCL special-casing in test-backend-ops.cpp (#28688)\r\n3ff67eb43 vulkan: fall back to shared-memory reduction for dmmv on PowerVR (#28341)\r\n8c322d5bc convert : expand Nemotron H conversion fix (#28689)\r\n311d4211b memory : avoid allocating V cache for indexer (it's not used) (#28330)\r\n72797e891 vulkan : add command-buffer debug labels for GPU profilers (#28101)\r\n4ea6d1bb6 ggml-cpu(s390x): add repack support for q4_0 (#28667)\r\nf1b6fbf35 ggml-cpu(s390x): add Q1_0 vector intrinsic support (#28606)\r\nd7e86430a model: fix all granite family parameter counts (#28643)\r\n434ddbbc0 ci: fix sanitizer tests (#28583)\r\n91f6a6cf3 vulkan: use spec constant for matrix matrix multiplication A-type (#25773)\r\n6d9c82ea2 hexagon: rope updates (#28628)\r\n22397c31a vulkan: Convert FILL to distribute workgroups in 2D to avoid exceeding maxComputeWorkGroupCount (#28592)\r\n4b98ab805 py : lower numpy to 2.2.6 (#28654)\r\n9cf3bf256 py : bump numpy to 2.4.6 (#28649)\r\n4850c7727 llama : use int32_t for llama_sampler_chain_n return type (#28631)\r\nd4abd573f CUDA: size routed MoE MMQ N-tiles from typical expert width on RDNA3 (#28552)\r\n5a4d0feca CUDA: replace GGML_FA_ALL_QUANTS with GGML_FA_QUANTS, more control over what is compiled (#28079)\r\n14a9d09f7 args: officially deprecate --mmap|mlock|dio (#28334)\r\ne2d2c0d6a model: fix granite3 moe unknown parameter count (#28632)\r\n6de9cdb26 mtmd: propagate video ID to bitmap (#28601)\r\nb31b71f3a jinja: treat a null left operand of in as a plain lookup (#28620)\r\ndf750f76b vulkan: add dedicated iq4_xs mat-vec shader (#28426)\r\n1945e0920 vulkan: add f16 B-type matmul pipelines and warp tile size tuning for Intel coopmat1 (#27471)\r\n30b6a755e tests : use less threads for data initialization (#28325)\r\n304665fe7 Add IQ type handling for MoE (#28476)\r\nf3f1a8f27 llama: disable lazy tensor loading by default on iGPUs (#28326)\r\n9113cc188 ggml : fix msvc+clang ggml_vld1q_u32 (#28284)\r\nd4389a4dd Revert \"ggml-cuda : restore prop.integrated on HIP builds (#24233)\" (#28604)\r\n5d806aa25 server : apply checkpoint min-step eviction only when the checkpoint list is full (#28302)\r\n88ada91c1 metal : fix idle threads in mul_mv_iq3_xxs for ne00 < 1024 (#28086)\r\n415e909d8 spec: single device drafter should create meta backend wrapper (#28390)\r\n03fa73cb2 ci : disable npm gha cache (#28600)\r\n1744c6bde ci : add PYTEST_WORKERS=1 to fix server-self-hosted job (#28603)\r\nca86fb222 llama : add missing headers (#28566)\r\n64e9bceb2 vulkan : fuse UNARY(GELU|SIGMOID|SILU|SOFTPLUS) + MUL (#27220)\r\nf014bfef8 Fix Vulkan-Hpp handle usage on 32-bit targets. (#22892)\r\n895c045fd chat : split specialized parsers into common/parsers (#27764)\r\n7d701b592 opencl: properly handle non-contiguous inputs to conv2d (#28503)\r\n5a6caa05f ggml : update ggml_prec specification (#26675)\r\n9dcf84e5a model : support Kimi-K3 recurrent-state rollback (#28466)\r\n050dde50c hexagon: add RELU and LEAKY_RELU ops (#28585)\r\n67672dc5b ci : bump ty to 0.0.78 (#28548)\r\nf114f91f9 tests : initialize the L2_NORM batch array (#28553)\r\ne71b80510 Revert \"CUDA: size routed MoE MMQ N-tiles from typical expert width on RDNA3 (#24546)\" (#28551)\r\nccc3646c6 nix : update deprecated expressions (#28145)\r\nc0b1871bc webgpu: format the GET_ROWS case block (#28542)\r\n160bd031b server: fix LRU hang on multiple requests same model (#28539)\r\ndbeb37548 sycl: add a batched L2_NORM kernel (#28222)\r\n7a333e724 vulkan: add DeepSeek-V4 hyper-connection fused ops (DSV4_HC_COMB/PRE/POST) (#26578)\r\n0c963452e CUDA: size routed MoE MMQ N-tiles from typical expert width on RDNA3 (#24546)\r\n473599738 ggml: add gfx90c HIP support (#26454)\r\nd23c47f2a convert : refactor Hy4-preview conversion - move HC tensor mapping to the global map (#28451)\r\n73ab7599b CUDA: branchless Q4_K/Q5_K unpack to speed up mmvq, L2 prefetch on DGX Spark (#26705)\r\n0cae43063 vulkan: support type-aligned GET_ROWS (#28253)\r\n1173700b9 examples : print ggml_version and ggml_commit in test-cmake [no ci] (#28538)\r\n5202104b5 caps : recheck typed content if template checks for string (#28511)\r\n9a7570587 convert : write explicit recurrent_layers for Qwen3-Next / Qwen3.5 (#28208)\r\nb74f590ea ggml-cuda: fix divergent barrier in f16 flash attention (#27870)\r\n992cb503c ggml: allow backend inputs to not create another split (#28387)\r\n9ac8c408a vulkan: rms_norm fusion opportunities (#28024)\r\n2092353c8 ci : add container image checking and tagging (wip) (#28394)\r\n8fe90e1fb vulkan: add TQ1_0 support (mm, mat-vec, mat-vec-id, dequant, get_rows) (#27765)\r\n465e49b9c convert : add `--fuse-qkv` flag to fuse Q/K/V into QKV during HF-to-GGUF conversion (#22780)\r\n5fdfa6282 models : fix GDN normalization from `max` to `rsqrt` (#28068)\r\n3ad1ba733 [Model] Support for Spark2_5ForCausalLM implementation (#27868)\r\nd03efa5d5 opencl: properly choose weights pack for q4_K, q5_K mul_mat (#28402)\r\n73a43d1f6 cuda: fixes races in mmid and mmf (#28475)\r\n9e0e22059 grammar : fix max repetition threshold (#28469)\r\n0afb805b1 ui: Improve Chat Messages rendering performance (#28460)\r\n7620399f5 common: add --log-jsonl (#28437)\r\nc457e3bf7 ui : embed assets directly with CMake (#28445)\r\n971595d66 metal : add remaining fa-vec tunings for M2 Max (#28458)\r\n74a7c897f Github: limit blank issues to maintainers (#28435)\r\n6a1a922d2 metal : fix memory leak in early return (#28399)\r\n4d9176092 sycl : fix test-backend-ops CI break && restore Kronecker product FWHT support (#28016) (#28254)\r\ncd8cdf397 sycl: attribute device allocations by site (GGML_SYCL_MEMTRACE) (#27631)\r\n427291b5b metal : add remaining fa-vec tunings for M3 (#28396)\r\n85d5703a3 ui : fix MCP image attachments not displayed in tool block (#25789) (#28089)\r\n1548a240e opencl: extend the elementwise and data\u2010movement op coverage (#27633)\r\n4acf4a4cb opencl: add Adreno xmem SDPA path (#26331)\r\n8b4b3558f ci : move more jobs to ccache-buckets (#28375)\r\n1863ac033 ui: export conversations from database instead of cached store (#27432)\r\n49c0dc82b model : add Tencent Hy 4 (hy_v4) preview architecture support (#28127)"
+ },
+ {
+ "repo": "vllm-project/vllm",
+ "tag_name": "v0.29.0",
+ "name": "v0.29.0",
+ "published_at": "2026-09-09T08:54:49Z",
+ "html_url": "https://github.com/vllm-project/vllm/releases/tag/v0.29.0",
+ "prerelease": false,
+ "body": "# v0.29.0\r\n\r\n## Highlights\r\n\r\nThis release features 594 commits from 277 contributors (91 new)!\r\n\r\n* **Model Runner V2 is now the default for all models** (#53183), completing the rollout that began with pooling models (#48290). MRV2 also gained CUDA graph memory profiling for KV cache auto-sizing (#53306), batch-sharded sampling that cuts per-step logits memory by 1/TP (#50465), prompt embeds (#42963), `extract_hidden_states` speculation (#49811), padded FULL cudagraph dispatch for uniform decode under spec decode (#53407), and DP-sync skipping before EAGLE/MTP draft prefill (#53694). MRV1 remains in use for a few ROCm models and features MRV2 does not yet support.\r\n* **New models**: Hy4-preview, Tencent's 770B/49B-active MoE with Gated DeepSeek Sparse Attention and native MTP (#54160); Qwen3.8-Flash-Next with BF16/FP8/NVFP4 and MTP (#53896); GraniteSWA and GraniteMoeSWA (#52706); NemotronH_Omni_Reasoning_V3 with MTP (#52929, #53121); Kimi K3 NVFP4 checkpoints (#53132).\r\n* **Kimi-K3 and DeepSeek V4 performance**: fused MXFP4 top-k finalization in the K3 latent tail (about 5% E2E latency, #53152), K3 Mamba metadata preparation in one Triton launch (6.6-7.6x kernel speedup, #52388), tuned Hopper low-latency GEMM (#54088) now also dispatched on SM100 (#53534) and used for `eh_proj` (12.9-25.2% kernel speedup, #53942), GEMM-RS extended to GEMM-AR (#53053), MLA gate merged into the QKV-A projection (#54015), K3 DCP with DSpark (#52188) and DCP partial prefix cache hits (#50493); DeepSeek V4 shared experts fused into MegaMoE (#53040), adaptive top-k width re-landed (#52823), a native SwiGLU clamp kernel for Humming MoE (#53685), and an opt-in FlashInfer `moe_ep` expert backend (#49636).\r\n* **Speculative decoding**: per-request acceptance stats in OpenAI API responses via `--per-request-spec-decode-metrics` (#48915), adaptive verification extended to logprobs (#52242), SM100 sparse MLA for GLM-5.2 (#52783) and DeepSeek V4 on SM90 (#52795), Qwen3-Omni DSpark drafts (#52560), PLaMo3 EAGLE-3/DFlash (#54239), and DFlash2 loading from speculators format (#53797).\r\n* **RL weight sync**: a new `sharded_rdt` P2P backend where each worker pulls only its TP/EP slice over NIXL or Ray Direct Transport (#43375), rank-local IPC weight updates (#52497), sparse checkpoint-coordinate updates through native weight loaders (#50723, #53751), and routed expert loading for gpt-oss (#52209).\r\n* **Mamba prefix caching**: internal prefill checkpoints deliver a 9%-25% TTFT improvement (#52789); `prefix_cache_retention_interval` is now a CLI argument defaulting to 0 (#52216), with dense retention automatically restored for hybrid models using EAGLE/MTP (#55760, #55861).\r\n* **New defaults**: FlashInfer all-reduce enabled by default for TP CUDA groups, opt out with `VLLM_ALLREDUCE_USE_FLASHINFER=0` (#52998); prefix-cache `NONE_HASH` is deterministic by default so distributed KV cache users no longer need to pin `PYTHONHASHSEED` (#51875); new `--max-num-queued-reqs` / `--max-num-queued-tokens` admission-control flags (#49445).\r\n* **Breaking changes**: ten deprecated model architectures removed (#53608); FlexOlmo, Olmo3 and Hunyuan V1/VL migrated to the Transformers modeling backend (#53615); PyAV video decoder backend removed (#54231); `python -m vllm.entrypoints.openai.api_server` deprecated in favor of `vllm serve` (#52131); `VLLM_TEST_FORCE_FP8_MARLIN` (#52182) and `VLLM_ROCM_USE_AITER_FP4_ASM_GEMM` (#53141) removed.\r\n\r\n## :warning: Model Runner V1 Deprecation\r\n\r\nNow that Model Runner V2 is used by default, we are considering Model Runner V1 deprecated and are targeting v0.32 for its removal. We do not intend to accept any more MRV1-specific improvements or optimizations.\r\n\r\nSome [features](https://github.com/vllm-project/vllm/issues/47172) are not yet supported in MRV2 but we are planning for these gaps to be closed within the next 2-3 weeks. These include sequence parallelism, dual-batch overlap, elastic expert parallellism, custom logits processors and certain speculative decoding methods. For now, vLLM will still fall back to use MRV1 if any of these features are configured.\r\n\r\n## Release Artifacts\r\n\r\n### Python Wheels\r\n\r\n| Platform | Install |\r\n|---|---|\r\n| PyPI (CUDA 13.0) | `pip install vllm` |\r\n| PyPI (CUDA 13.0, uv) | `uv pip install vllm --torch-backend=auto` |\r\n| ROCm | `pip install vllm --extra-index-url https://wheels.vllm.ai/rocm/0.29.0/rocm723` |\r\n| XPU | `uv pip install vllm --extra-index-url https://wheels.vllm.ai/0.29.0/xpu --extra-index-url https://download.pytorch.org/whl/xpu --index-strategy unsafe-best-match` |\r\n\r\n### Docker Images\r\n\r\n| Platform | Docker Image |\r\n|---|---|\r\n| CUDA 13.0 (Default) | `docker pull vllm/vllm-openai:v0.29.0` (`v0.29.0-cu130` also works) |\r\n| CUDA 12.9 | `docker pull vllm/vllm-openai:v0.29.0-cu129` |\r\n| CUDA 13.0 + Ubuntu 24.04 | `docker pull vllm/vllm-openai:v0.29.0-ubuntu2404` |\r\n| CUDA 12.9 + Ubuntu 24.04 | `docker pull vllm/vllm-openai:v0.29.0-cu129-ubuntu2404` |\r\n| ROCm | `docker pull vllm/vllm-openai-rocm:v0.29.0` |\r\n| CPU | `docker pull vllm/vllm-openai-cpu:v0.29.0` |\r\n| XPU | `docker pull vllm/vllm-openai-xpu:v0.29.0` |\r\n\r\n### Other Artifacts\r\n\r\nPre-built release artifacts are available in the **Assets** section at the bottom of this page, including:\r\n- Source distribution tarball\r\n- CUDA 12.9 Python wheels for x86_64 and arm64\r\n- CUDA 13.0 Python wheels for x86_64 and arm64\r\n- CPU Python wheels for x86_64, arm64, and macOS\r\n- XPU Python wheel for x86_64\r\n\r\n## Model Support\r\n* **New models**: Hy4-preview (#54160), Qwen3.8-Flash-Next (#53896), GraniteSWA and GraniteMoeSWA via the existing Granite implementation (#52706), NemotronH_Omni_Reasoning_V3 (#52929) with MTP for Nemotron VL models (#53121), bidirectional attention for DeepSeek-backbone embedding models (#52948), FP8 ModernBERT (#53101), and Kimi K3 NVFP4 checkpoints (#53132).\r\n* **Transformers modeling backend**: FlexOlmo, Olmo3 and Hunyuan V1/VL migrated off native implementations (#53615), multimodal path hardened (#51827), and `RMSNormFuser.fuse` performance fixed (#52766).\r\n* **Weight loading**: weight tying now inspects the checkpoint so a real `lm_head` is loaded even when the config claims tied embeddings (#51665, #53170).\r\n* **LoRA**: DeepSeek V4 (#53361), Qwen3-Omni multimodal LoRA (#52786, #53557), tower/connector LoRA for LLaVA-NeXT (#49788) and LFM2-VL (#51498), plus fixes for Muse-Glimmer (#53513), Qwen3.5 embedding modules (#48850), partial LoRA on Qwen3.5/3.6 GatedDeltaNet (#47640), int32 overflow in punica kernels at long context (#53034), false target matches on unsupported module types (#52313), and base-layer/routed-expert prefix ordering (#52552).\r\n* **Multimodal performance**: ViT full CUDA graph for Idefics3 and SmolVLM (#47625), packed encoder attention for Pixtral (#52185), fused Kimi vision Q/K RoPE kernel (#50400), Dots3 NOTE runtime (#53517) and Omni encoder optimizations (#53460), MM tensors no longer broadcast to workers for prefix-cache-covered items (#52041), redundant placeholder scans skipped (#52925), common token sequences cached (#53560), async media resolved concurrently across modalities (#54537), and H2D copies pinned to avoid stream stalls (#53412, #54292, #54299).\r\n* **Multimodal inputs**: video embeds accepted by the Python frontend (#54242), modality-scoped `mm_processor_kwargs` honored (#53808), Qwen3-VL profiling honors `cap_pixels_per_frame` (#54380), video frame sampling respects MP4 edit-list trims (#48608), oversized items skip the MM processor cache instead of crashing (#53016), encoder cache entries survive until their last use (#54284), and repeated multimodal requests with the SHM cache no longer terminate the engine (#54994).\r\n* **Correctness**: PaliGemma stale image scaling removed (#52692), Mistral3 placeholder grid with processor size overrides (#52874), MiniCPM-o on Transformers v5 (#54501), JinaVL cache order (#53553), mixed CLIP/SigLIP pooling batches (#53165), XD-RoPE models on prefix-cache hits (#53456), Qwen3-Omni audio encoder with non-divisible TP (#50858), GLM-5.2 no longer uses dense MHA (#52512), Moondream3 MoE all-reduce (#54152), deepseek-vl2 config defaults (#51302), HY-V3 compressed-tensors ignore matching (#48682), MiniMax-M3 FP8 query allocation under CUDA graph replay (#51203), and GraniteMoeHybrid quantized expert loading (#54052).\r\n\r\n## Engine Core\r\n* **Model Runner V2**: default for all models (#53183) and pooling models (#48290), CUDA graph memory reservation (#53306), batch-sharded sampling (#50465), prompt embeds (#42963), `extract_hidden_states` (#49811), padded FULL cudagraph dispatch (#53407), DP-sync skipping for drafts (#53694), decoupled draft/target gumbel noise streams (#54282), encoder-only path split out (#53176), and memory released correctly on shutdown and sleep (#53508, #54246, #54162, #53955, #53682).\r\n* **Speculative decoding**: per-request acceptance stats (#48915), adaptive verification with logprobs (#52242), on SM100 sparse MLA (#52783) and DSv4 + SM90 (#52795), varlen trtllm-gen decode (#52157), FlashInfer MLA for DSpark drafting under DCP (#54277), fused GDN MTP for all Qwen head ratios (#52539), widest uniform decode batch captured by default (#50488) with memory-safe graph sizes (#54418), and fixes for short_conv/LFM2 targets (#50272), speculators-format config overrides (#42376), DFlash draft RoPE layout (#54373), draft models with a different GQA ratio (#53002), draft models with a larger hidden size under TP (#52193), DSpark backend inheritance scoped to DeepSeek V4 (#52809), generic `DSparkDraftModel` configs for Qwen3 (#52197), and Gemma4 MTP under CUDA graphs (#53884).\r\n* **Prefix caching & scheduling**: Mamba internal prefill checkpoints (#52789), `prefix_cache_retention_interval` argument (#52216, #55760, #55861), deterministic `NONE_HASH` (#51875), queue admission control (#49445), KV null block reserved when validating `max_model_len` (#47272), spec decode no longer padded up to `max_model_len` (#53962), and negative external block allocation prevented (#52707).\r\n* **RL workflows**: `sharded_rdt` P2P weight sync (#43375), rank-local IPC updates (#52497), sparse checkpoint updates (#50723, #53751), gpt-oss routed expert loading (#52209), stable DeepSeek V4 mHC broadcast buffers across weight sync (#52626), and packed weight transfer stream reuse capping reserved-memory waste (#52951).\r\n* **Determinism**: `trace_decode_token_ids` for deterministic decode replay (#46701), per-arch tuned batch-invariant matmul configs (about 3x decode kernels on RTX 4090D/H20, #53247), Blackwell autotuning with 33.6% E2E latency reduction (#53649), deterministic MoE combine under DP+EP (#45683), and `fuse_allreduce_rms` disabled under batch invariance (#51292).\r\n* **Kernels**: fused embedding kernel (#53677), FA4 re-enabled for head_size=256 on Blackwell (#52980), vectorized sparse MLA mask loads (#52217), masked MHA prefill for GLM-5 head dimensions (#53785), GLM-5.2 sparse MLA Q concatenation fused with head padding (#53878), fused QK-norm + partial MRoPE + gate for Qwen3.6 (#52676), FlashInfer CuTeDSL BF16 low-latency GEMM as opt-in `--linear-backend flashinfer_cutedsl` (#50572), replicated embedding and norm fusion for DSV3 flat models (#48484), standardized fused shared-expert selection (#51695), GPT-OSS MoE topk metadata reuse (#45457), tuned cooperative topk (#53382), and tuned FP8 fused_moe for Qwen3.5 on L40S (+7%, #53819).\r\n* **Robustness**: KV cache layout standardized under a `KVCacheLayout` enum (#51718), JIT warmup provider registry (#50174), `--cpu-offload-params` now reaches vision/audio towers (#53120), attention backend probe failures no longer crash init (#51703), FlashInfer XQA falls back on unsupported head_dim (#53111), FlashInfer prefill LSE normalized before merging to fix prefix-cache logits divergence (#52796), seed preserved when a batch mixes seeded and unseeded requests (#51866), startup thread allocation accounts for local DP workers (#52385), int32 overflow fixes in fused SiLU block quant (#53409) and LoRA kernels (#53034), a shared-memory race in fused groupwise RMSNorm quantization (#54111), BLHNC addressing for FlashInfer sparse MLA (#54465), Mamba state copy race (#50729), and a `start_profile` no-op after auto-stop (#51839).\r\n\r\n## Hardware & Performance\r\n* **NVIDIA**: DeepSeek V3.2 / GLM-5.2 DSA routed to the optimized CUDA path on all GPUs (#52861), PCP for DSv3.2 sparse MLA (#52046), cuBLAS out_dtype router GEMM on all archs including GB10 (#54048), FlashInfer all-reduce tuning on SM103 (#53318, #53606), FA4 hdim256 on SM100 (#52980), SM120 sparse MLA fixes (#51395, #53574), DeepSeek V3.2 fused kernel grids hardened for 65k+ token launches (#52381), FlashMLA sparse decode workspace fix (#53755), MNNVL Lamport corruption fix (#53000), and opt-in Rubin Docker builds for CUDA 13.4/13.5 (#53443).\r\n* **AMD ROCm**: dual-stream decode with hipgraphs (#52033), W4A4 preshuffled asm GEMM by default (+15% throughput on Llama-3.3-70B MXFP4, #53141), ROCr/CLR update fixing graph replay segfaults with up to about 20% TPOT improvement (#53712), fused KDA decode on MI325X (#52293), FULL cudagraphs for AITER MLA spec decode (#51171), FP8 asm MLA prefill for non-divisor head counts (#51040), DCP causal multi-token verification (#51705) and prefix cache hits for Kimi-K3 (#53598), AITER PA gluon decode for MiniMax-M3 (#52849), DeepSeek-V4 fusions for mHC/RMSNorm (#52737), C4A top-k (#52882), C4 compressor GEMMs (#53838) and SWA q/kv norm + FP8 quant (#53540), fused shared experts for block-FP8 (#53097), CPU offload on ROCm 7.13+ (#43018), TheRock 7.14 preview docker (#49925), int4/int8 quantization fixes (#52112, #48998, #51632, #53110), CUDA graphs captured on the current stream (#53818), AITER metadata preserved across graph replay (#53821), and improved ROCm detection under WSL (#38434).\r\n* **Intel XPU**: INC int4 W4A8 linear backend (#50501), AutoRound MXFP8 MoE (#51248), EC connector KV offloading (#49532), and fixes for HunyuanOCR XD-RoPE (#52174), sparse-MLA metadata sync (#52066), Mamba state pointer overflow (#48109), oneCCL warm-up at world size 1 (#52389), and MRoPE (#53201).\r\n* **CPU**: AMX high-performance MLA backend for DeepSeek V2/V3/R1 (#52616) with MLA now running end-to-end (#51471), FP16/BF16 persisted GDN state on AMX (#52191), Int8 MoE through zentorch on AMD Zen (#44834), Voxtral support (#53921), C++ causal_conv1d GDN on non-AMX AVX-512BF16 (#49688), and assorted fixes (#54042).\r\n\r\n## Large Scale Serving\r\n* **Context parallelism**: Kimi-K3 DCP with DSpark (#52188) and partial prefix cache hits (#50493), FlashInfer native CP for MLA decode (#54012), FlashMLA sparse DCP on Hopper with MTP (#46514), NIXL P/D DCP for MLA models (#50611), PCP for DSv3.2 (#52046) and NIXL PCP producers (#52779), `--dcp-q-replicate` with query replication default-on for GLM sparse attention (#50382), DCP fused attention fix for DeepSeek-V3.2 / GLM-5.2 (#50005), sparse MLA metadata and kernel block sizes under DCP (#52377, #51031), PCP PIECEWISE cudagraph fixes (#53869, #53515), and PCP compatibility checks delegated to the PCP manager so plugins can enable GQA+PCP (#53853).\r\n* **Elastic EP**: reduced eager-mode reconfiguration downtime (#51885), AOT cache reuse preserved during scaling (#53378), and scale-below-minimum rejected (#52702).\r\n* **MoE communication**: DeepEP v2 receiver CPU overhead (#51114) and MXFP8 activation scale dispatch (#51398), FlashInfer one-sided All2All refinements (#51924), DeepEP v2 fixes for `--enforce-eager` startup (#51824) and the decode/cudagraph path (#52632), NCCL>=2.31 heap overflow fix (#53008), cross-node MNNVL all-reduce gated by capability (#53253), and MNNVL all-reduce buffers sized for DSpark (#50932).\r\n* **KV connectors**: Mooncake Store decode KV saving via `save_decode_cache` (#52466) and hybrid DCP prefix caching (#53324), externally transferable KV cache group identification (#53779), async KV loads deferred past forward launch (#53333), `kv_transfer_params` for `/inference/v1/generate` (#42644), MoRIIO shared KV region registration after the layout refactor (#53698), and Mamba fixes for Mooncake (#51362, #51358, #53663) and NIXL (#53523).\r\n* **KV offloading**: EC offloading connector driven by CUDA events (#49994), ownership in KV cache events (#52067, #52068), P2P tier request-level offload (#52912) and abort handling (#52571), `/dev/shm` leak on crash fixed (#52596), CPU->GPU loads ordered against compute stream (#50696), `store_threshold` counting fixed (#52227), in-flight primary keys cascaded (#53329), and padded GPU cache storage handled (#54021).\r\n* **Data/pipeline parallel**: PP silent corruption fix (#54962, #49274), DP coordinator wake handling (#51481), device sync on pause (#52914), TCPStore port fixes for Ray (#53666, #50969), DP supervisor inheriting the uvicorn config (#52473), EPD encoder round-robin fix (#52491), and producer-only EC config normalization (#53656).\r\n\r\n## Quantization\r\n* **New backends**: FlashInfer TRT-LLM MXFP8 linear (#52204), b12x FP4 MoE for SM120/SM121 (#52018), AutoRound block-wise FP8 (#47434), Humming MoE with MXFP4 weights + block-FP8 activations (#51332), and Humming for compressed-tensors WNA16 MoE (#48918).\r\n* **Fixes**: weight-only NVFP4 checkpoints routed through W4A16 (#54427), compressed-tensors block FP8 with Marlin (#52966) and int8 grouped WNA16 MoE (#52002), OCP MX mxfp6 activation emulation (#52704), MXFP8 FlashInfer path guarded on availability (#52648), and Humming activation aliasing (#54056).\r\n\r\n## API & Frontend\r\n* **New endpoints & options**: `/v1/messages/render` for the Anthropic Messages API (#45803), `/cohere/v2/chat/render` (#53219), SSE keep-alive comments for idle streams (#51034), per-request spec decode metrics (#48915), video embeds input (#54242), `--max-num-queued-reqs` / `--max-num-queued-tokens` (#49445), and pooling requests can set padding (#51157).\r\n* **Anthropic & Cohere**: `vllm_xargs` forwarded to sampling params (#53308), `stop_sequence` stop reason reported (#45807), Cohere citation/tool 500s fixed (#52175), and stop string limit applied (#53750).\r\n* **OpenAI compatibility**: `logprobs=-1` in Completions (#46175), streamed logprob offsets with echo (#47815), batched chat echo (#52529), all choices returned from `/inference/v1/generate` with n>1 (#52399), `cache_salt` forwarded for content parts (#54315), `stop_token_ids` validated against vocab (#54196), and 4xx for client errors in `/detokenize` (#52622), malformed namespace tools (#53763), malformed base64 audio (#53744), and unknown chat roles in DeepSeek encoders (#53071); tool-call arguments in replayed history are parsed defensively (#48922), empty `FlatLogprobs` slices are handled (#53704), and empty bad-word tokenizations are rejected (#53433).\r\n* **Structured output & parsers**: reasoning-end detection scoped to the current turn (#54089), terminal grammars stop under `min_tokens` (#54218), unsupported pattern+length schemas rejected (#49996), MistralCommonBackend tokenizers (#52720), XGrammar termination in batches (#52805), spurious FSM errors after speculative reasoning end (#53046), shared parser engine adapters (#52830), Gemma4 parenthesized tool calls (#53657) and `enable_thinking` default (#52430), HY-V3 parallel calls in one delta (#53965), GPT-OSS Harmony strict grammar (#52222), Kimi K3 reserved markers excluded from response text (#52889), and unused request-local reasoners skipped (#52573).\r\n* **Rust frontend**: HY3 unified parser with local XGrammar structural tags (#53054), gRPC audio/video inputs (#53760), gRPC LoRA lifecycle control (#52840, #52031, #53756), `--generation-config vllm` (#53044), `truncate_prompt_tokens` (#48584), OpenAI edge-case alignment (#53218), optimized SSE hot path (#51321), pure-Rust `protox` replacing `protoc` (#52892), RL world-size reporting (#53204) and routed expert prompt offsets (#52703), and fixes for GLM-5.2 template parity (#51426), Qwen parser auto-detection (#51169), Kimi K3 `reasoning_effort=\"none\"` (#53043), `n > 1` rejection on `/inference/v1/generate` (#52844), and the `LogprobsTensors` wire schema (#53939).\r\n* **Pooling**: BGE-M3 throughput (+3.13%, #53464) and task validation (#51823), prompts truncated before padding (#54364), parallel arrays truncated with the prompt (#54407, #54509), and batched input throughput in `vllm bench serve` (#53213).\r\n* **CLI & tooling**: `vllm launch` runs the serve argument checks (#52825), `run_batch.py` moved out of the openai folder (#53500) with its upload retry fixed (#50588), `vllm bench` warns on a warm prefix cache for random runs (#53920) and restores multimodal datasets on the `vllm` throughput backend (#52168), and the vLLM recipes tool gained sweep recommendations and alias parsing (#53325, #53946).\r\n\r\n## Security\r\n* `cache_salt` length bounded to 1024 to prevent scheduler CPU exhaustion (#54353).\r\n* Oversized media rejected before full download (#51896); `VLLM_MAX_AUDIO_CLIP_FILESIZE_MB` enforced on all audio paths (#53561).\r\n* Decoder prompt-length validation enforced for processors that skip the check (#46588); PyNvVideoCodec decoder slot limit bypass fixed (#52126).\r\n* `api_key` and `hf_token` redacted from startup logs, compile cache factors, and the Rust frontend launch log (#52523, #53625, #53738).\r\n\r\n## Dependencies\r\n* FlashInfer 0.6.18 (#54313), huggingface-hub 1.28.0 (#52797), tpu-inference v0.28.0 (#54020), NIXL 1.3.2 (#51777).\r\n* InstantTensor added to CUDA dependencies; the loader is not enabled by default (#52801). CuPy constraint relaxed to exclude only 14.1.0 (#44284).\r\n* ROCm base image: ROCr/CLR update (#53712), rocprofiler-sdk 1.3.2 (#53182), TheRock 7.14 preview (#49925), LMCache connector packages (#51208).\r\n* XPU: vllm_xpu_kernels 0.1.14.1 (#54203), UCX install updated (#53817).\r\n* Build fails closed when the selected precompiled CUDA variant is unavailable (#52545).\r\n\r\n## Breaking Changes & Deprecations\r\n* Ten deprecated architectures removed: Arctic, Chameleon, Cheers, Fairseq2Llama, FireRedLID, GritLM, HCXVision, MPT, the `RWForCausalLM` and `StableLMEpochForCausalLM` aliases, and `PrithviGeoSpatialMAE` (superseded by `Terratorch`) (#53608).\r\n* FlexOlmo, Olmo3, Hunyuan V1 and Hunyuan VL are now served through the Transformers modeling backend (#53615).\r\n* PyAV video decoder backend removed; use OpenCV or Torchcodec (#54231).\r\n* `python -m vllm.entrypoints.openai.api_server` is deprecated; use `vllm serve` (#52131).\r\n* Model Runner V2 is the default runner for all models (#53183).\r\n* `prefix_cache_retention_interval` default changed from dense to 0 for SWA/SSM models; the env var is deprecated in favor of the argument (#52216).\r\n* FlashInfer all-reduce enabled by default (#52998).\r\n* `VLLM_TEST_FORCE_FP8_MARLIN` removed in favor of `--linear-backend` / `--moe-backend` (#52182); `VLLM_ROCM_USE_AITER_FP4_ASM_GEMM` removed (#53141); dead `--attention-config.use_prefill_decode_attention` removed (#52557); other long-deprecated parameters cleaned up (#53559).\r\n\r\n## New Contributors\r\n* @030611 made their first contribution in https://github.com/vllm-project/vllm/pull/51823\r\n* @92hyungjun made their first contribution in https://github.com/vllm-project/vllm/pull/47272\r\n* @ActiveSky made their first contribution in https://github.com/vllm-project/vllm/pull/52692\r\n* @adisivaprasad made their first contribution in https://github.com/vllm-project/vllm/pull/53854\r\n* @Agoni-02 made their first contribution in https://github.com/vllm-project/vllm/pull/48850\r\n* @AmitMY made their first contribution in https://github.com/vllm-project/vllm/pull/48608\r\n* @Andy365-365 made their first contribution in https://github.com/vllm-project/vllm/pull/52523\r\n* @andyluo7 made their first contribution in https://github.com/vllm-project/vllm/pull/53821\r\n* @AnkitNakhawa made their first contribution in https://github.com/vllm-project/vllm/pull/52491\r\n* @anmolgupt made their first contribution in https://github.com/vllm-project/vllm/pull/52874\r\n* @bobboli made their first contribution in https://github.com/vllm-project/vllm/pull/51924\r\n* @brianosaurus made their first contribution in https://github.com/vllm-project/vllm/pull/52557\r\n* @CalvinXKY made their first contribution in https://github.com/vllm-project/vllm/pull/50858\r\n* @canlahlah made their first contribution in https://github.com/vllm-project/vllm/pull/53409\r\n* @CherryLemon made their first contribution in https://github.com/vllm-project/vllm/pull/53877\r\n* @CHIPMUNK-T0T made their first contribution in https://github.com/vllm-project/vllm/pull/47625\r\n* @cogniera made their first contribution in https://github.com/vllm-project/vllm/pull/53839\r\n* @cr-zhao made their first contribution in https://github.com/vllm-project/vllm/pull/52385\r\n* @daviswer made their first contribution in https://github.com/vllm-project/vllm/pull/52706\r\n* @DCoEngine made their first contribution in https://github.com/vllm-project/vllm/pull/48682\r\n* @dineshchitlangia made their first contribution in https://github.com/vllm-project/vllm/pull/49688\r\n* @dkrisman made their first contribution in https://github.com/vllm-project/vllm/pull/54380\r\n* @dmvevents made their first contribution in https://github.com/vllm-project/vllm/pull/52632\r\n* @Edge-Explorer made their first contribution in https://github.com/vllm-project/vllm/pull/53435\r\n* @eilamc14 made their first contribution in https://github.com/vllm-project/vllm/pull/47640\r\n* @eligotts made their first contribution in https://github.com/vllm-project/vllm/pull/54315\r\n* @Eoin-Houstoun made their first contribution in https://github.com/vllm-project/vllm/pull/51703\r\n* @floatlibai made their first contribution in https://github.com/vllm-project/vllm/pull/52144\r\n* @fuzzifikation made their first contribution in https://github.com/vllm-project/vllm/pull/51034\r\n* @hagaikwa-redhat made their first contribution in https://github.com/vllm-project/vllm/pull/53230\r\n* @haoyangqian made their first contribution in https://github.com/vllm-project/vllm/pull/52690\r\n* @Hert4 made their first contribution in https://github.com/vllm-project/vllm/pull/51157\r\n* @ima-helikoptaaa made their first contribution in https://github.com/vllm-project/vllm/pull/54427\r\n* @jbyczkow made their first contribution in https://github.com/vllm-project/vllm/pull/52174\r\n* @JC-ut0 made their first contribution in https://github.com/vllm-project/vllm/pull/53071\r\n* @jiahaoliang made their first contribution in https://github.com/vllm-project/vllm/pull/51262\r\n* @JiataiWang made their first contribution in https://github.com/vllm-project/vllm/pull/53553\r\n* @jl9876 made their first contribution in https://github.com/vllm-project/vllm/pull/51248\r\n* @JulianZJN made their first contribution in https://github.com/vllm-project/vllm/pull/53253\r\n* @jungjiyu made their first contribution in https://github.com/vllm-project/vllm/pull/53939\r\n* @kyleliang-nv made their first contribution in https://github.com/vllm-project/vllm/pull/51203\r\n* @LH-and-FPGA made their first contribution in https://github.com/vllm-project/vllm/pull/52648\r\n* @li-ukumar made their first contribution in https://github.com/vllm-project/vllm/pull/52571\r\n* @LioEinaudi made their first contribution in https://github.com/vllm-project/vllm/pull/53247\r\n* @LironKesem made their first contribution in https://github.com/vllm-project/vllm/pull/53921\r\n* @Lossfull made their first contribution in https://github.com/vllm-project/vllm/pull/52948\r\n* @lxy-alexander made their first contribution in https://github.com/vllm-project/vllm/pull/52430\r\n* @lxyxinyi made their first contribution in https://github.com/vllm-project/vllm/pull/50809\r\n* @machero made their first contribution in https://github.com/vllm-project/vllm/pull/53531\r\n* @matthewkotila made their first contribution in https://github.com/vllm-project/vllm/pull/48915\r\n* @mhoqueanik made their first contribution in https://github.com/vllm-project/vllm/pull/49636\r\n* @mhuzaifa3 made their first contribution in https://github.com/vllm-project/vllm/pull/53625\r\n* @minjang made their first contribution in https://github.com/vllm-project/vllm/pull/53530\r\n* @MKQuantum made their first contribution in https://github.com/vllm-project/vllm/pull/51866\r\n* @new-TonyWang made their first contribution in https://github.com/vllm-project/vllm/pull/53704\r\n* @nicholaskh-ai made their first contribution in https://github.com/vllm-project/vllm/pull/53819\r\n* @oliverholworthy made their first contribution in https://github.com/vllm-project/vllm/pull/52185\r\n* @prakharPant made their first contribution in https://github.com/vllm-project/vllm/pull/52743\r\n* @Prudhvivuda made their first contribution in https://github.com/vllm-project/vllm/pull/53016\r\n* @rajathpi made their first contribution in https://github.com/vllm-project/vllm/pull/52622\r\n* @ray24777 made their first contribution in https://github.com/vllm-project/vllm/pull/53120\r\n* @roachsinai made their first contribution in https://github.com/vllm-project/vllm/pull/53528\r\n* @RookieCoder-Camera made their first contribution in https://github.com/vllm-project/vllm/pull/49274\r\n* @sandeep-maddipatla made their first contribution in https://github.com/vllm-project/vllm/pull/51777\r\n* @seonjinn made their first contribution in https://github.com/vllm-project/vllm/pull/52204\r\n* @ShengleiFu made their first contribution in https://github.com/vllm-project/vllm/pull/53650\r\n* @shepark made their first contribution in https://github.com/vllm-project/vllm/pull/51302\r\n* @shijuzhao made their first contribution in https://github.com/vllm-project/vllm/pull/45683\r\n* @shipiyouniao made their first contribution in https://github.com/vllm-project/vllm/pull/51979\r\n* @ShuaiShao93 made their first contribution in https://github.com/vllm-project/vllm/pull/53034\r\n* @simon-veitner-redhat made their first contribution in https://github.com/vllm-project/vllm/pull/52980\r\n* @sseanliu made their first contribution in https://github.com/vllm-project/vllm/pull/52041\r\n* @studioego made their first contribution in https://github.com/vllm-project/vllm/pull/52389\r\n* @tanchao made their first contribution in https://github.com/vllm-project/vllm/pull/50191\r\n* @thanhpt1110 made their first contribution in https://github.com/vllm-project/vllm/pull/52720\r\n* @theamalsebastian made their first contribution in https://github.com/vllm-project/vllm/pull/52588\r\n* @thunguo made their first contribution in https://github.com/vllm-project/vllm/pull/53101\r\n* @tolleybot made their first contribution in https://github.com/vllm-project/vllm/pull/51292\r\n* @tommy-asai-sonarsource made their first contribution in https://github.com/vllm-project/vllm/pull/51395\r\n* @tthakkal made their first contribution in https://github.com/vllm-project/vllm/pull/50501\r\n* @ukannika made their first contribution in https://github.com/vllm-project/vllm/pull/52849\r\n* @VBS2004 made their first contribution in https://github.com/vllm-project/vllm/pull/48922\r\n* @wyettzeng made their first contribution in https://github.com/vllm-project/vllm/pull/52209\r\n* @Xuan-1998 made their first contribution in https://github.com/vllm-project/vllm/pull/53008\r\n* @y0hnn made their first contribution in https://github.com/vllm-project/vllm/pull/52002\r\n* @Yiqin-17 made their first contribution in https://github.com/vllm-project/vllm/pull/52925\r\n* @YukioZzz made their first contribution in https://github.com/vllm-project/vllm/pull/51705\r\n* @ZHIHANCHEN03 made their first contribution in https://github.com/vllm-project/vllm/pull/53432\r\n* @Zhou248 made their first contribution in https://github.com/vllm-project/vllm/pull/52560\r\n* @zllion made their first contribution in https://github.com/vllm-project/vllm/pull/46701\r\n* @zwischenraum made their first contribution in https://github.com/vllm-project/vllm/pull/50272\r\n\r\n## Contributors\r\n@AndreasKaratzas, @khluu, @mgoin, @taneem-ibrahim, @yewentao256, @njhill, @BugenZhao, @hmellor, @GirasoleY, @DarkLight1337, @mayuyuace, @WoosukKwon, @jeejeelee, @noooop, @jperezdealgaba, @LucasWilkinson, @wzhao18, @stefankoncarevic, @gau-nernst, @okorzh-amd, @ZJY0516, @HollowMan6, @connorcarpenter15, @gcanlin, @aoshen02, @linitra24, @mhuzaifa3, @Isotr0py, @Etelis, @MatthewBonanni, @TheEpicDolphin, @zupengwang, @atalman, @NickLucche, @rasmith, @elvircrn, @JasonKeyiL, @Hotragn, @waizuichougou, @pmanczak, @divakar-amd, @he-yufeng, @xuebwang-amd, @qgallouedec, @esmeetu, @ZeldaHuang, @Prudhvivuda, @chaunceyjiang, @zxd1997066, @louie-tsai, @andyxning, @pisceskkk, @tjtanaa, @qli88, @tlrmchlsmth, @LopezCastroRoberto, @micah-wil, @hongxiayang, @BabyDrangoner, @yimdev, @mganczarenko, @SageMoore, @biswapanda, @sfeng33, @russellb, @itayalroy, @bigPYJ1151, @drakosha, @qwerqwerqwe8688-jpg, @akii96, @shen-shanshan, @meiyeh123, @reidliu41, @yma11, @frgossen, @lukealonso, @tanchao, @Fangzhou-Ai, @vllm-agent, @djramic, @andrewbcohere, @zyongye, @gty111, @ShuoleiWang, @ZHIHANCHEN03, @KurodaKanbei, @alexeldeib, @khushali9, @xiaohuguo2023, @hungnnvidia, @thisjiang, @arpera, @ShengleiFu, @zhenwei-intel, @jungjiyu, @yzong-rh, @charlifu, @hclsys, @Ronald1995, @mkhazraee, @YukioZzz, @theamalsebastian, @030611, @cr-zhao, @jbyczkow, @tommy-asai-sonarsource, @jikunshang, @bobboli, @LH-and-FPGA, @SayHelloToWorld, @kkt-cohere, @floatlibai, @rajathpi, @lxy-alexander, @gangula-karthik, @ActiveSky, @fxmarty-amd, @sstamenk, @mpashkovskii, @LiuYinfeng01, @chaojun-zhang, @Oxygen56, @daviswer, @vineethsaivs, @Agoni-02, @Andy365-365, @wangxiyuan, @haoyangqian, @Naveassaf, @eilamc14, @sseanliu, @y0hnn, @dineshchitlangia, @yiliu30, @Lossfull, @lxyxinyi, @vanshbhatia-amd, @jcotant-inferact, @chengy-sysu, @92hyungjun, @xyang16, @AmitMY, @zllion, @AnkitNakhawa, @dmvevents, @lengrongfu, @JC-ut0, @Eoin-Houstoun, @sagearc, @wjabbour, @sandeep-maddipatla, @seonjinn, @Yiqin-17, @kyleliang-nv, @thanhpt1110, @MKQuantum, @matthewkotila, @vhagor, @tthakkal, @ShuaiShao93, @hagaikwa-redhat, @shijuzhao, @elwhyjay, @Gregory-Pereira, @ErenAta16, @morrison-turnansky, @DCoEngine, @SoluMilken, @xianbaoqian, @kliuae, @nascheme, @waynehacking8, @stecasta, @vMaroon, @zwischenraum, @Rohan138, @Zhou248, @brianosaurus, @anmolgupt, @wyettzeng, @hao-aaron, @frank-suwen, @danisereb, @fuzzifikation, @KernelClint, @thunguo, @studioego, @shipiyouniao, @cjackal, @LioEinaudi, @guan404ming, @libinta, @mhoqueanik, @minjang, @Edge-Explorer, @shepark, @JiataiWang, @jiahaoliang, @therealnaveenkamal, @ColinZ22, @tolleybot, @almogtavor, @simon-veitner-redhat, @hyeongyun0916, @nicholaskh-ai, @mawong-amd, @adisivaprasad, @ray24777, @robertgshaw2-redhat, @new-TonyWang, @oliverholworthy, @fanxingran, @lucianommartins, @hallerite, @avininjamay8, @VBS2004, @omerpaz95, @Sunt-ing, @Hert4, @wangshangsam, @cogniera, @jeffreywang88, @Xarbirus, @fynnsu, @jiahanc, @zixi-qi, @prakharPant, @haic0, @maobaolong, @xinyu-intel, @CHIPMUNK-T0T, @hangy-amd, @afriedri, @canlahlah, @Alnusjaponica, @ukannika, @pranavthakur0-0, @zzaebok, @Xuan-1998, @JulianZJN, @roachsinai, @ivanium, @CalvinXKY, @yu-xin-c, @rchalamala, @QwertyJack, @Kaif10, @yudigege86, @yiz-liu, @dkrisman, @simondanielsson, @machero, @CherryLemon, @ima-helikoptaaa, @RookieCoder-Camera, @LironKesem, @peakcrosser7, @eligotts, @foraxe, @zufangzhu, @tianmu-li, @jl9876, @lucifer1004, @liranschour, @maithilijoshi20, @li-ukumar, @andyluo7, @Zhenzhong1, @faaany, @joerowell, @juhi10071998, @ganeshr10, @Priyjain-amd, @SubSir, @luyixiao95, @djw8605, @codex\r\n"
+ },
+ {
+ "repo": "sgl-project/sglang",
+ "tag_name": "v0.5.20",
+ "name": "v0.5.20",
+ "published_at": "2026-09-18T22:41:33Z",
+ "html_url": "https://github.com/sgl-project/sglang/releases/tag/v0.5.20",
+ "prerelease": false,
+ "body": "# Highlights\r\n\r\n*713 PRs from 237 contributors.*\r\n\r\n**New models in this release** (see the [cookbook](https://docs.sglang.io/cookbook) for all supported models):\r\n\r\n| Model | Type | PRs | Cookbook |\r\n|---|---|---|---|\r\n| GLM-5.3-Flash | Autoregressive | [#36507](https://github.com/sgl-project/sglang/pull/36507), [#38621](https://github.com/sgl-project/sglang/pull/38621) | [link](https://docs.sglang.io/cookbook/autoregressive/GLM/GLM-5.3-Flash) |\r\n| Hy4-Preview | Autoregressive | [#36805](https://github.com/sgl-project/sglang/pull/36805) | [link](https://docs.sglang.io/cookbook/autoregressive/Tencent/Hy4-Preview) |\r\n| Qwen3.8-Flash-Next | Autoregressive | [#37500](https://github.com/sgl-project/sglang/pull/37500) | [link](https://docs.sglang.io/cookbook/autoregressive/Qwen/Qwen3.8-Flash-Next) |\r\n| K2 Horizon | Autoregressive | [#37654](https://github.com/sgl-project/sglang/pull/37654), [#38033](https://github.com/sgl-project/sglang/pull/38033) | [link](https://docs.sglang.io/cookbook/autoregressive/IFM/K2-Horizon) |\r\n| Nanbeige4.2 | Autoregressive | [#32151](https://github.com/sgl-project/sglang/pull/32151) | |\r\n| SenseNova-U1.5-8B-MoT | Diffusion | [#36606](https://github.com/sgl-project/sglang/pull/36606) | [link](https://docs.sglang.io/cookbook/diffusion/SenseNova/SenseNova-U1.5-8B-MoT) |\r\n| FastH3 (4-step MiniMax-H3 distill) | Diffusion | [#37480](https://github.com/sgl-project/sglang/pull/37480) | [link](https://docs.sglang.io/cookbook/diffusion/MiniMax/MiniMax-H3) |\r\n| VDN-H3 (hybrid-attention MiniMax-H3 distill) | Diffusion | [#37903](https://github.com/sgl-project/sglang/pull/37903) | [link](https://docs.sglang.io/cookbook/diffusion/MiniMax/MiniMax-H3) |\r\n\r\n**Sampling masks for RL rollouts.** With `return_sampling_mask`, each decode step returns the exact token support the sampler drew from and the log-probability of the sampled token under it, so a trainer can replay the rollout without reconstructing top-k or top-p ([#36630](https://github.com/sgl-project/sglang/pull/36630)). Masks now run under overlap scheduling: on Qwen3-8B, decode throughput is 17% higher at batch 1 and 52% higher at batch 64 than the previous implementation. Capacity is set by `--sampling-mask-max-tokens` (default 4096) ([#36631](https://github.com/sgl-project/sglang/pull/36631)). `DisallowedTokensLogitsProcessor` is supported alongside masks ([#38279](https://github.com/sgl-project/sglang/pull/38279)).\r\n\r\n**Unified radix tree.** Branching-point caching for the SWA component keeps the sliding-window state at the point where requests fork from a shared prefix, so branches reuse it instead of recomputing. On DeepSeek-V4-Flash with a shared system prompt, token hit rate rises from 43.8% to 60.8% and mean TTFT falls from 1.57 s to 1.07 s ([#34565](https://github.com/sgl-project/sglang/pull/34565)). An opt-in external linker lets the tree address a shared global memory pool through Mooncake or UMBP ([#37381](https://github.com/sgl-project/sglang/pull/37381)).\r\n\r\n**DSpark under PD with decode context parallelism.** A DCP1 prefill can now transfer its DSpark draft KV to a DCP-N decode, so hybrid models such as Kimi-Linear run DSpark in disaggregated, context-parallel serving. Verified on 8x B300 over NIXL and Mooncake up to 256K input ([#37709](https://github.com/sgl-project/sglang/pull/37709)).\r\n\r\n**Responses API storage is opt-in.** `/v1/responses` no longer retains results in memory unless the server starts with `--enable-response-store`. Without it, retrieval, `previous_response_id` chaining, and background requests return 400; PD deployments cannot enable it ([#39122](https://github.com/sgl-project/sglang/pull/39122)).\r\n\r\n**SGLang Simulator.** A CPU-only simulator runs the real scheduler, radix cache, and hierarchical cache with a latency predictor in place of the model forward. Against measured serving traces it predicts TTFT within about 6% on most traces (up to 10% on the longest 32K to 128K ones) and prefix reuse within 0.05 percentage points, for cache and scheduling studies without GPUs ([#33824](https://github.com/sgl-project/sglang/pull/33824)).\r\n\r\n**Prefill context parallelism v1 removed.** The strategy-based implementation is now the only prefill CP path; the v1 runtime and its CLI options are gone. Prefill CP on HIP, NPU, and MUSA is rejected until those platforms are ported ([#36228](https://github.com/sgl-project/sglang/pull/36228)).\r\n\r\n**Faster model loading on ROCm.** Large pageable host-to-device copies are staged rather than pinned in place, which stops the driver from suspending GPU queues on every eviction. GLM-5.2 at TP4 on 4x MI355X loads in 40.4 s instead of 505.7 s ([#37720](https://github.com/sgl-project/sglang/pull/37720)).\r\n\r\n**Intel XPU joins the release images.** Every tagged release now builds and publishes `lmsysorg/sglang:vX.Y.Z-xpu` from the XPU Dockerfile, so Intel GPU users get a versioned image instead of relying on nightly builds ([#37340](https://github.com/sgl-project/sglang/pull/37340)).\r\n\r\n**DeepSeek-V4 on Blackwell.** TRT-LLM attention kernels now cover DeepSeek-V4's CSA and HCA layers on SM100 and SM103: about 1.2x faster prefill and 1.45x faster decode than FlashMLA at the kernel level on B200 ([#30805](https://github.com/sgl-project/sglang/pull/30805)). FlashInfer MegaMoE is available as `--moe-runner-backend flashinfer_megamoe`; on DeepSeek-V4-Flash NVFP4 at TP4/DP4 it adds up to 11.9% prefill throughput at 8192 tokens per rank, with decode within 2% of the trtllm runner at saturation ([#31470](https://github.com/sgl-project/sglang/pull/31470)).\r\n\r\n**DeepSeek-V4 on RTX PRO 6000.** On SM120 the sparse-MLA indexer now runs on DeepGEMM's paged-MQA kernel and the DeepGEMM FP4 MoE backend is enabled, replacing the torch fallback that was the only working path. On 4x RTX PRO 6000, DeepSeek-V4-Flash decode TPOT drops from 36.1 to 10.5 ms at batch 1 and TTFT falls 20% from 8K to 128K input. Opt-in through the environment flags in the PR ([#29927](https://github.com/sgl-project/sglang/pull/29927)).\r\n\r\n**Dependencies and images.** The CUDA 12 lane is retired; v0.5.19 was the last release with `-cu12x` wheels and images ([#38404](https://github.com/sgl-project/sglang/pull/38404)). sglang-kernel moves to 0.4.7 ([#39346](https://github.com/sgl-project/sglang/pull/39346)) and sgl-deep-gemm to 0.2.0 ([#39371](https://github.com/sgl-project/sglang/pull/39371)). New images: ROCm 10 for MI30x and MI35x with a matching kernel wheel ([#38763](https://github.com/sgl-project/sglang/pull/38763)), gfx1151 for Strix Halo / Ryzen AI MAX+ ([#33939](https://github.com/sgl-project/sglang/pull/33939)), and Moore Threads MUSA ([#36709](https://github.com/sgl-project/sglang/pull/36709)). ROCm 7.0 CI, images, and kernel wheel are retired ([#38632](https://github.com/sgl-project/sglang/pull/38632), [#38767](https://github.com/sgl-project/sglang/pull/38767)).\r\n\r\n*Full release notes by category below; breaking changes are at the end.*\r\n\r\n## Speculative Decoding\r\n\r\n* [Speculative Decoding] Add native UNO serving support: [#37667](https://github.com/sgl-project/sglang/pull/37667)\r\n* feat: add optimized Domino rollout to DFlash V2: [#36899](https://github.com/sgl-project/sglang/pull/36899)\r\n* feat: support TP>1 Domino rollout for DFlash V2: [#37069](https://github.com/sgl-project/sglang/pull/37069)\r\n* [Spec] Stage Inkling MTP draft metadata before verify: [#38169](https://github.com/sgl-project/sglang/pull/38169)\r\n* [Spec] Allow speculative workers to stage prefill shared reads: [#38554](https://github.com/sgl-project/sglang/pull/38554)\r\n* [Spec] Support large MTP batches in short-convolution metadata: [#38558](https://github.com/sgl-project/sglang/pull/38558)\r\n* [EAGLE] Prune draft-extend logits to selected rows (+2.5% output tokens/s/GPU at 102K context per rank): [#35546](https://github.com/sgl-project/sglang/pull/35546)\r\n* [KDA] Support ReplaySSM ring-write in the fused chain-verify kernel: [#36821](https://github.com/sgl-project/sglang/pull/36821)\r\n* [GDN] Amortize ReplaySSM checkpoint materialization: [#35544](https://github.com/sgl-project/sglang/pull/35544)\r\n* perf(gdn): select ReplaySSM verify loop unrolling by shape: [#36970](https://github.com/sgl-project/sglang/pull/36970)\r\n* Allow custom policy for adaptive speculative decoding: [#37274](https://github.com/sgl-project/sglang/pull/37274)\r\n* Support speculative decoding with unified SWA memory: [#36403](https://github.com/sgl-project/sglang/pull/36403)\r\n* Improve CUDA graph and speculative execution output handling: [#37329](https://github.com/sgl-project/sglang/pull/37329)\r\n* Fix DSpark CUDA graph replay with MegaMoE TP attention: [#34919](https://github.com/sgl-project/sglang/pull/34919)\r\n* [Fix][Mamba] Clear deferred init metadata before speculative decode: [#37165](https://github.com/sgl-project/sglang/pull/37165)\r\n* [Fix] Track DFlash Mamba state at checkpoint boundaries: [#37818](https://github.com/sgl-project/sglang/pull/37818)\r\n* [Fix] Load Qwen3.5 MTP embedding under PP: [#37471](https://github.com/sgl-project/sglang/pull/37471)\r\n* fix: stop shadowing the DSpark shared-experts fusion guard: [#39366](https://github.com/sgl-project/sglang/pull/39366)\r\n* [Profiler] Label draft-runner steps DRAFT and target verify VERIFY in step spans: [#38630](https://github.com/sgl-project/sglang/pull/38630)\r\n\r\n## Piecewise & Breakable CUDA Graph\r\n\r\n* [Memory] Size the CUDA graph pool from warmup measurements and fix graph-pool borrowing: [#36911](https://github.com/sgl-project/sglang/pull/36911)\r\n* [Memory] Retire graph borrow pool before updating static runs: [#37966](https://github.com/sgl-project/sglang/pull/37966)\r\n* [Memory] Reuse output storage across full prefill CUDA graphs: [#38038](https://github.com/sgl-project/sglang/pull/38038)\r\n* [Unified Memory] Enable prefill cuda-graph capture: [#37418](https://github.com/sgl-project/sglang/pull/37418)\r\n* Decouple ragged CUDA graph request and token capacities: [#37300](https://github.com/sgl-project/sglang/pull/37300)\r\n* Reuse live CUDA graph executables during dedup registration: [#39176](https://github.com/sgl-project/sglang/pull/39176)\r\n* Scope graph-pool borrowing to the runtime and reduce fragmentation: [#39177](https://github.com/sgl-project/sglang/pull/39177)\r\n* Expose a capacity check for graph-pool borrows: [#39178](https://github.com/sgl-project/sglang/pull/39178)\r\n* Keep graph-pool borrows on their allocation stream: [#39180](https://github.com/sgl-project/sglang/pull/39180)\r\n* Add a provider hook for prefill-buffer ceilings: [#39182](https://github.com/sgl-project/sglang/pull/39182)\r\n* [Refactor] Generalize attention graph variants in the decode runner: [#38993](https://github.com/sgl-project/sglang/pull/38993)\r\n* [Fix] Coordinate FullCG prefix variants across DP ranks: [#37888](https://github.com/sgl-project/sglang/pull/37888)\r\n* [Fix] Keep a shared MAX_LEN prefill CUDA graph bucket when the graph captures a DP gather (MegaMoE sparse-DP hang): [#37933](https://github.com/sgl-project/sglang/pull/37933)\r\n* [Fix] Key CUDA graph dedup signatures on kernel function identity: [#37657](https://github.com/sgl-project/sglang/pull/37657)\r\n* [Bug] Include DSA variant in exact-bucket graph admission: [#38992](https://github.com/sgl-project/sglang/pull/38992)\r\n* Fix KV-canary workspace accounting after graph capture: [#38596](https://github.com/sgl-project/sglang/pull/38596)\r\n\r\n## Attention Backends\r\n\r\n* [DSv4] Integrate TRT-LLM DSv4 Attention for SM100/103 (kernel-level prefill about 1.2x and decode about 1.45x faster than FlashMLA on B200): [#30805](https://github.com/sgl-project/sglang/pull/30805)\r\n* [DSV4] Support raw-index output in TopK v2: [#33672](https://github.com/sgl-project/sglang/pull/33672)\r\n* [DSV4] hc-prenorm: fuse the combine step into a Triton kernel: [#35118](https://github.com/sgl-project/sglang/pull/35118)\r\n* [FlashInfer] Avoid D2H sync for sliding-window lengths (TPOT 5.4 to 5.1 ms on 8x A800): [#32218](https://github.com/sgl-project/sglang/pull/32218)\r\n* [Attention] Size FlashInfer MLA indptr buffers to the padded max batch: [#38590](https://github.com/sgl-project/sglang/pull/38590)\r\n* [Perf] trtllm_mla: reuse the fused fp8 KV/Q prepare on target verify: [#39232](https://github.com/sgl-project/sglang/pull/39232)\r\n* [Performance] Optimize Qwen3.5 GDN prefill projection layouts: [#36267](https://github.com/sgl-project/sglang/pull/36267)\r\n* Fix Qwen3.5 GDN multi-item scoring (+18.8% throughput, P50 latency -16%): [#33922](https://github.com/sgl-project/sglang/pull/33922)\r\n* Support NoPE layers in the tokenspeed_mla FP8 prefill hook: [#38152](https://github.com/sgl-project/sglang/pull/38152)\r\n* fix(qsa): make the paged sparse-decode gather memory-safe (zero-fill scratch, int64 offsets, dequant FP8 on gather): [#38851](https://github.com/sgl-project/sglang/pull/38851)\r\n* fix(qsa): dequantize FP8 cached prefixes in the sparse prefill kernels: [#38855](https://github.com/sgl-project/sglang/pull/38855)\r\n* Reland fix(qsa): clamp the compress gather to the rows (#38346): [#39446](https://github.com/sgl-project/sglang/pull/39446)\r\n* [Qwen 3.8 Next] reuse old cuda stream instead of endlessly creating streams: [#39474](https://github.com/sgl-project/sglang/pull/39474)\r\n* [Fix][DSA] Bound prefill Triton specializations for page-table stride: [#37093](https://github.com/sgl-project/sglang/pull/37093)\r\n* [Fix] Wait for PDL before reading DeepSeek V4 K cache locations: [#38409](https://github.com/sgl-project/sglang/pull/38409)\r\n* [Fix] Fix Llama 4 FA3 local attention with paged KV cache: [#32902](https://github.com/sgl-project/sglang/pull/32902)\r\n\r\n## MoE & Expert Parallelism\r\n\r\n* [NVIDIA] Support flashinfer Mega Moe (prefill input throughput up to +11.9% vs the trtllm runner on DeepSeek-V4-Flash NVFP4 at 8192 tokens per rank): [#31470](https://github.com/sgl-project/sglang/pull/31470)\r\n* [MoE Refactor] Migrate SM100 trtllm-gen mxfp4 MoE onto MoeRunner: [#32405](https://github.com/sgl-project/sglang/pull/32405)\r\n* [Deepseek V4] Keep fp32 routing weights in the mxfp4 trtllm MoE: [#33608](https://github.com/sgl-project/sglang/pull/33608)\r\n* Keep fp32 routing weights in the fp8 block-scale and bf16 trtllm MoE: [#33631](https://github.com/sgl-project/sglang/pull/33631)\r\n* Drop the routing bias casts in flashinfer trtllm MoE: [#33591](https://github.com/sgl-project/sglang/pull/33591)\r\n* Fix DeepSeek-V4 routing: sqrtsoftplus underflow and unfloored renorm: [#34459](https://github.com/sgl-project/sglang/pull/34459)\r\n* keeping router GEMM in fp32 for deterministic inference (DeepSeek V3/V4): [#38176](https://github.com/sgl-project/sglang/pull/38176)\r\n* Fuse Nemotron latent MoE projection and shared add: [#30430](https://github.com/sgl-project/sglang/pull/30430)\r\n* [Kernel] Replace dsv3_router_gemm with the unified tiny GEMM: [#34693](https://github.com/sgl-project/sglang/pull/34693)\r\n* [Kernel] Avoid zero-bias allocation in fused softmax routing: [#36811](https://github.com/sgl-project/sglang/pull/36811)\r\n* [Kimi-K3] Accept fp32 routing weights in the fused MoE finalize: [#38612](https://github.com/sgl-project/sglang/pull/38612)\r\n* Cast fp32 routing weights to bf16 in the Kimi-K3 fused finalize: [#38588](https://github.com/sgl-project/sglang/pull/38588)\r\n* Use fp32 in TRTLLM all reduce buffers: [#36143](https://github.com/sgl-project/sglang/pull/36143)\r\n* [Comm] Drop the in-tree MNNVL CuTe DSL port in favor of FlashInfer 0.6.18: [#37206](https://github.com/sgl-project/sglang/pull/37206)\r\n* [Moe] Fix flashinfer_trtllm silently dropping swiglu_limit clamped SwiGLU activation (GLM-5.3-Flash FP8 GSM8K 0.928 to 0.936, matching the Triton runner): [#39920](https://github.com/sgl-project/sglang/pull/39920)\r\n* Fix MegaMoE buffer allocation and caching for effective SM budgets: [#39223](https://github.com/sgl-project/sglang/pull/39223)\r\n* Fix native MoE handling of noncontiguous top-k IDs: [#36407](https://github.com/sgl-project/sglang/pull/36407)\r\n* [Fix] Accept int64 top-k IDs in FlashInfer routed MoE packer: [#32882](https://github.com/sgl-project/sglang/pull/32882)\r\n* fix(moe): cast filtered-activation expert_ids to int32 for torch.compile: [#38085](https://github.com/sgl-project/sglang/pull/38085)\r\n* fix(gpt-oss): avoid duplicate MoE reduction with DP attention: [#37199](https://github.com/sgl-project/sglang/pull/37199)\r\n* Fix mooncake scale joiner groups: [#38503](https://github.com/sgl-project/sglang/pull/38503)\r\n\r\n## Quantization\r\n\r\n* [Quant][ue8m0 fix] group requant_weight_ue8m0 reduce reserved gpu memory: [#31755](https://github.com/sgl-project/sglang/pull/31755)\r\n* [1/N] Quantization Refactor: remove dead code and dedup the FP4 marlin helpers: [#37552](https://github.com/sgl-project/sglang/pull/37552)\r\n* Delete cutlass_mla, non-Marlin GPTQ, AWQ AOT kernel, and Dual Chunk Flash Attention: [#32114](https://github.com/sgl-project/sglang/pull/32114)\r\n* fix(modelopt_fp4): skip NVFP4 swiglu-fusion interleave for shared experts with swiglu_limit: [#37378](https://github.com/sgl-project/sglang/pull/37378)\r\n* Disable Hopper GLM shared-expert fusion for modelopt_fp4 Marlin: [#37325](https://github.com/sgl-project/sglang/pull/37325)\r\n* [Fix] Preserve FP32 in SM107 MXFP8 fallback: [#37489](https://github.com/sgl-project/sglang/pull/37489)\r\n* Fix block-scale swizzling device placement: [#37849](https://github.com/sgl-project/sglang/pull/37849)\r\n* Keep NVFP4 blockscale swizzle padding on the input device: [#39141](https://github.com/sgl-project/sglang/pull/39141)\r\n* Fix dummy initialization of inverse weight scales: [#35491](https://github.com/sgl-project/sglang/pull/35491)\r\n* [Chore] harden checkpoint quantization metadata parsing: [#36922](https://github.com/sgl-project/sglang/pull/36922)\r\n\r\n## Parallelism & Disaggregation\r\n\r\n* [PD] Transfer the DCP-replicated DSPARK draft KV in DCP1->DCP-N relayouts: [#37709](https://github.com/sgl-project/sglang/pull/37709)\r\n* [Qwen3.8-Next] Add PD state transfer for Flash Next: [#36651](https://github.com/sgl-project/sglang/pull/36651)\r\n* [Intra-node PD][DSV4] Pack all layers into one batch for INTRA_NODE_NVLINK path (KV transfer latency 178 to 119 ms, E2E latency -20% on a 2P1D B200 node): [#38984](https://github.com/sgl-project/sglang/pull/38984)\r\n* [PD] Share the prefill->decode failure notification across backends: [#36612](https://github.com/sgl-project/sglang/pull/36612)\r\n* [PD] Gate deferred decode KV release on backend capability: [#37454](https://github.com/sgl-project/sglang/pull/37454)\r\n* [PD] Bound transfer engine init with `SGLANG_DISAGGREGATION_ENGINE_INIT_TIMEOUT`: [#37874](https://github.com/sgl-project/sglang/pull/37874)\r\n* [PD] Preserve the prefill rank during rebootstrap: [#39357](https://github.com/sgl-project/sglang/pull/39357)\r\n* [PD] Optimize paged allocator free-list release: [#37146](https://github.com/sgl-project/sglang/pull/37146)\r\n* [PD][LoRA] Gate decode admission on adapter slots: [#39332](https://github.com/sgl-project/sglang/pull/39332)\r\n* [Unified Memory] PD disaggregation for every unified pool shape: [#37506](https://github.com/sgl-project/sglang/pull/37506)\r\n* fix(disagg): poll receivers during decode preallocation: [#37483](https://github.com/sgl-project/sglang/pull/37483)\r\n* Fix DSA compression tail capacity for PD decode request slots: [#38417](https://github.com/sgl-project/sglang/pull/38417)\r\n* fix(glm-5.2-nvfp4): bound Mooncake synchronous transfer batches: [#32758](https://github.com/sgl-project/sglang/pull/32758)\r\n* [DCP] Allow fi_a2a on single-node systems Blackwell without MNNVL fabric ( ex B200 B300) (median TPOT 1.5 to 5.5% lower than the NCCL all-to-all on 8x B200): [#37767](https://github.com/sgl-project/sglang/pull/37767)\r\n* [DCP] Resolve --dcp-comm-backend to fi_a2a/a2a by default for every model: [#39165](https://github.com/sgl-project/sglang/pull/39165)\r\n* [Feature] Unified memory: support decode context parallelism for Kimi-Linear: [#36890](https://github.com/sgl-project/sglang/pull/36890)\r\n* [Feature] Unified memory: support decode context parallelism for the trtllm_mla family: [#37693](https://github.com/sgl-project/sglang/pull/37693)\r\n* [Perf] Unified memory: close the DCP decode gap on Blackwell: [#37926](https://github.com/sgl-project/sglang/pull/37926)\r\n* [Fix] Merge adjacent KV-row frees so a mid-page split under DCP cannot double-free: [#38941](https://github.com/sgl-project/sglang/pull/38941)\r\n* [Fix][DCP] Localize widened KV ids in MLA retraction CPU backup/restore: [#39487](https://github.com/sgl-project/sglang/pull/39487)\r\n* [CP V1 Deprecation 2/5] Make strategy prefill CP canonical: [#36223](https://github.com/sgl-project/sglang/pull/36223)\r\n* [CP V1 Deprecation 3/5] Remove generic prefill CP v1 runtime: [#36228](https://github.com/sgl-project/sglang/pull/36228)\r\n* [CP V1 Deprecation 3.5/5] Deprecate HIP/NPU/MUSA prefill CP and remove legacy implementation: [#38293](https://github.com/sgl-project/sglang/pull/38293)\r\n* [CP V1 Deprecation 4/5] Canonicalize prefill CP API names: [#36229](https://github.com/sgl-project/sglang/pull/36229)\r\n* [CP V1 Deprecation 5/5] Update prefill CP documentation: [#36230](https://github.com/sgl-project/sglang/pull/36230)\r\n* fix: gather CP-sharded tokens before TP-sharded dense MLP under prefill CP: [#38078](https://github.com/sgl-project/sglang/pull/38078)\r\n* Fix inflated row pitch when a CP round-robin shard has a single row: [#34142](https://github.com/sgl-project/sglang/pull/34142)\r\n* [SP] Make attention-TP sequence sharding a per-forward batch property: [#37546](https://github.com/sgl-project/sglang/pull/37546)\r\n* [Fix] Stamp sequence-parallel state on dummy forward batches: [#38564](https://github.com/sgl-project/sglang/pull/38564)\r\n* [DP Attn] Fix crash for no token all-gather case: [#39899](https://github.com/sgl-project/sglang/pull/39899)\r\n* [Fix] Disable NCCL graph buffer registration for the TP LM-head all-to-all (pure-DP decode hang under request bursts): [#38936](https://github.com/sgl-project/sglang/pull/38936)\r\n* [Misc] Extract PP dynamic chunk sizing into a `DynamicChunkSizer` scheduler component: [#37674](https://github.com/sgl-project/sglang/pull/37674)\r\n* [Fix] Lock PP dynamic-chunk profiling requests before releasing through the tree cache: [#37509](https://github.com/sgl-project/sglang/pull/37509)\r\n* [Fix] Apply the attention-CP broadcast result in PP dynamic-chunk profiling: [#37669](https://github.com/sgl-project/sglang/pull/37669)\r\n* [Fix] Broadcast PP dynamic-chunk profiling failures so every rank disables together: [#37675](https://github.com/sgl-project/sglang/pull/37675)\r\n\r\n## Scheduler & Runtime\r\n\r\n* [Scheduler] Add HRRN schedule policy to significantly reduce TTFT (`--schedule-policy hrrn`; mean TTFT -69% and p99 -8% vs FCFS on a GLM-5.2 production trace): [#32911](https://github.com/sgl-project/sglang/pull/32911)\r\n* [Scheduler] Make request-timeout aborts rank-consistent to fix TP collective hangs: [#37143](https://github.com/sgl-project/sglang/pull/37143)\r\n* [Scheduler] Count the parked chunked-prefill request in the busy mem check: [#37502](https://github.com/sgl-project/sglang/pull/37502)\r\n* [Scheduler] Unify per-iteration request intake into ingest_requests(): [#38389](https://github.com/sgl-project/sglang/pull/38389)\r\n* Fail fast on undersized swa pool: [#37610](https://github.com/sgl-project/sglang/pull/37610)\r\n* Reduce tokenizer overhead and offload CUDA VMM publication: [#37330](https://github.com/sgl-project/sglang/pull/37330)\r\n* Keep VMM capability votes on CPU: [#38722](https://github.com/sgl-project/sglang/pull/38722)\r\n* state_capturer: pin the exact host-cache size via mmap + cudaHostRegister: [#37285](https://github.com/sgl-project/sglang/pull/37285)\r\n* Generalize auxiliary outputs: [#39164](https://github.com/sgl-project/sglang/pull/39164)\r\n* [Model Loader] Split weight loading from postprocessing: [#34981](https://github.com/sgl-project/sglang/pull/34981)\r\n* Fix gpt-oss RunAI streamer weight ownership: [#38908](https://github.com/sgl-project/sglang/pull/38908)\r\n* Fix RunAI object-storage checkpoint index filtering: [#38988](https://github.com/sgl-project/sglang/pull/38988)\r\n* Clean logging under --weight-loader-prefetch-checkpoints: [#33930](https://github.com/sgl-project/sglang/pull/33930)\r\n* [Fix] Release NCCL on scheduler exit and let the ASGI server own shutdown: [#39560](https://github.com/sgl-project/sglang/pull/39560)\r\n* [Fix] Aggregate all IPC weight update responses: [#39534](https://github.com/sgl-project/sglang/pull/39534)\r\n* [Fix] Avoid scanning crash-dump token buffers during GC: [#37297](https://github.com/sgl-project/sglang/pull/37297)\r\n* add reindex_device_id to device OOT plugin: [#36220](https://github.com/sgl-project/sglang/pull/36220)\r\n* [Metrics] Add rolling scheduler utilization counters: [#37461](https://github.com/sgl-project/sglang/pull/37461)\r\n* [Metrics] Export scheduler stage wall time: [#37636](https://github.com/sgl-project/sglang/pull/37636)\r\n* [Metrics] Report logical prefill token counts: [#38566](https://github.com/sgl-project/sglang/pull/38566)\r\n* [Observability] Fix missing e2e/decode/inference latency span attributes: [#37789](https://github.com/sgl-project/sglang/pull/37789)\r\n* [Ray] Support Ray metric backend for engine metrics: [#31415](https://github.com/sgl-project/sglang/pull/31415)\r\n* [Profiler] Add SGLANG_PROFILE_BY_STAGE_DECODE_MIN_BS to defer the decode-stage capture: [#38067](https://github.com/sgl-project/sglang/pull/38067)\r\n* Add num_prealloc_ready_tokens to decode load snapshot: [#38065](https://github.com/sgl-project/sglang/pull/38065)\r\n* [Bench] Support real-traffic replay with early-stop-aware steady-state metrics in bench_one_batch_server: [#37469](https://github.com/sgl-project/sglang/pull/37469)\r\n* [Benchmark] Add an opt-out for the token-capacity check: [#39284](https://github.com/sgl-project/sglang/pull/39284)\r\n* Fix reasoning metrics and add TPOT to bench_multiturn: [#35443](https://github.com/sgl-project/sglang/pull/35443)\r\n\r\n## Sampling\r\n\r\n* [Sampling] Capture masks from sampler support: [#36630](https://github.com/sgl-project/sglang/pull/36630)\r\n* [Sampling] Support sampling masks with overlap scheduling (Qwen3-8B decode throughput +17% at batch 1 and +52% at batch 64 vs the previous non-overlap path): [#36631](https://github.com/sgl-project/sglang/pull/36631)\r\n* [Sampling] Allow sampling-mask replay with DisallowedTokensLogitsProcessor: [#38279](https://github.com/sgl-project/sglang/pull/38279)\r\n* perf: use Gumbel-max trick in the main sampler to cut decode CPU dispatch (Qwen3.5-2B decode +75% at batch 1 and +23% at batch 32): [#38117](https://github.com/sgl-project/sglang/pull/38117)\r\n* Fix custom logit processor params when num_tokens_in_batch is used: [#38730](https://github.com/sgl-project/sglang/pull/38730)\r\n\r\n## HiCache & Radix Cache\r\n\r\n* [Unified Tree] Support Branching-Point Caching for the SWA Component (shared-prefix token hit rate 43.8% to 60.8%, mean TTFT 1.57 to 1.07 s on DeepSeek-V4-Flash): [#34565](https://github.com/sgl-project/sglang/pull/34565)\r\n* [Unified Tree] Port SWA Branching-Point Caching to the Rust TreeCore: [#37584](https://github.com/sgl-project/sglang/pull/37584)\r\n* [Unified Tree] fix: make dfs weight ordering iterative: [#38313](https://github.com/sgl-project/sglang/pull/38313)\r\n* [Unified Tree] Preserve aux LRU recency when splitting nodes: [#38482](https://github.com/sgl-project/sglang/pull/38482)\r\n* [Unified Cache][5/N]: Integrate external linker mode end to end: [#37381](https://github.com/sgl-project/sglang/pull/37381)\r\n* [Unified Cache][6/N]: Add UMBP external linker: [#37578](https://github.com/sgl-project/sglang/pull/37578)\r\n* [Unified Cache][7/N] Support MTP, EAGLE, and DSpark draft KV caches in the external linker: [#37914](https://github.com/sgl-project/sglang/pull/37914)\r\n* [Unified Cache][AMD] Support DeepSeek-V4 unified KV in direct external linkers: [#38269](https://github.com/sgl-project/sglang/pull/38269)\r\n* [Rust TreeCore] Harden runtime and CI parity: [#37303](https://github.com/sgl-project/sglang/pull/37303)\r\n* [Rust TreeCore] Support external cache linker: [#37306](https://github.com/sgl-project/sglang/pull/37306)\r\n* [Rust] Rename mem-cache to sglang-radix-tree: [#37290](https://github.com/sgl-project/sglang/pull/37290)\r\n* [KV Shard 1/4] Logical-page placement with UnifiedRadixCache: [#38356](https://github.com/sgl-project/sglang/pull/38356)\r\n* [Cache] Forward fast prefix matching capability: [#37844](https://github.com/sgl-project/sglang/pull/37844)\r\n* [Refactor] Let eviction policies take construction parameters: [#37795](https://github.com/sgl-project/sglang/pull/37795)\r\n* [Perf] Walk the radix tree by offset instead of re-slicing token storage (ported from #36507): [#37324](https://github.com/sgl-project/sglang/pull/37324)\r\n* [Perf] Vectorize alloc_extend_naive to remove the per-request Python loop: [#37938](https://github.com/sgl-project/sglang/pull/37938)\r\n* [Radix Cache] Fix PureSWA tail release without insertion: [#38349](https://github.com/sgl-project/sglang/pull/38349)\r\n* fix: preserve SWA host lock on node split: [#38138](https://github.com/sgl-project/sglang/pull/38138)\r\n* fix: collect prefix hash values iteratively: [#38204](https://github.com/sgl-project/sglang/pull/38204)\r\n* fix: align write-through pending across tree cores: [#37278](https://github.com/sgl-project/sglang/pull/37278)\r\n* [HiCache] Add MLA host-dedup primitives: [#36800](https://github.com/sgl-project/sglang/pull/36800)\r\n* [HiCache] Replace skip_lock_node_ids with a segment lock protocol: [#36848](https://github.com/sgl-project/sglang/pull/36848)\r\n* [HiCache] Buffer mode support sidecar pool: [#37424](https://github.com/sgl-project/sglang/pull/37424)\r\n* [HiCache] buffer mode: anchor-lock staged prefetches by default: [#37464](https://github.com/sgl-project/sglang/pull/37464)\r\n* [HiCache] L3 storage prefetch lifecycle metrics and cross-tier attribution fixes: [#37503](https://github.com/sgl-project/sglang/pull/37503)\r\n* [HiCache] Count hit allocations and in-flight backups in the buffer pipeline idle check: [#37883](https://github.com/sgl-project/sglang/pull/37883)\r\n* [HiCache] Fix side pools to use resolved host allocator: [#38350](https://github.com/sgl-project/sglang/pull/38350)\r\n* [HiCache] Account for newly pinned ancestors in load-back quota: [#38481](https://github.com/sgl-project/sglang/pull/38481)\r\n* [HiCache] Release buffer prefetch anchor locks during storage cleanup: [#38483](https://github.com/sgl-project/sglang/pull/38483)\r\n* [HiCache] Publish a host store event for storage-prefetch refills: [#38486](https://github.com/sgl-project/sglang/pull/38486)\r\n* [HiCache][LoRA] Isolate storage pages by extra key: [#38577](https://github.com/sgl-project/sglang/pull/38577)\r\n* [HiCache][LoRA] Simplify decode offload hash inputs: [#39162](https://github.com/sgl-project/sglang/pull/39162)\r\n* [HiCache] fix: preserve SWA host lock boundaries across splits: [#38835](https://github.com/sgl-project/sglang/pull/38835)\r\n* [HiCache] Keep the file backend temp file name within NAME_MAX: [#38925](https://github.com/sgl-project/sglang/pull/38925)\r\n* [HiCache] Rework the buffer-mode storage prefetch pipeline and retry bookkeeping: [#39283](https://github.com/sgl-project/sglang/pull/39283)\r\n* [HiCache] Optimize buffer-mode storage existence bookkeeping: [#39480](https://github.com/sgl-project/sglang/pull/39480)\r\n* HiCache: Add @rank_consensus to various functions: [#37425](https://github.com/sgl-project/sglang/pull/37425)\r\n* HiCache: Reduce the number of `all_reduce` in `check_hicache_events` for PP (TTFT -7% on DeepSeek-V4-Flash with HiCache L3, PP4 TP2): [#37562](https://github.com/sgl-project/sglang/pull/37562)\r\n* Fix buffer-mode idle tracking and VLM memory sizing: [#37567](https://github.com/sgl-project/sglang/pull/37567)\r\n* fix: keep queued Mooncake linker loads after abort: [#38195](https://github.com/sgl-project/sglang/pull/38195)\r\n* fix(nixl): make FILE path-mode devId globally unique: [#34362](https://github.com/sgl-project/sglang/pull/34362)\r\n* Scope prefetch cache state to the request attempt: [#39318](https://github.com/sgl-project/sglang/pull/39318)\r\n* [Fix] HiCache startup ImportError on the pinned kernel wheel: [#39516](https://github.com/sgl-project/sglang/pull/39516)\r\n* [Mem Cache] Make `free_swa` sync-free on `page_size == 1`: [#36723](https://github.com/sgl-project/sglang/pull/36723)\r\n* [Mem Cache] Drop the `torch.unique` sync from the SWA page expansion: [#37463](https://github.com/sgl-project/sglang/pull/37463)\r\n* [Mem Cache] Split duplicate insert frees at the SWA eviction floor: [#37481](https://github.com/sgl-project/sglang/pull/37481)\r\n* [Mem Cache] Require page-aligned starts in `free_segment` and drop the boundary trim: [#37729](https://github.com/sgl-project/sglang/pull/37729)\r\n* [Mem Cache] Route hybrid SWA full-side kv-row frees through `free_segment`: [#37876](https://github.com/sgl-project/sglang/pull/37876)\r\n* [Mem Cache] Move the unified-memory allocators into `allocator/` and split the composites out: [#38072](https://github.com/sgl-project/sglang/pull/38072)\r\n* [Mem Cache] Free hybrid SWA pages by one representative per page on `page_size > 1`: [#38159](https://github.com/sgl-project/sglang/pull/38159)\r\n* [Mem Cache] skip duplicates host evict via environ: [#38462](https://github.com/sgl-project/sglang/pull/38462)\r\n* [Misc] Resolve SWA ownership at enqueue time for grouped free(): [#36646](https://github.com/sgl-project/sglang/pull/36646)\r\n* Fix unified SWA: size a non-owner's v2p by the id space it must address: [#37560](https://github.com/sgl-project/sglang/pull/37560)\r\n* Size the unified read-table grid from bs, and fuse the allocator's tombstone scatters: [#37511](https://github.com/sgl-project/sglang/pull/37511)\r\n* Build the unified read stream directly, without the page-table rectangle (decode CUDA graph memory 2.89 to 1.89 GB on Kimi-Linear-48B TP2): [#37512](https://github.com/sgl-project/sglang/pull/37512)\r\n* fix(unified-memory): evict Full KV for Mamba byte shortfalls: [#36713](https://github.com/sgl-project/sglang/pull/36713)\r\n* Store mamba prefix-cache checkpoints at the configured SSM state dtype: [#34820](https://github.com/sgl-project/sglang/pull/34820)\r\n* Fix mamba radix cache ssm state indexing: [#37836](https://github.com/sgl-project/sglang/pull/37836)\r\n* [Fix] Vacuous marker writes in the cache tests, and an undebited Mamba admission slot: [#36415](https://github.com/sgl-project/sglang/pull/36415)\r\n\r\n## LoRA\r\n\r\n* [LoRA] Support MoE in full and breakable prefill CUDA graphs (prefill throughput +39.5 to 40.9% with Triton LoRA on Qwen breakable graphs, GB300): [#38578](https://github.com/sgl-project/sglang/pull/38578)\r\n* [Intel][XPU][LoRA] Enable LoRA on Intel XPU: [#30345](https://github.com/sgl-project/sglang/pull/30345)\r\n\r\n## Multimodal\r\n\r\n* [Perf] Optimize Qwen3-VL unique-image serving on H100: [#36411](https://github.com/sgl-project/sglang/pull/36411)\r\n* fix(multimodal): handle tensor images in exact-token preprocessing: [#30368](https://github.com/sgl-project/sglang/pull/30368)\r\n* Fix multimodal embedding cache retaining full batches through views: [#39120](https://github.com/sgl-project/sglang/pull/39120)\r\n* [Fix][Qwen-VL] Normalize `<image>` sentinel on artifact fast path: [#39278](https://github.com/sgl-project/sglang/pull/39278)\r\n* fix(glm4v): disambiguate mixed image video offsets: [#37971](https://github.com/sgl-project/sglang/pull/37971)\r\n* [Fix] Alpha-channel images and tool-result media ordering (port of #36507): [#37320](https://github.com/sgl-project/sglang/pull/37320)\r\n* [MM] Add flag to force Kimi image preprocessing onto CPU: [#39148](https://github.com/sgl-project/sglang/pull/39148)\r\n* [VLM] Retire aborted disaggregated prefill results: [#36988](https://github.com/sgl-project/sglang/pull/36988)\r\n* fix(vlm): contain EPD request lifecycle failures: [#36944](https://github.com/sgl-project/sglang/pull/36944)\r\n* fix(vlm): harden EPD receiver validation and liveness: [#36945](https://github.com/sgl-project/sglang/pull/36945)\r\n* fix(vlm): make EPD cache publication transactional: [#36949](https://github.com/sgl-project/sglang/pull/36949)\r\n\r\n## Model Support & Optimizations\r\n\r\n* [Cosmos3] Add cosmos3 Reasoner to llm only inference: [#33572](https://github.com/sgl-project/sglang/pull/33572)\r\n* Support NemotronH_Omni_Reasoning_V3 in SGLang: [#35599](https://github.com/sgl-project/sglang/pull/35599)\r\n* Inference Support Mamba 2 and 1 (Mamba-Codestral-7B, Falcon-Mamba-7B, and the state-spaces Mamba checkpoints): [#34556](https://github.com/sgl-project/sglang/pull/34556)\r\n* [GLM-5.3 Flash] Restore and enable KPool metadata fusion: [#38845](https://github.com/sgl-project/sglang/pull/38845)\r\n* MiniMax-M3: share the sparse index top-k across layers and reuse the decode top-k buffer (output throughput +6.4 to +20.6% at 80K input on 8x MI350X): [#36527](https://github.com/sgl-project/sglang/pull/36527)\r\n* MiniMax-M3: Triton split-K router GEMV with in-kernel fixup: [#36557](https://github.com/sgl-project/sglang/pull/36557)\r\n* perf(lfm2): fuse gating and short convolution on SM90: [#37622](https://github.com/sgl-project/sglang/pull/37622)\r\n* [GraniteMoE] Load split per-expert quantized MoE weights: [#37679](https://github.com/sgl-project/sglang/pull/37679)\r\n* Fix Muse Glimmer ModelOpt mixed weight mapping: [#37510](https://github.com/sgl-project/sglang/pull/37510)\r\n* [Fix] Fix Spark2.5 hybrid SWA config: [#37335](https://github.com/sgl-project/sglang/pull/37335)\r\n* fix(mamba): unify causal_conv1d col* dtype to x (MiniCPM-V-4.6 GDN prefill bf16/fp16 mismatch): [#38039](https://github.com/sgl-project/sglang/pull/38039)\r\n* [Refactor] Clarify DeepSeek V4 metadata names for V4.1: [#38947](https://github.com/sgl-project/sglang/pull/38947)\r\n* [Refactor] Generalize DeepSeek V4 compressed pool management: [#38954](https://github.com/sgl-project/sglang/pull/38954)\r\n\r\n## Kernel Library\r\n\r\n* [Kernel] GLM 5.3 Flash related kernels (ported from #36507): [#37477](https://github.com/sgl-project/sglang/pull/37477)\r\n* [Kernel] Register merged diffusion agent kernels with KDA backend: [#37385](https://github.com/sgl-project/sglang/pull/37385)\r\n* [Kernel] Add fused MoE Triton configs for Qwen3.8-Flash-Next FP8 on NVIDIA H200 NVL (TP2+EP2) (1.14 to 1.23x at decode-sized batches): [#38116](https://github.com/sgl-project/sglang/pull/38116)\r\n* [Kernel] Add tuned LFM2.5 Triton MoE configs on B300: [#37158](https://github.com/sgl-project/sglang/pull/37158)\r\n* [Kernel] Drop the vendored dense BF16 GEMM port in favor of FlashInfer 0.6.18: [#38124](https://github.com/sgl-project/sglang/pull/38124)\r\n* [Kernel] Add OOT dispatch for clamp position: [#38687](https://github.com/sgl-project/sglang/pull/38687)\r\n* [JIT] Port the expert-pack MXFP4 kernels to load_jit and fix their launch limits: [#38830](https://github.com/sgl-project/sglang/pull/38830)\r\n* feat(kernels): generalize persistent CuTe JIT cache: [#33911](https://github.com/sgl-project/sglang/pull/33911)\r\n* Add SGLANG_CRASH_ON_JIT_COMPILE to forbid on-the-fly JIT compilation: [#36615](https://github.com/sgl-project/sglang/pull/36615)\r\n\r\n## Configuration System\r\n\r\n* [Config] Round 6.1: \"unset\" gets its own spelling, and the declaration says what it means: [#38046](https://github.com/sgl-project/sglang/pull/38046)\r\n* [Config] Round 6.2: the field declarations move to their namespaces, and the record is assembled from them: [#38047](https://github.com/sgl-project/sglang/pull/38047)\r\n* [Config] Round 6.3: the record remembers how it was asked for, and is sealed while resolution runs: [#38048](https://github.com/sgl-project/sglang/pull/38048)\r\n* [Config] Round 6.4: the runtime reads the bags, not the record: [#38049](https://github.com/sgl-project/sglang/pull/38049)\r\n* [Config] Round 6.5: a namespace declares what it derives, next to what it derives it from: [#38113](https://github.com/sgl-project/sglang/pull/38113)\r\n* [Config] Retire get_global_server_args, and clear the deprecated flags that have a replacement: [#38375](https://github.com/sgl-project/sglang/pull/38375)\r\n* [Config] One writer for the declaration stash; no exception to the write seal: [#38752](https://github.com/sgl-project/sglang/pull/38752)\r\n* [Config] msgspec.Struct for the config tier: [#38753](https://github.com/sgl-project/sglang/pull/38753)\r\n* config: an out-of-tree replacement point for every resolution-pipeline step: [#39134](https://github.com/sgl-project/sglang/pull/39134)\r\n* Fix dataclasses.asdict on the msgspec ServerArgs: [#38958](https://github.com/sgl-project/sglang/pull/38958)\r\n\r\n## OpenAI-Compatible API\r\n\r\n* [Feat][Responses API] Support custom tools, encrypted reasoning replay, developer tier and model validation: [#38690](https://github.com/sgl-project/sglang/pull/38690)\r\n* [PD][OpenAI] Gate /v1/responses persistence behind --enable-response-store, default off: [#39122](https://github.com/sgl-project/sglang/pull/39122)\r\n* [OpenAI] Propagate PD routing metadata through /v1/responses: [#35503](https://github.com/sgl-project/sglang/pull/35503)\r\n* [Responses] Fix empty-prompt routing for token-first chat encoders (kimi_k3, inkling): [#35486](https://github.com/sgl-project/sglang/pull/35486)\r\n* [Feature] Add response-level input/output token ids to chat completions via SglExt: [#34488](https://github.com/sgl-project/sglang/pull/34488)\r\n* Support non-strict GLM47 tool calls with EBNF constraints: [#38890](https://github.com/sgl-project/sglang/pull/38890)\r\n* [Fix] Preserve GLM tool argument types across JSON Schema unions: [#39136](https://github.com/sgl-project/sglang/pull/39136)\r\n* Auto-detect GLM-5.3 chat templates as glm45/glm47 parsers: [#38297](https://github.com/sgl-project/sglang/pull/38297)\r\n* Force reasoning mode for GLM-5.3 chat templates: [#39227](https://github.com/sgl-project/sglang/pull/39227)\r\n* Add granite_thinking_parser reasoning parser for Granite 4.2: [#38693](https://github.com/sgl-project/sglang/pull/38693)\r\n* [Kimi K3] Rework skipped-think fix as opt-in force_nonempty_content with streaming coverage: [#34187](https://github.com/sgl-project/sglang/pull/34187)\r\n* [Kimi-K3] Recover the reply when the model skips the think channel: [#37743](https://github.com/sgl-project/sglang/pull/37743)\r\n* [Session] Fix session idle timeout after rejected requests: [#39035](https://github.com/sgl-project/sglang/pull/39035)\r\n* [Session] Work with PD and Fix empty continuations: [#39038](https://github.com/sgl-project/sglang/pull/39038)\r\n* [Session] Fix image append positions and parent metadata: [#39145](https://github.com/sgl-project/sglang/pull/39145)\r\n* [Session + MM] Fix text positions in session continuations: [#39144](https://github.com/sgl-project/sglang/pull/39144)\r\n* feat(agent sessions): attribute stored KV cache blocks to sessions: [#37482](https://github.com/sgl-project/sglang/pull/37482)\r\n* Fix /model_info serialization when a config value is a class: [#39237](https://github.com/sgl-project/sglang/pull/39237)\r\n* Fix: abort handling for dispatched requests after client disconnect: [#35255](https://github.com/sgl-project/sglang/pull/35255)\r\n\r\n## Rust Server\r\n\r\n* [Router] Add composable scoring and eligibility policies: [#37731](https://github.com/sgl-project/sglang/pull/37731)\r\n* [Router] Add load-aware prefill admission and bounded policy proposals: [#37843](https://github.com/sgl-project/sglang/pull/37843)\r\n* [Router] Add bucket-aware policy domains and native cache indexing: [#38108](https://github.com/sgl-project/sglang/pull/38108)\r\n* [Router] Publish cache-aware load state: [#38139](https://github.com/sgl-project/sglang/pull/38139)\r\n* [Router] Preserve global cache affinity with bucket routing (TTFT p50 -18.5% vs power-of-two bucket routing): [#38814](https://github.com/sgl-project/sglang/pull/38814)\r\n* [Router] Honor KV-event storage tiers in the cache-aware tree (1/4): [#39108](https://github.com/sgl-project/sglang/pull/39108)\r\n* [Router] Expose the KV storage-tier stream and tree occupancy on /metrics (2/4): [#39109](https://github.com/sgl-project/sglang/pull/39109)\r\n* [Router] Stream outcome observability for 2xx SSE streams: [#38737](https://github.com/sgl-project/sglang/pull/38737)\r\n* [PD] Add /v1/responses support to the HTTP PD router: [#36141](https://github.com/sgl-project/sglang/pull/36141)\r\n* [Rust] Gate health on startup warmup completion: [#37994](https://github.com/sgl-project/sglang/pull/37994)\r\n* [Rust] Use Dynamo native renderers when chat templates are missing: [#38939](https://github.com/sgl-project/sglang/pull/38939)\r\n* [Rust Server] Use node-local HTTP ports for DP attention: [#34430](https://github.com/sgl-project/sglang/pull/34430)\r\n* [Rust Server] fix p/d bootstrap across dp listeners: [#36234](https://github.com/sgl-project/sglang/pull/36234)\r\n* Rust server: align launcher and request validation behavior: [#37327](https://github.com/sgl-project/sglang/pull/37327)\r\n* Allow CUDA VMM feature transport with the Rust frontend: [#39347](https://github.com/sgl-project/sglang/pull/39347)\r\n* [gRPC] Stream engine state changes: [#39915](https://github.com/sgl-project/sglang/pull/39915)\r\n* [gRPC] Expose native pause status: [#37488](https://github.com/sgl-project/sglang/pull/37488)\r\n* [Fix] Seed raw tokenizer_path for smg-grpc-servicer in gRPC mode: [#39105](https://github.com/sgl-project/sglang/pull/39105)\r\n* [SGL Router] Prepare dynamo-render dependencies: [#39457](https://github.com/sgl-project/sglang/pull/39457)\r\n\r\n## Simulator\r\n\r\nA CPU-only inference simulator under `tools/sglang-simulator` that reuses the real scheduler and cache stack with a latency predictor in place of the model forward.\r\n\r\n* [Simulator] Add high-fidelity CPU-based inference simulator: [#33824](https://github.com/sgl-project/sglang/pull/33824)\r\n* [Simulator] Give the OFFLINE/BLOCKING comparison tolerances real headroom: [#38732](https://github.com/sgl-project/sglang/pull/38732)\r\n\r\n## SGLang-Diffusion\r\n\r\n* [Diffusion] Add MiniMax H3 cube sparse attention (denoise 19.3% faster with the aggressive schedule on 2x H100): [#34893](https://github.com/sgl-project/sglang/pull/34893)\r\n* [Diffusion][MiniMax-H3] Add SM90 Sage compute for SubBlock sparse attention (generation time -6.3 to -20.9% vs BF16 SubBlock): [#37982](https://github.com/sgl-project/sglang/pull/37982)\r\n* [Diffusion] Add SpargeAttention backend: [#37437](https://github.com/sgl-project/sglang/pull/37437)\r\n* [Diffusion] Add request-scoped Skip Softmax attention: [#37959](https://github.com/sgl-project/sglang/pull/37959)\r\n* [Diffusion] Pick the attention backend by measuring it: [#38689](https://github.com/sgl-project/sglang/pull/38689)\r\n* [Diffusion] Admit explicit attention backends by capability: [#37441](https://github.com/sgl-project/sglang/pull/37441)\r\n* [Diffusion] Add cumulative extra-high quality tier: [#37422](https://github.com/sgl-project/sglang/pull/37422)\r\n* [Diffusion] Support MiniMax-H3 Singularity hybrid checkpoints: [#38455](https://github.com/sgl-project/sglang/pull/38455)\r\n* [Diffusion] MiniMax-H3: tiered AdaLN plan cache (pinned-host tier + per-plan LRU): [#37266](https://github.com/sgl-project/sglang/pull/37266)\r\n* [Diffusion] MiniMax-H3: warm up at the served clip shape: [#37945](https://github.com/sgl-project/sglang/pull/37945)\r\n* [Diffusion] MiniMax-H3: Only use fused qk_norm on NV: [#33994](https://github.com/sgl-project/sglang/pull/33994)\r\n* [Diffusion] Fix MiniMax H3 WebUI inference settings: [#36320](https://github.com/sgl-project/sglang/pull/36320)\r\n* [Diffusion] fix: stabilize H3 reference audio across repeated requests: [#38225](https://github.com/sgl-project/sglang/pull/38225)\r\n* [Diffusion] Return Qwen-Image-Layered outputs and preserve CFG2 rounding (engine E2E 34.2 s on one H200 to 17.3 s on two, byte-identical layers): [#38549](https://github.com/sgl-project/sglang/pull/38549)\r\n* [Diffusion] Optimize Qwen-Image-Edit attention on Hopper: [#38584](https://github.com/sgl-project/sglang/pull/38584)\r\n* [Diffusion] Port the Wan VAE decoder fast paths to the Qwen-Image VAE: [#38020](https://github.com/sgl-project/sglang/pull/38020)\r\n* [Diffusion] Keep the Wan VAE decoder channels_last and add a Triton NHWC nearest upsample: [#38182](https://github.com/sgl-project/sglang/pull/38182)\r\n* [Diffusion] Fuse FLUX.2 NVFP4 FC1, SwiGLU, and FC2 quantization: [#37096](https://github.com/sgl-project/sglang/pull/37096)\r\n* [Diffusion] Enable lossless BCG for FLUX.1-dev: [#38591](https://github.com/sgl-project/sglang/pull/38591)\r\n* [Diffusion] Enable breakable CUDA graph for JoyEcho: [#38110](https://github.com/sgl-project/sglang/pull/38110)\r\n* [Diffusion] Optimize LTX-2 QKNorm and split RoPE on Hopper: [#38396](https://github.com/sgl-project/sglang/pull/38396)\r\n* [Diffusion] Preserve BF16 rounding in Hopper LTX QKNorm and RoPE fusion: [#38533](https://github.com/sgl-project/sglang/pull/38533)\r\n* [Diffusion] Support diffusion decoder parallel tiling for LTX-2.5: [#36026](https://github.com/sgl-project/sglang/pull/36026)\r\n* [Diffusion] Fuse LingBot per-token gated residual and RMSNorm modulate: [#37910](https://github.com/sgl-project/sglang/pull/37910)\r\n* [Diffusion] fuse LingBot MoE group-limited top-k index selection: [#38044](https://github.com/sgl-project/sglang/pull/38044)\r\n* [Diffusion] add Helios per-token gated-residual fusion (quality-gated): [#38042](https://github.com/sgl-project/sglang/pull/38042)\r\n* [Diffusion] Fuse LongCat Image normalization and modulation: [#38530](https://github.com/sgl-project/sglang/pull/38530)\r\n* [Diffusion] Optimize SANA-WM convolution post-processing and streaming GDN: [#38529](https://github.com/sgl-project/sglang/pull/38529)\r\n* Cosmos3 fp8 mixed precision: [#36380](https://github.com/sgl-project/sglang/pull/36380)\r\n* [Diffusion] Support mixed INT8 embeddings and Comfy NVFP4 encoders: [#38506](https://github.com/sgl-project/sglang/pull/38506)\r\n* [Diffusion] auto-keep video DiT resident on high-memory GPUs (SANA-WM end to end -38% on one B300): [#38001](https://github.com/sgl-project/sglang/pull/38001)\r\n* [Diffusion] Add explicit snapshot-offload component residency: [#38535](https://github.com/sgl-project/sglang/pull/38535)\r\n* [Diffusion] Stream mapped weights on a shared host/device pool (layerwise-offload request 478 to 302 s): [#38441](https://github.com/sgl-project/sglang/pull/38441)\r\n* [Diffusion] Read mapped layers directly when the host cannot cache them: [#39022](https://github.com/sgl-project/sglang/pull/39022)\r\n* [Diffusion] Measure warmup memory and layer usage per phase for residency calibration (1/4): [#37916](https://github.com/sgl-project/sglang/pull/37916)\r\n* [Diffusion] feat: spill large tensors over shared memory like numpy arrays: [#38656](https://github.com/sgl-project/sglang/pull/38656)\r\n* [Diffusion] Set an explicit x264 preset for video output: [#38657](https://github.com/sgl-project/sglang/pull/38657)\r\n* [Diffusion] Compose third-party component bundles safely: [#37816](https://github.com/sgl-project/sglang/pull/37816)\r\n* [Diffusion] Consolidate plain state-dict component loaders: [#38128](https://github.com/sgl-project/sglang/pull/38128)\r\n* [Diffusion] loader: reuse plain state-dict loading without per-model classes: [#38127](https://github.com/sgl-project/sglang/pull/38127)\r\n* [Diffusion] loader: filter duplicate precision variants across custom loaders: [#37616](https://github.com/sgl-project/sglang/pull/37616)\r\n* [Diffusion] Remove component loader capability switches: [#36824](https://github.com/sgl-project/sglang/pull/36824)\r\n* [Diffusion] Key the VAE decode-dtype store by module layout: [#38496](https://github.com/sgl-project/sglang/pull/38496)\r\n* [Diffusion] Serve requests that turn CFG off on a CFG-parallel server: [#39303](https://github.com/sgl-project/sglang/pull/39303)\r\n* [Diffusion] Make the SP sequence gather pass contiguous shards: [#39291](https://github.com/sgl-project/sglang/pull/39291)\r\n* [Diffusion] Don't route an unreadable checkpoint into the native fallback: [#39292](https://github.com/sgl-project/sglang/pull/39292)\r\n* [Diffusion] Recover IPC JIT initialization after interrupted builds: [#39034](https://github.com/sgl-project/sglang/pull/39034)\r\n* [Diffusion][Refactor] Refactor and unify RoPE execution for DiT models using RotaryEmbedding based on CustomOp: [#33555](https://github.com/sgl-project/sglang/pull/33555)\r\n* [Diffusion] feat: add maybe_record_function profiler spans for request phases: [#35922](https://github.com/sgl-project/sglang/pull/35922)\r\n* [Diffusion] Improve BCG warmup frame-count diagnostics for video models: [#37890](https://github.com/sgl-project/sglang/pull/37890)\r\n* [Diffusion] Fix #38166: Restore non-layer placeholders before releasing host copies: [#38171](https://github.com/sgl-project/sglang/pull/38171)\r\n* [Diffusion] fix: preserve mapped courier tensor lifetime: [#37965](https://github.com/sgl-project/sglang/pull/37965)\r\n* [Diffusion] fix host-resident vocab tables loaded on GPU: [#38012](https://github.com/sgl-project/sglang/pull/38012)\r\n* [Multimodal Gen] feat: support key masks on USPAttention's replicated-prefix path: [#36735](https://github.com/sgl-project/sglang/pull/36735)\r\n\r\n## Local & Desktop AI\r\n\r\n* [SM120] DeepSeek-V4: DeepGEMM paged-MQA indexer +FP4 MoE+ page-split (decode TPOT up to 3.4x faster and TTFT -20% on 4x RTX PRO 6000): [#29927](https://github.com/sgl-project/sglang/pull/29927)\r\n* [SM120] Use exact query-head widths for DeepSeek-V4 sparse MLA decode: [#36655](https://github.com/sgl-project/sglang/pull/36655)\r\n* [Kernel] Add KDA NVFP4 GEMM for Qwen3.x on SM120 (1.32x geometric-mean kernel speedup at decode shapes on RTX PRO 6000): [#36865](https://github.com/sgl-project/sglang/pull/36865)\r\n* [Kernel] Add KDA FP8 skinny GEMM for SM120: [#38082](https://github.com/sgl-project/sglang/pull/38082)\r\n* [FP8] SM120: route FP8 linear to per-tensor (cudnn/nvjet) instead of channelwise cutlass (Ministral-3-14B decode 46.9 to 86.6 tokens/s at batch 1): [#38006](https://github.com/sgl-project/sglang/pull/38006)\r\n* sm120 32GB mem-tier: raise decode cuda-graph max_bs 24->48 + chunked_prefill 2k->4k: [#37898](https://github.com/sgl-project/sglang/pull/37898)\r\n* [Qwen3.8] Enable NVIDIA NVFP4 on DGX Spark with file-backed PLE and PDL router fix: [#39126](https://github.com/sgl-project/sglang/pull/39126)\r\n* [Diffusion][minimax-h3] Add SM120 support for SubBlock sparse attention (1.22 to 1.85x lower latency on 8x RTX PRO 5000): [#37332](https://github.com/sgl-project/sglang/pull/37332)\r\n* [Diffusion] MiniMax-H3 VAE decoder: unfused w2 bias on SM12.x (cuBLAS 16x16 kernel mis-dispatch): [#37835](https://github.com/sgl-project/sglang/pull/37835)\r\n* [Diffusion] Pin layerwise host stores in place at their exact size: [#39021](https://github.com/sgl-project/sglang/pull/39021)\r\n* [Fix] Register triton.runtime.cache.triton_key in the MPS stub so torch.compile keeps working: [#37937](https://github.com/sgl-project/sglang/pull/37937)\r\n\r\n## AMD / ROCm\r\n\r\n* [ROCm] Stage large pageable H2D copies instead of pinning them in place (GLM-5.2 TP4 load on 4x MI355X from 505.7 s to 40.4 s): [#37720](https://github.com/sgl-project/sglang/pull/37720)\r\n* [AMD] Optimize Kimi-K3 Triton MLA prefill on gfx950 (1.73 to 2.09x faster prefill kernels; +16.7% geometric-mean throughput at 68K/350 TP8): [#35770](https://github.com/sgl-project/sglang/pull/35770)\r\n* [AMD] Perf Kimi-K3 MoE optimization: [#33838](https://github.com/sgl-project/sglang/pull/33838)\r\n* [AMD] Perf Kimi-K3 fuse ROCm KDA decode boundary: [#34198](https://github.com/sgl-project/sglang/pull/34198)\r\n* [AMD] [Kimi-K3] Fuse the KDA input projection into a single GEMM on ROCm: [#35176](https://github.com/sgl-project/sglang/pull/35176)\r\n* [AMD] support qlen>1 for aiter gluon path for Kimi K3: [#37601](https://github.com/sgl-project/sglang/pull/37601)\r\n* [AMD] Support aiter fa mha chunked kv for Kimi-K3: [#37691](https://github.com/sgl-project/sglang/pull/37691)\r\n* [AMD] Enable topk v2 GLM ROCm (+10.2% throughput per GPU and -9.9% median TPOT geometric mean on MI355X): [#36851](https://github.com/sgl-project/sglang/pull/36851)\r\n* [AMD] Enable Fast Triton Sparse MLA backend: [#30575](https://github.com/sgl-project/sglang/pull/30575)\r\n* [AMD][gfx95] Fill the chunked-prefill compute budget exactly (mean TTFT -5.4 to -9.5% on GLM-5.2-MXFP4 MI355X TP4): [#32888](https://github.com/sgl-project/sglang/pull/32888)\r\n* [ROCm] Take the fused DSA metadata kernels and drop redundant work from the absorb path: [#37124](https://github.com/sgl-project/sglang/pull/37124)\r\n* [ROCm] Make DSA indexer top-k exact with cooperative selection (about 3x faster top-k kernel): [#37591](https://github.com/sgl-project/sglang/pull/37591)\r\n* [AMD] Skip unused TOPK v2 plan kernel on ROCm: [#37580](https://github.com/sgl-project/sglang/pull/37580)\r\n* [AMD][GLM-5.2] Keep GlmMoeDsa MoE e_score_correction_bias in fp32: [#37133](https://github.com/sgl-project/sglang/pull/37133)\r\n* [AMD] Enable FP4 indexer for Deepseek V4 (+5.8% throughput per GPU, -5.5% median TTFT on MI355X): [#37353](https://github.com/sgl-project/sglang/pull/37353)\r\n* [AMD] Fix FP4 indexer OOR: [#37660](https://github.com/sgl-project/sglang/pull/37660)\r\n* [AMD][DSv4] Fuse the DSv4 FP4 indexer prefill-schedule preamble into one kernel (output throughput +15.3% at concurrency 4): [#37764](https://github.com/sgl-project/sglang/pull/37764)\r\n* [AMD][DSv4] Switch output projection gemm (oproj_a) to fp8: [#37423](https://github.com/sgl-project/sglang/pull/37423)\r\n* [AMD][DSv4] Fuse inverse-RoPE into the fp8 wo_a quant (stacked on #37423) (decode TPOT -14.3% vs bf16 at 128K/1K, concurrency 64): [#37658](https://github.com/sgl-project/sglang/pull/37658)\r\n* [AMD][DSV4] feat: enable fp8 two-pool unified_kv on gfx950 (about 1.5x more resident KV tokens): [#37413](https://github.com/sgl-project/sglang/pull/37413)\r\n* [AMD][DSV4] Reland unified-KV pool sizing and SWA ring accounting, fully gated (+83.6% full-attention KV tokens): [#38192](https://github.com/sgl-project/sglang/pull/38192)\r\n* [AMD][DSV4] Skip the paged SWA page return under the per-request ring: [#38571](https://github.com/sgl-project/sglang/pull/38571)\r\n* [AMD][gfx95] DSV4 wo_b (dp-attention): route to tuned bpreshuffle GEMM instead of triton: [#38227](https://github.com/sgl-project/sglang/pull/38227)\r\n* [AMD] Fix DSV4 unified attention sink TP slice: [#35092](https://github.com/sgl-project/sglang/pull/35092)\r\n* [AMD] Fix Dspark accept length and reduce host bubble on DSV4: [#39116](https://github.com/sgl-project/sglang/pull/39116)\r\n* [AMD] Fix DeepSeek block-FP8 loading on gfx94x: [#38446](https://github.com/sgl-project/sglang/pull/38446)\r\n* [AMD] gfx950 assembly attention for EAGLE verify, draft extend and decode (median TPOT -25.6% on Qwen3.5-397B MXFP4 TP4): [#37465](https://github.com/sgl-project/sglang/pull/37465)\r\n* [AMD] gfx950 assembly attention: length-aware split-KV for dynamic workload: [#39172](https://github.com/sgl-project/sglang/pull/39172)\r\n* [AMD] Parallelize aiter spec-decode KV index building over token blocks: [#37659](https://github.com/sgl-project/sglang/pull/37659)\r\n* [AMD] Quantize the bf16 MTP draft experts online to MXFP4 for Qwen3.5 (+13.6% throughput per GPU, -10.3% median TPOT): [#38748](https://github.com/sgl-project/sglang/pull/38748)\r\n* [AMD][Fix] Fix aiter bpreshuffle GEMM for output sizes it cannot dispatch for qwen3.5 mxfp-attn-fp8-v2 TP4: [#37564](https://github.com/sgl-project/sglang/pull/37564)\r\n* [AMD][DCP 1/N] add dcp support for aiter backend: [#34432](https://github.com/sgl-project/sglang/pull/34432)\r\n* [AMD] aiter: honor per-layer softmax scale: [#38754](https://github.com/sgl-project/sglang/pull/38754)\r\n* [AMD] aiter: fail loudly on cross-layer KV sharing in target_verify: [#38755](https://github.com/sgl-project/sglang/pull/38755)\r\n* [AMD] aiter: resolve SWA KV pool for draft workers + guard paged decode: [#38756](https://github.com/sgl-project/sglang/pull/38756)\r\n* [AMD] aiter: route head_dim>256 prefill through Triton unified_attention: [#38757](https://github.com/sgl-project/sglang/pull/38757)\r\n* [AMD] Allow aiter attention backend for Gemma-4 (about 17% more output throughput than Triton at concurrency 128): [#38758](https://github.com/sgl-project/sglang/pull/38758)\r\n* [AMD] Skip AITER FP8 ASM prefill when GQA is unsupported: [#38467](https://github.com/sgl-project/sglang/pull/38467)\r\n* [AMD] fix: use the hardware fp8 e4m3 convert on gfx950: [#37140](https://github.com/sgl-project/sglang/pull/37140)\r\n* [AMD] Fix weight checking for AITER-shuffled block FP8 weights: [#34330](https://github.com/sgl-project/sglang/pull/34330)\r\n* [AMD] Fix Quark load of MiniMax-M3 MXFP4 index_qkv_proj: [#37254](https://github.com/sgl-project/sglang/pull/37254)\r\n* [AMD] Restore SWA reprefill-tail on UnifiedRadixCache when HiCache is off: [#32759](https://github.com/sgl-project/sglang/pull/32759)\r\n* [AMD] Fix registered HiCache host pointer aliases: [#35233](https://github.com/sgl-project/sglang/pull/35233)\r\n* [ROCm] Raise HiCache JIT block quota for mapped-host throughput: [#39036](https://github.com/sgl-project/sglang/pull/39036)\r\n* [AMD] Fix EAGLE crash when no kv_index_translator is bound on the DSA fp8 read door: [#38318](https://github.com/sgl-project/sglang/pull/38318)\r\n* [AMD] CI: fix Lean decode crash on the EAGLE path: [#37119](https://github.com/sgl-project/sglang/pull/37119)\r\n* [MoE][ROCm] Admit the unified Triton router on ROCm, including single-group routing: [#38328](https://github.com/sgl-project/sglang/pull/38328)\r\n* [AMD][Diffusion] Migrate FlyDSL fused norm kernels to the v0.3.0 stable API: [#36349](https://github.com/sgl-project/sglang/pull/36349)\r\n* [AMD] Fix ROCm VAE Conv2D fast path breaking spatial-parallel decode: [#34424](https://github.com/sgl-project/sglang/pull/34424)\r\n* [AMD][Fix] Fix regression in jit build error with FLUX.2-dev on gfx1250: [#38329](https://github.com/sgl-project/sglang/pull/38329)\r\n* [AMD] Add gfx1151 (Strix Halo / Ryzen AI MAX+) Docker image: [#33939](https://github.com/sgl-project/sglang/pull/33939)\r\n* [AMD][CI] Publish ROCm 10 release images and kernel wheel: [#38763](https://github.com/sgl-project/sglang/pull/38763)\r\n\r\n## NPU / Ascend\r\n\r\n* [NPU] Add NPU arch35 support and enhance DSV4 processing in DeepSeek-V4: [#37373](https://github.com/sgl-project/sglang/pull/37373)\r\n* [NPU] Support DSV4 host memory cache management: [#37382](https://github.com/sgl-project/sglang/pull/37382)\r\n* [NPU] Add sparsity-driven KV offload for DeepSeek DSA on Ascend (decode throughput 1.6x): [#33089](https://github.com/sgl-project/sglang/pull/33089)\r\n* [NPU] Support GLM5.2 and FP8 DSA&Indexer kvcache for 950: [#38250](https://github.com/sgl-project/sglang/pull/38250)\r\n* [NPU] glm5.2 fp8 memory opt: [#38807](https://github.com/sgl-project/sglang/pull/38807)\r\n* [NPU] Adapt DFlash2 speculative decoding to Ascend NPUs: [#35629](https://github.com/sgl-project/sglang/pull/35629)\r\n* [NPU] Support DFlash speculative decoding for MiMo-V2.5-Pro (mxfp4): [#37565](https://github.com/sgl-project/sglang/pull/37565)\r\n* [NPU] Enable non-greedy MTP sampling: [#32495](https://github.com/sgl-project/sglang/pull/32495)\r\n* [NPU] Support EAGLE when PP enabled in prefill nodes: [#32207](https://github.com/sgl-project/sglang/pull/32207)\r\n* [NPU][HiCache] Optimize HiCache L2 IO with Memfabric acc_offload: [#38826](https://github.com/sgl-project/sglang/pull/38826)\r\n* [NPU][HiCache] Add Ascend Memcache Hicache L3 storage backend: [#38827](https://github.com/sgl-project/sglang/pull/38827)\r\n* [NPU] support mf device urma and host rdma trans type: [#38174](https://github.com/sgl-project/sglang/pull/38174)\r\n* [NPU] Strip padding before FIA kernel for vision encoder padded sequences: [#36329](https://github.com/sgl-project/sglang/pull/36329)\r\n* [NPU] Fix ViT graph key layout handling: [#37758](https://github.com/sgl-project/sglang/pull/37758)\r\n* [NPU] fix pp 2 hang on npu: [#38249](https://github.com/sgl-project/sglang/pull/38249)\r\n* [NPU] [Fix] Fix NPU MLA HiCache backup accessing missing data_ptrs.: [#36813](https://github.com/sgl-project/sglang/pull/36813)\r\n* [NPU] fix extend_seq_lens_cpu shape in eager mode: [#36843](https://github.com/sgl-project/sglang/pull/36843)\r\n* [NPU] Fix error due to missing parameter quant_linear passing: [#37384](https://github.com/sgl-project/sglang/pull/37384)\r\n* [NPU] Fix device mismatch in SWA mask for DSpark verify graph capture: [#39353](https://github.com/sgl-project/sglang/pull/39353)\r\n* [NPU][Fix] Disable pinned memory to fix DeepSeek-V2 DP-attention hang: [#39423](https://github.com/sgl-project/sglang/pull/39423)\r\n* [Feat] Add base NpuSRTPlatform implementation: [#36472](https://github.com/sgl-project/sglang/pull/36472)\r\n* [Diffusion] [NPU] Optimize LTX-2/2.3 inference performance for NPU (latency -7.6 to -10.9%): [#34722](https://github.com/sgl-project/sglang/pull/34722)\r\n\r\n## CPU / Intel / XPU\r\n\r\n* [CPU] Support FP8 KV cache: [#32733](https://github.com/sgl-project/sglang/pull/32733)\r\n* [CPU] Implement fused QK Norm and RoPE kernels: [#37748](https://github.com/sgl-project/sglang/pull/37748)\r\n* [CPU] Add native CPU kernel for MurmurHash32: [#35604](https://github.com/sgl-project/sglang/pull/35604)\r\n* [CPU] Support Qwen3.8 text+video: adding torchcodec, ffmpeg and removing pin_memory: [#35492](https://github.com/sgl-project/sglang/pull/35492)\r\n* [Diffusion][CPU] Enable MiniMax-H3 on Xeon CPU: [#35147](https://github.com/sgl-project/sglang/pull/35147)\r\n* [CPU] Fix shm allreduce collision and sglang-router import: [#37179](https://github.com/sgl-project/sglang/pull/37179)\r\n* [CPU] Fix native KV hash compilation in Xeon image: [#34767](https://github.com/sgl-project/sglang/pull/34767)\r\n* [CPU] Update base image to Ubuntu 26.04: [#35313](https://github.com/sgl-project/sglang/pull/35313)\r\n* XPU: Enable GLM5.1 (GlmMoeDsaForCausalLM) DSA Attention: [#24959](https://github.com/sgl-project/sglang/pull/24959)\r\n* [Intel GPU] Add MLA support to Intel XPU Attention backend for Prefill: [#35866](https://github.com/sgl-project/sglang/pull/35866)\r\n* Speculative Decoding support for intel_xpu attention backend on XPU target: [#30548](https://github.com/sgl-project/sglang/pull/30548)\r\n* DFLASH support added for XPU: [#32798](https://github.com/sgl-project/sglang/pull/32798)\r\n* [XPU] Support GPT-OSS MXFP4 checkpoints on Intel XPU: [#35751](https://github.com/sgl-project/sglang/pull/35751)\r\n* [Feature][Intel XPU] Add memory saver support for Intel XPU via upstream torch_memory_saver: [#29935](https://github.com/sgl-project/sglang/pull/29935)\r\n* [Intel][XPU] Add NUMA node binding support for Intel XPU: [#31113](https://github.com/sgl-project/sglang/pull/31113)\r\n* [XPU][Diffusion] Enable MiniMax H3 on XPU platforms: [#33366](https://github.com/sgl-project/sglang/pull/33366)\r\n* [XPU] Re-add intel xpu on triton paths in diffusion platforms: [#36654](https://github.com/sgl-project/sglang/pull/36654)\r\n* [XPU] Adapt device agnostic API usage: [#32093](https://github.com/sgl-project/sglang/pull/32093)\r\n* [XPU] Make checkpoint_engine worker device-agnostic: [#32382](https://github.com/sgl-project/sglang/pull/32382)\r\n* [XPU][Fix] Pack device-pointer tables as uint64 to avoid 64-bit address overflow: [#35051](https://github.com/sgl-project/sglang/pull/35051)\r\n* [XPU] Add xpu forward in Gemma3RMSNorm & Add test and benchmark for Gemma3RMSNorm: [#36278](https://github.com/sgl-project/sglang/pull/36278)\r\n* Fix/whisper xpu varlen encoder decoder: [#36298](https://github.com/sgl-project/sglang/pull/36298)\r\n* Xpu/weekly simple model enablement 2026 08 30: [#37193](https://github.com/sgl-project/sglang/pull/37193)\r\n* [Intel GPU] Align XPU toml file for rust support: [#31031](https://github.com/sgl-project/sglang/pull/31031)\r\n\r\n## MUSA / Moore Threads\r\n\r\nFirst installation path for Moore Threads GPUs: a MUSA Dockerfile, install guide, and runtime extras.\r\n\r\n* [MUSA] Add installation guide and Dockerfile: [#36709](https://github.com/sgl-project/sglang/pull/36709)\r\n\r\n## Security\r\n\r\n* [Rust] Bound multimodal media ingress: [#37967](https://github.com/sgl-project/sglang/pull/37967)\r\n* [Router] Raise chat body cap to 32 MB for multimodal payloads: [#38735](https://github.com/sgl-project/sglang/pull/38735)\r\n\r\n## Cookbook Updates\r\n\r\n* MiniCPM5-2B deployment guide: [link](https://docs.sglang.io/cookbook/autoregressive/OpenBMB/MiniCPM5-2B)\r\n* DeepSeek-V4: DGX Spark (2x GB10) Flash FP4 recipes, Flash Vision verified on B200 and GB300, Pro B200 FP4 agentic HiCache with DSpark: [#37479](https://github.com/sgl-project/sglang/pull/37479), [#37737](https://github.com/sgl-project/sglang/pull/37737), [#37468](https://github.com/sgl-project/sglang/pull/37468), [#37492](https://github.com/sgl-project/sglang/pull/37492), [#38026](https://github.com/sgl-project/sglang/pull/38026)\r\n* DeepSeek-V4 on MI355X: agentic-workload arguments, DSpark config, and PD disaggregation recipes for all three strategies: [#38677](https://github.com/sgl-project/sglang/pull/38677), [#39252](https://github.com/sgl-project/sglang/pull/39252), [#39396](https://github.com/sgl-project/sglang/pull/39396)\r\n* GLM-5.3-Flash: FP8 speed data on B300, H100, and B200, NVFP4 rows, MTP with the flashinfer_trtllm runner on Blackwell, and a breakable prefill CUDA graph opt-in: [#37412](https://github.com/sgl-project/sglang/pull/37412), [#37576](https://github.com/sgl-project/sglang/pull/37576), [#39213](https://github.com/sgl-project/sglang/pull/39213), [#38522](https://github.com/sgl-project/sglang/pull/38522)\r\n* GLM-5.3: DFlash2 speculative option: [#37392](https://github.com/sgl-project/sglang/pull/37392)\r\n* GLM-5.2: NVFP4 on B200/B300 for AgentX with HiCache, and MI355X MXFP4 on the Triton DSA backend with TP8/EP1 and TP4/EP4 recipes: [#35368](https://github.com/sgl-project/sglang/pull/35368), [#39106](https://github.com/sgl-project/sglang/pull/39106), [#39230](https://github.com/sgl-project/sglang/pull/39230), [#39406](https://github.com/sgl-project/sglang/pull/39406)\r\n* Kimi-K3: measured B300 1x8 numbers, DCP under HiCache L1+L2 with DSpark, and measured MI350X/MI355X cells: [#37878](https://github.com/sgl-project/sglang/pull/37878), [#39190](https://github.com/sgl-project/sglang/pull/39190), [#39029](https://github.com/sgl-project/sglang/pull/39029), [#37781](https://github.com/sgl-project/sglang/pull/37781)\r\n* Qwen3.8-Flash-Next NVFP4 on DGX Spark (1x, 2x) and RTX PRO 6000, and the NVIDIA NVFP4 export for Qwen3.8-27B: [#37995](https://github.com/sgl-project/sglang/pull/37995), [#38611](https://github.com/sgl-project/sglang/pull/38611)\r\n* Qwen3.5: FlashInfer GDN on B200, FP8 on B200/B300 with trtllm-gen MoE and symmetric memory, and MI355X MXFP4 with the AttnFP8-V2 checkpoint and HiCache: [#37360](https://github.com/sgl-project/sglang/pull/37360), [#38374](https://github.com/sgl-project/sglang/pull/38374), [#39104](https://github.com/sgl-project/sglang/pull/39104), [#39358](https://github.com/sgl-project/sglang/pull/39358)\r\n* MiniMax-H3: GB300 and GB200 recipes, DGX Spark, a measured RTX 5090 desktop recipe, and VDN-H3 numbers on B200: [#38296](https://github.com/sgl-project/sglang/pull/38296), [#37456](https://github.com/sgl-project/sglang/pull/37456), [#39373](https://github.com/sgl-project/sglang/pull/39373), [#39244](https://github.com/sgl-project/sglang/pull/39244)\r\n* JoyEcho: measured H200 residency and BCG recipe: [#38534](https://github.com/sgl-project/sglang/pull/38534)\r\n* Diffusion performance guide: per-model tuning decision table and single-GPU B300 notes: [#38148](https://github.com/sgl-project/sglang/pull/38148), [#37891](https://github.com/sgl-project/sglang/pull/37891)\r\n\r\n## Dependencies\r\n\r\n* [Deps] Retire the CUDA 12 lane: [#38404](https://github.com/sgl-project/sglang/pull/38404)\r\n* chore: bump sglang-kernel version to 0.4.7: [#39346](https://github.com/sgl-project/sglang/pull/39346)\r\n* chore: bump sgl-kernel version to 0.4.7: [#39324](https://github.com/sgl-project/sglang/pull/39324)\r\n* bumping sgl-deep-gemm to 0.2.0: [#39371](https://github.com/sgl-project/sglang/pull/39371)\r\n* [Deps] Raise smg-grpc-servicer floor to >=0.9.0 to unbreak SMG E2E CI: [#38801](https://github.com/sgl-project/sglang/pull/38801)\r\n* [CI] Install sgl-eval from PyPI through the test extra: [#37504](https://github.com/sgl-project/sglang/pull/37504)\r\n* Add missing test dependencies to pyproject.toml variants: [#36098](https://github.com/sgl-project/sglang/pull/36098)\r\n* [DeepSeek-V4.1] Bump FlashMLA to the fork's rebase head (v4.1 kernels): [#39171](https://github.com/sgl-project/sglang/pull/39171)\r\n* [AMD] Update ROCm AITER pin to 4ad9983: [#37784](https://github.com/sgl-project/sglang/pull/37784)\r\n* [AMD] [Docker] Update MoRI to v1.2.3: [#38411](https://github.com/sgl-project/sglang/pull/38411)\r\n* [AMD][CI] Retire the ROCm 7.0 kernel wheel: [#38767](https://github.com/sgl-project/sglang/pull/38767)\r\n* [NPU] Update sgl-kernel-npu version to 2026.9.0 and move memfabric deps into pyproject: [#37399](https://github.com/sgl-project/sglang/pull/37399)\r\n* [NPU] Bump memfabric and sgl-kernel-npu versions in docs and pyproject_npu.toml: [#38437](https://github.com/sgl-project/sglang/pull/38437)\r\n* [XPU] install xpu-kernel by released wheel: [#37394](https://github.com/sgl-project/sglang/pull/37394)\r\n* [XPU] Add Regular Docker Image Release workflow for Intel XPU: [#37340](https://github.com/sgl-project/sglang/pull/37340)\r\n* docker(xpu): unblock nightly build (setvars.sh + sgl-kernel rename): [#38617](https://github.com/sgl-project/sglang/pull/38617)\r\n* pyproject(xpu): drop human-eval git dep to unblock image build: [#38824](https://github.com/sgl-project/sglang/pull/38824)\r\n\r\n## Breaking Changes & Upgrade Notes\r\n\r\n* **The CUDA 12 lane is retired.** No `-cu129` / `-cu12` wheels or images are published from this release on; already-published tags are unaffected, and v0.5.19 is the last release with a CUDA 12 lane: [#38404](https://github.com/sgl-project/sglang/pull/38404)\r\n* **Prefill context parallelism v1 is removed.** The strategy-based CP path is canonical; the `SGLANG_ENABLE_CP_V2` gate, the v1 runtime, and the generic v1 CLI options are gone, and CP APIs are renamed to versionless names: [#36223](https://github.com/sgl-project/sglang/pull/36223), [#36228](https://github.com/sgl-project/sglang/pull/36228), [#36229](https://github.com/sgl-project/sglang/pull/36229). `--enable-prefill-cp` is rejected on HIP, NPU, and MUSA until their CP refactor lands; decode CP and non-CP inference are unaffected: [#38293](https://github.com/sgl-project/sglang/pull/38293)\r\n* **`/v1/responses` storage is off by default.** Start with `--enable-response-store` to keep retrieval, `previous_response_id` chaining, and background requests; PD deployments cannot enable it and reject those features at admission. `background=true` with `store=false` is rejected: [#39122](https://github.com/sgl-project/sglang/pull/39122)\r\n* **Sampling-mask capacity moves to `--sampling-mask-max-tokens`** (default 4096). `SGLANG_DISAGGREGATION_SAMPLING_MASK_MAX_TOKENS` is rejected at startup, and masks under PD require `SGLANG_ENABLE_DISAGG_SAMPLING_MASK=1` on both prefill and decode servers: [#36631](https://github.com/sgl-project/sglang/pull/36631)\r\n* **The `cutlass_mla` attention backend, the non-Marlin `gptq` quantization choice, the AWQ AOT dequantize kernel, and Dual Chunk Flash Attention are deleted.** GPTQ checkpoints continue through `gptq_marlin`; `auto-round` and `moe_wna16` lose their non-Marlin GPU fallback: [#32114](https://github.com/sgl-project/sglang/pull/32114)\r\n* **Seventeen deprecated CLI flags are removed and `get_global_server_args` now raises**; read resolved values from the namespace bags or the raw input from `get_server_args()`: [#38375](https://github.com/sgl-project/sglang/pull/38375). The config tier, including `ServerArgs`, is now `msgspec.Struct`, so `dataclasses.asdict` and dataclass introspection on it no longer work: [#38753](https://github.com/sgl-project/sglang/pull/38753), [#38958](https://github.com/sgl-project/sglang/pull/38958)\r\n* **`--dcp-comm-backend` now resolves per platform for every model**: `fi_a2a` on Blackwell when the DCP group sits inside one node or MNNVL domain, otherwise `a2a` on CUDA and ROCm; pass `--dcp-comm-backend ag_rs` to restore the old default: [#37767](https://github.com/sgl-project/sglang/pull/37767), [#39165](https://github.com/sgl-project/sglang/pull/39165)\r\n* **SM120 FP8 linear layers with per-channel weight scales take the per-tensor route** (cudnn/nvjet) instead of channelwise CUTLASS: [#38006](https://github.com/sgl-project/sglang/pull/38006). On 32 GB GPUs the decode CUDA-graph cap rises from 24 to 48 (160 at TP >= 4) and chunked prefill from 2K to 4K: [#37898](https://github.com/sgl-project/sglang/pull/37898)\r\n* **An undersized SWA pool now fails at startup** with a `ValueError` instead of hanging at warmup: [#37610](https://github.com/sgl-project/sglang/pull/37610)\r\n* **gRPC clients: `GetIsReady` is replaced by the `WatchEngineState` stream**, which publishes health, pause state, instance ID, and discovery snapshots: [#39915](https://github.com/sgl-project/sglang/pull/39915)\r\n* **Diffusion defaults change**: video DiTs stay resident on GPUs with enough free memory unless `--dit-cpu-offload` is passed explicitly ([#38001](https://github.com/sgl-project/sglang/pull/38001)); the SDPA attention backend is chosen by measurement, retiring the no-op `allow_cudnn_sdp` ([#38689](https://github.com/sgl-project/sglang/pull/38689)); component loader capability switches are replaced by resolver contracts that fail closed ([#36824](https://github.com/sgl-project/sglang/pull/36824))\r\n* **ROCm 10 is the default AMD lane.** ROCm 7.0 CI, nightly images, and the `rocm700` kernel wheel are retired; ROCm 10 release images ship for MI30x and MI35x with a `+rocm1000` kernel wheel, and ROCm 7.2.x images remain: [#38659](https://github.com/sgl-project/sglang/pull/38659), [#38632](https://github.com/sgl-project/sglang/pull/38632), [#38767](https://github.com/sgl-project/sglang/pull/38767), [#38763](https://github.com/sgl-project/sglang/pull/38763)\r\n* **The CPU image moves to Ubuntu 26.04** with GCC 15.2 as the default compiler: [#35313](https://github.com/sgl-project/sglang/pull/35313)\r\n* **Contributors: Python formatting moves from black to ruff-format**, with a `.git-blame-ignore-revs` entry for the reformat commit: [#37210](https://github.com/sgl-project/sglang/pull/37210), [#37695](https://github.com/sgl-project/sglang/pull/37695)\r\n\r\n## New Contributors\r\n* @EricKing626 made their first contribution in https://github.com/sgl-project/sglang/pull/36851\r\n* @hekhong-png made their first contribution in https://github.com/sgl-project/sglang/pull/34362\r\n* @Ankur-singh made their first contribution in https://github.com/sgl-project/sglang/pull/37360\r\n* @hn1209 made their first contribution in https://github.com/sgl-project/sglang/pull/32218\r\n* @akashpalla made their first contribution in https://github.com/sgl-project/sglang/pull/36220\r\n* @Viol2000 made their first contribution in https://github.com/sgl-project/sglang/pull/37300\r\n* @wes-lyu made their first contribution in https://github.com/sgl-project/sglang/pull/35443\r\n* @akhauriyash made their first contribution in https://github.com/sgl-project/sglang/pull/37655\r\n* @dujifeng made their first contribution in https://github.com/sgl-project/sglang/pull/37616\r\n* @tristan-me made their first contribution in https://github.com/sgl-project/sglang/pull/37332\r\n* @Bo-Vincent made their first contribution in https://github.com/sgl-project/sglang/pull/37731\r\n* @hickeyma made their first contribution in https://github.com/sgl-project/sglang/pull/37788\r\n* @kediwu0331 made their first contribution in https://github.com/sgl-project/sglang/pull/36380\r\n* @littlefatfat made their first contribution in https://github.com/sgl-project/sglang/pull/33824\r\n* @heachary made their first contribution in https://github.com/sgl-project/sglang/pull/37764\r\n* @mottopanikeiku made their first contribution in https://github.com/sgl-project/sglang/pull/32172\r\n* @Grypse made their first contribution in https://github.com/sgl-project/sglang/pull/37965\r\n* @amykchang made their first contribution in https://github.com/sgl-project/sglang/pull/34488\r\n* @DUXin-ece made their first contribution in https://github.com/sgl-project/sglang/pull/37971\r\n* @Beihao-Zhou made their first contribution in https://github.com/sgl-project/sglang/pull/32405\r\n* @zqlcode made their first contribution in https://github.com/sgl-project/sglang/pull/32151\r\n* @yuttian1 made their first contribution in https://github.com/sgl-project/sglang/pull/30315\r\n* @yu-depend made their first contribution in https://github.com/sgl-project/sglang/pull/38171\r\n* @sghhhh made their first contribution in https://github.com/sgl-project/sglang/pull/37199\r\n* @JohnQinAMD made their first contribution in https://github.com/sgl-project/sglang/pull/37720\r\n* @samnordmann made their first contribution in https://github.com/sgl-project/sglang/pull/30430\r\n* @KMS07 made their first contribution in https://github.com/sgl-project/sglang/pull/36654\r\n* @lixiufei-leo made their first contribution in https://github.com/sgl-project/sglang/pull/38227\r\n* @pinkgom made their first contribution in https://github.com/sgl-project/sglang/pull/38116\r\n* @jeffreywang88 made their first contribution in https://github.com/sgl-project/sglang/pull/31415\r\n* @nogumanov made their first contribution in https://github.com/sgl-project/sglang/pull/37143\r\n* @kedarpotdar-nv made their first contribution in https://github.com/sgl-project/sglang/pull/38374\r\n* @EdwardXuy made their first contribution in https://github.com/sgl-project/sglang/pull/38125\r\n* @cl-vv-h made their first contribution in https://github.com/sgl-project/sglang/pull/32495\r\n* @AnuSajikumar6264 made their first contribution in https://github.com/sgl-project/sglang/pull/30345\r\n* @sagearc made their first contribution in https://github.com/sgl-project/sglang/pull/36234\r\n* @davidli1515 made their first contribution in https://github.com/sgl-project/sglang/pull/38399\r\n* @vorapolsiloai made their first contribution in https://github.com/sgl-project/sglang/pull/38467\r\n* @123liuziming made their first contribution in https://github.com/sgl-project/sglang/pull/37789\r\n* @nzbej made their first contribution in https://github.com/sgl-project/sglang/pull/33089\r\n* @jianuo-huang made their first contribution in https://github.com/sgl-project/sglang/pull/36899\r\n* @LinPoly made their first contribution in https://github.com/sgl-project/sglang/pull/36655\r\n* @rystewart-nvidia made their first contribution in https://github.com/sgl-project/sglang/pull/35599\r\n* @hhy-seven made their first contribution in https://github.com/sgl-project/sglang/pull/35486\r\n* @vpremseth23 made their first contribution in https://github.com/sgl-project/sglang/pull/38754\r\n* @MeowMiaoJ made their first contribution in https://github.com/sgl-project/sglang/pull/36278\r\n* @rishabhsinha17 made their first contribution in https://github.com/sgl-project/sglang/pull/34977\r\n* @yousafshah made their first contribution in https://github.com/sgl-project/sglang/pull/38693\r\n* @Xinyu-Kang made their first contribution in https://github.com/sgl-project/sglang/pull/34330\r\n* @simple-sun made their first contribution in https://github.com/sgl-project/sglang/pull/37818\r\n* @tnguyeng made their first contribution in https://github.com/sgl-project/sglang/pull/37254\r\n* @knlb-tml made their first contribution in https://github.com/sgl-project/sglang/pull/38908\r\n* @desmond-intel made their first contribution in https://github.com/sgl-project/sglang/pull/34556\r\n* @jlqibm made their first contribution in https://github.com/sgl-project/sglang/pull/37679\r\n* @littledgg made their first contribution in https://github.com/sgl-project/sglang/pull/38486\r\n* @Hritik003 made their first contribution in https://github.com/sgl-project/sglang/pull/37015\r\n* @200lz made their first contribution in https://github.com/sgl-project/sglang/pull/36141\r\n* @cx22757 made their first contribution in https://github.com/sgl-project/sglang/pull/37382\r\n* @ichbinblau made their first contribution in https://github.com/sgl-project/sglang/pull/39396\r\n* @CharlesXu-HQ made their first contribution in https://github.com/sgl-project/sglang/pull/38097"
+ },
+ {
+ "repo": "ml-explore/mlx-lm",
+ "tag_name": "v0.31.3",
+ "name": "v0.31.3",
+ "published_at": "2026-04-22T07:43:57Z",
+ "html_url": "https://github.com/ml-explore/mlx-lm/releases/tag/v0.31.3",
+ "prerelease": false,
+ "body": "## Highlights\r\n* Lots of bugfixes\r\n* Thread local generation stream to accompany MLX v0.31.2\r\n\r\n## What's Changed\r\n* Bump the patch version by @angeloskath in https://github.com/ml-explore/mlx-lm/pull/1124\r\n* Fix batch dimension mismatch in BatchKVCache and BatchRotatingKVCache extend() by @razorback16 in https://github.com/ml-explore/mlx-lm/pull/1141\r\n* Fix parallel tool call handling in server by @kernelpool in https://github.com/ml-explore/mlx-lm/pull/1170\r\n* Fix MiniMax M2 parallel tool calling by @kernelpool in https://github.com/ml-explore/mlx-lm/pull/1171\r\n* Fix missing tree_reduce import in models/cache.py by @siiea-ai in https://github.com/ml-explore/mlx-lm/pull/1165\r\n* Apertus tie_word_embeddings fix by @BlackSamorez in https://github.com/ml-explore/mlx-lm/pull/1143\r\n* Fix batch dimension mismatch in ArraysCache extend() by @techtoboggan in https://github.com/ml-explore/mlx-lm/pull/1169\r\n* Fix dwq: check for actual safetensors in target_dir by @micuentadecasa in https://github.com/ml-explore/mlx-lm/pull/1173\r\n* fix: handle NoneType check for think tokens in TokenizerWrapper by @yuetyeelo2855 in https://github.com/ml-explore/mlx-lm/pull/1167\r\n* Fix Gemma4 tool parser: support hyphenated function names and braces in string args by @AkashKhamkar in https://github.com/ml-explore/mlx-lm/pull/1150\r\n* Fix empty tool_call_end breaking Mistral tool calls by @eyupcanakman in https://github.com/ml-explore/mlx-lm/pull/1151\r\n* Fix ArraysCache extend by @angeloskath in https://github.com/ml-explore/mlx-lm/pull/1177\r\n* Fix Gemma 4 KV-shared layers creating unused projections by @glyphVault in https://github.com/ml-explore/mlx-lm/pull/1158\r\n* Thread local generation stream by @angeloskath in https://github.com/ml-explore/mlx-lm/pull/1090\r\n\r\n## New Contributors\r\n* @razorback16 made their first contribution in https://github.com/ml-explore/mlx-lm/pull/1141\r\n* @siiea-ai made their first contribution in https://github.com/ml-explore/mlx-lm/pull/1165\r\n* @BlackSamorez made their first contribution in https://github.com/ml-explore/mlx-lm/pull/1143\r\n* @techtoboggan made their first contribution in https://github.com/ml-explore/mlx-lm/pull/1169\r\n* @micuentadecasa made their first contribution in https://github.com/ml-explore/mlx-lm/pull/1173\r\n* @yuetyeelo2855 made their first contribution in https://github.com/ml-explore/mlx-lm/pull/1167\r\n* @AkashKhamkar made their first contribution in https://github.com/ml-explore/mlx-lm/pull/1150\r\n* @glyphVault made their first contribution in https://github.com/ml-explore/mlx-lm/pull/1158\r\n\r\n**Full Changelog**: https://github.com/ml-explore/mlx-lm/compare/v0.31.2...v0.31.3"
+ },
+ {
+ "repo": "jundot/omlx",
+ "tag_name": "v0.6.4",
+ "name": "0.6.4",
+ "published_at": "2026-08-29T17:05:15Z",
+ "html_url": "https://github.com/jundot/omlx/releases/tag/v0.6.4",
+ "prerelease": false,
+ "body": "# oMLX 0.6.4\n\noMLX 0.6.4 substantially improves Qwen3.8-Flash-Next prefill and generation performance, while fixing continuous batching, prefix-cache reconstruction, Lightning MTP state handling, GLM-5.3 correctness, and several model-loading and settings issues.\n\n## Qwen3.8-Flash-Next Performance\n\n- **Added exact QSA prefill and decode acceleration.** Eligible batch-one text workloads now use native FP32 QSA scoring, deterministic block selection, direct sparse-GQA Metal attention, and a selected-K/V decode path. Unsupported layouts and execution modes continue through the official fallback path.\n- **Accelerated resident PLE, GDN, and hyperconnection projections** without replacing the checkpoint state used by fallback paths.\n- **Added warm-prefix restoration for Lightning MTP prompt history.** Matching prefix-cache entries can restore the MTP sidecar instead of replaying the full reusable prompt head.\n- **Fixed Qwen vision grid compatibility** with newer MLX releases.\n\nBy @jonathan308 in [#3244](https://github.com/jundot/omlx/pull/3244).\n\nMaintainer benchmark on an Apple M3 Ultra with 512 GB of unified memory, using `Qwen3.8-Flash-Next-oQ4e-mtp`, Code (Python) context, 128 generated tokens, temperature 1.0, top-p 1.0, and Lightning MTP with adaptive maximum depth 3. Each length was measured once with zero cached prompt tokens.\n\n| Context | PP before | PP after | Change | TG before | TG after | Change | Total before | Total after | Change |\n|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|\n| 4K | 999.61 tok/s | 1061.34 tok/s | +6.2% | 48.65 tok/s | 53.59 tok/s | +10.2% | 6.73s | 6.25s | -7.1% |\n| 16K | 950.06 tok/s | 1061.31 tok/s | +11.7% | 48.74 tok/s | 55.70 tok/s | +14.3% | 19.87s | 17.74s | -10.7% |\n| 32K | 834.31 tok/s | 1113.63 tok/s | +33.5% | 40.08 tok/s | 45.92 tok/s | +14.6% | 42.47s | 32.21s | -24.2% |\n\nAt 32K context, prompt processing improved by 33.5%, generation throughput improved by 14.6%, and total request time decreased by 24.2%.\n\n## Bug Fixes and Operational Improvements\n\n- **Isolated DFlash and Lightning MTP patches across loaded engines.** Loading one engine no longer mutates shared model classes in a way that breaks resident or subsequently loaded engines, preventing repetition loops and Qwen4 Lightning MTP hidden-state shape failures. By @e-Evolution in [#3285](https://github.com/jundot/omlx/pull/3285).\n\n- **Fixed GLM-5.3 affine prefill correctness for strided inputs.** Split projection views are now made contiguous before entering native quantized tiles, preventing GLM-5.3-Flash from losing track of its prompt at longer input lengths. By @davidtorcivia in [#3275](https://github.com/jundot/omlx/pull/3275).\n\n- **Fixed restored-prefix mRoPE state on Qwen3.8 ANE and NAX paths.** The restored text-position seed is materialized before the prefill graph consumes it, preventing Metal timeouts, immediate EOS, stale tool-content continuation, and run-to-run output variance. By @hesara in [#3279](https://github.com/jundot/omlx/pull/3279).\n\n- **Prevented Metal resource-count exhaustion during long decode.** ArraysCache hybrid models such as GLM-5.3 now periodically materialize recurrent state, release boundary snapshots and paged-cache blocks on failures, and avoid retaining unnecessary hot-cache payloads. By @Pietroski in [#3227](https://github.com/jundot/omlx/pull/3227).\n\n- **Fixed Qwen3.8-Flash-Next late-join continuous batching.** Model-owned caches now convert to their batch representation correctly, ragged QSA positions remain aligned, and indexer arrays are trimmed with the rest of the cache. This removes join-time crashes and subsequent cache-recovery cycles. By @daveycodez in [#3246](https://github.com/jundot/omlx/pull/3246).\n\n- **Fixed loading models symlinked from the Hugging Face cache.** Qwen4 and MiniMax sanitization now recognizes symlinked checkpoint shards without accidentally matching unrelated safetensors files. By @ChildOTK in [#3224](https://github.com/jundot/omlx/pull/3224).\n\n- **Fixed GLM-5.3 mixed text and image conversations.** Registering `glm5_next` with mlx-vlm's prompt configuration prevents image placeholders from being dropped when an image or screenshot appears alongside plain conversation turns or tool results. By @BTankut in [#3278](https://github.com/jundot/omlx/pull/3278).\n\n- **Restored Qwen4 PLE state after rejected Lightning MTP drafts.** N-gram history and short-convolution state now roll back to the committed prefix together with QSA and GDN state, with fail-closed validation around incomplete snapshots. By @hojin12312 in [#3232](https://github.com/jundot/omlx/pull/3232).\n\n- **Enabled RAM-safe Qwen4 oQe calibration.** Calibration now initializes the Qwen4 compatibility runtime, preserves SSD-mapped PLE behavior, uses the intended lenient VLM loading contract, and accounts for the tensors actually resident during calibration. By @actualrealhuman in [#3269](https://github.com/jundot/omlx/pull/3269).\n\n- **Reduced Qwen4 long-prefill memory growth.** QSA boolean masks now use tiled attention instead of dense score slabs, and QSA boundary snapshots store incremental slices instead of complete prefixes. On the reported 64 GB system, the clean-prefill ceiling increased from approximately 43K to 96K-100K tokens. By @alytaphoenix in [#3283](https://github.com/jundot/omlx/pull/3283).\n\n- **Fixed mixed text and mRoPE prefix-cache reconstruction.** One-channel text positions are promoted to the three-channel representation when joined with multimodal cache blocks, avoiding silent reconstruction failure and unnecessary full re-prefill. By @rsnow in [#3219](https://github.com/jundot/omlx/pull/3219).\n\n- **Allowed TurboQuant KV cache and Lightning MTP to be enabled together in the macOS app.** The stale client-side conflict check is removed while the actual DFlash and VLM incompatibility checks remain intact. By @popfido in [#3254](https://github.com/jundot/omlx/pull/3254).\n\n- **Fixed model-setting persistence through the admin API.** `mtp_num_draft_tokens`, `preserve_thinking`, and `turboquant_skip_last` are no longer silently dropped. Lightning MTP draft depth is validated against the supported range, and unknown settings now fail explicitly. By @yanzhaohui1999 in [#3280](https://github.com/jundot/omlx/pull/3280).\n\n- **Added hybrid-cache boundary diagnostics.** Runtime cache observability now reports boundary capture, SSD fallback, restore, fail-closed store-skip, and prefix re-prefill reasons without exposing prompt content. By @MDX-Tom in [#3249](https://github.com/jundot/omlx/pull/3249).\n\n## Contributors\n\nThanks to @jonathan308, @e-Evolution, @davidtorcivia, @hesara, @Pietroski, @daveycodez, @ChildOTK, @BTankut, @hojin12312, @actualrealhuman, @alytaphoenix, @rsnow, @MDX-Tom, @popfido, and @yanzhaohui1999 for their contributions to oMLX 0.6.4.\n\n**Full Changelog**: https://github.com/jundot/omlx/compare/v0.6.3...v0.6.4\n"
+ },
+ {
+ "repo": "exo-explore/exo",
+ "tag_name": "v1.0.71",
+ "name": "v1.0.71",
+ "published_at": "2026-04-23T15:04:10Z",
+ "html_url": "https://github.com/exo-explore/exo/releases/tag/v1.0.71",
+ "prerelease": false,
+ "body": "# EXO v1.0.71 Release Notes\r\n\r\nThis is a small patch release, with better defaults for sampling and bug fixes for M5 series Macs and RDMA, along with support for the new Kimi K2.6.\r\n\r\n## Models\r\n- Add Kimi K2.6, including support for multimodality ([#1952](https://github.com/exo-explore/exo/pull/1952))\r\n- Add model cards for Qwen3.6 27B, GLM 5.1 and Kimi K2.6 quantisations ([#1970](https://github.com/exo-explore/exo/pull/1970))\r\n\r\n\r\n## Performance\r\n- Add recommended sampling defaults (from model providers) to improve model output quality, including min_p and top_k ([#1947](https://github.com/exo-explore/exo/pull/1947))\r\n- Prevent JACCL all_sum from corrupting output ([#1952](https://github.com/exo-explore/exo/pull/1952))\r\n\r\n## Quality of Life\r\n- Add a Pi integration tab! ([#1925](https://github.com/exo-explore/exo/pull/1925))\r\n- Handle multiple tool calls correctly in the Claude and Responses APIs ([#1952](https://github.com/exo-explore/exo/pull/1952))\r\n- Add a friendly context message for bug reporting ([#1959](https://github.com/exo-explore/exo/pull/1959))\r\n\r\n## Bug Fixes\r\n- Fix vision models on M5 series Macs by fixing addmm on NAX ([#1927](https://github.com/exo-explore/exo/pull/1927))\r\n- Fix prefix cache for Qwen3.6 ([#1952](https://github.com/exo-explore/exo/pull/1952))\r\n- Prevent complete downloads from losing their Completed status ([#1956](https://github.com/exo-explore/exo/pull/1956), [#1958](https://github.com/exo-explore/exo/pull/1958))\r\n- Populate the image cache from events to prevent crashes ([#1961](https://github.com/exo-explore/exo/pull/1961))\r\n- Fix DeepSeekV32 parsing and tensor parallelism for Gemma 4 models with effective parameters ([#1967](https://github.com/exo-explore/exo/pull/1967))\r\n\r\n## New Contributors\r\nAnd a thank you to [@nhwaani](https://github.com/nhwaani) who made their first contribution here!\r\n\r\n**Full Changelog**: [https://github.com/exo-explore/exo/compare/v1.0.70...v1.0.71](https://github.com/exo-explore/exo/compare/v1.0.70...v1.0.71)"
+ },
+ {
+ "repo": "waybarrios/vllm-mlx",
+ "tag_name": "v0.5.0",
+ "name": "v0.5.0",
+ "published_at": "2026-09-17T21:48:07Z",
+ "html_url": "https://github.com/waybarrios/vllm-mlx/releases/tag/v0.5.0",
+ "prerelease": false,
+ "body": "vllm-mlx 0.5.0 adds reusable conversation caching, broader model and parser support, and more reliable streaming and resource management for Apple Silicon.\n\n## Highlights\n\n- **SimpleEngine prefix-trie caching:** reuse growing conversation prefixes with configurable entry and memory limits. This release also fixes a completion race so a normally finished stream stores its cache entry reliably.\n- **DeepSeek-V4-Flash support:** native prompt encoding, reasoning extraction, and DSML tool calls, including streamed and multi-turn tool use.\n- **MLLM assistant drafting:** support configured assistant drafters with continuous batching, plus MTP routing and mlx-lm compatibility fixes.\n- **More reliable API responses:** preserve assistant text alongside tool calls, retain replayed reasoning, forward reasoning effort, and normalize Anthropic system messages consistently.\n- **Operational controls:** configure embedding input limits and overflow handling, override registry memory budgets, and report registry engine state and MLLM cache memory more accurately.\n\n## Reliability improvements\n\n- Keep model loading and generation on the owning thread in SimpleEngine, BatchedEngine, and VLM text routes.\n- Improve prefix-cache ownership, snapshot isolation, non-trimmable cache reuse, and rotating-cache guards.\n- Apply `--max-kv-size` limits and honor `--prefill-step-size` with continuous batching.\n- Preserve streamed tokens when `--stream-interval` is greater than one; avoid leaking split reasoning markers and isolate parser state between requests.\n- Add Step3p5 parsing and recognize additional Llama 3.1/3.3 tool-call formats.\n- Clear generator-owned vision caches correctly, release idle model memory, and close delegated streams and SSD caches during shutdown.\n\n## Upgrade notes\n\n- Runtime inference requires Apple Silicon and Python 3.10 or later.\n- Structured-output enforcement now rejects requests when the requested constraints cannot be enforced, instead of silently allowing unconstrained output. The package includes `llguidance>=1.7.6`.\n- Prefix-trie caching is opt-in with `--prefix-trie-cache`. Use `--prefix-trie-cache-size` and `--prefix-trie-cache-memory-mb` to control its capacity.\n- Embedding operators can set `--embedding-max-length` and choose `--embedding-overflow-policy truncate` or `error`.\n\n## Install\n\n```bash\npython -m pip install --upgrade 'vllm-mlx==0.5.0'\n```\n\nOptional Harmony support:\n\n```bash\npython -m pip install --upgrade 'vllm-mlx[harmony]==0.5.0'\n```\n\n## What's Changed\n* fix: guard non-trimmable rotating prefix caches by @Thump604 in https://github.com/waybarrios/vllm-mlx/pull/680\n* fix(server): isolate streaming parser state by @Thump604 in https://github.com/waybarrios/vllm-mlx/pull/654\n* fix(scheduler): make the prefix cache usable on non-trimmable KV caches by @janhilgard in https://github.com/waybarrios/vllm-mlx/pull/683\n* test(cache): cover real scheduler replay parity by @Thump604 in https://github.com/waybarrios/vllm-mlx/pull/702\n* fix(server): emit terminal finish_reason chunk when parsers swallow the finished output by @funkymonkeymonk in https://github.com/waybarrios/vllm-mlx/pull/673\n* feat(embeddings): add operator-configurable max length ceiling and overflow policy by @brandy975 in https://github.com/waybarrios/vllm-mlx/pull/695\n* Link the public documentation site from main by @waybarrios in https://github.com/waybarrios/vllm-mlx/pull/715\n* fix: pin SimpleEngine model load and generation to one thread by @janhilgard in https://github.com/waybarrios/vllm-mlx/pull/679\n* fix(batched): load the model on the thread that steps it by @janhilgard in https://github.com/waybarrios/vllm-mlx/pull/684\n* Add SimpleEngine prefix trie cache by @Thump604 in https://github.com/waybarrios/vllm-mlx/pull/574\n* fix(llama-tool-parser): recognize Llama 3.1+ / 3.3 tool-call formats by @CBribiescas in https://github.com/waybarrios/vllm-mlx/pull/580\n* docs: fix broken star history chart in READMEs by @FaintFlower in https://github.com/waybarrios/vllm-mlx/pull/716\n* Keep VLM TextModel generation on owner thread by @Thump604 in https://github.com/waybarrios/vllm-mlx/pull/543\n* Reduce EngineCore idle polling by @Thump604 in https://github.com/waybarrios/vllm-mlx/pull/552\n* fix(step): add Step3p5 parser support by @Thump604 in https://github.com/waybarrios/vllm-mlx/pull/598\n* fix(engine): close SSD cache tier on stop paths by @djacobsmeyer in https://github.com/waybarrios/vllm-mlx/pull/634\n* feat(registry): add CLI override for memory budget by @lunarlinger in https://github.com/waybarrios/vllm-mlx/pull/640\n* Fix fail-open structured-output enforcement by @Thump604 in https://github.com/waybarrios/vllm-mlx/pull/636\n* fix(registry): honor auto-unload-idle-seconds and release MLX memory on unload by @brandy975 in https://github.com/waybarrios/vllm-mlx/pull/638\n* fix: detach stored prefix-cache entries from live batch state by @rohnach29 in https://github.com/waybarrios/vllm-mlx/pull/642\n* fix(mllm): harden prefix cache ownership and rewind safety by @Thump604 in https://github.com/waybarrios/vllm-mlx/pull/691\n* fix(mtp): validate Qwen MTP artifact conversion by @Thump604 in https://github.com/waybarrios/vllm-mlx/pull/661\n* feat(mtp): support assistant drafters with MLLM batching by @Thump604 in https://github.com/waybarrios/vllm-mlx/pull/670\n* fix(engine): defer unused MLLM TextModel for draft routes by @Thump604 in https://github.com/waybarrios/vllm-mlx/pull/650\n* Restore paged-cache bulk release API by @Thump604 in https://github.com/waybarrios/vllm-mlx/pull/693\n* fix(engine): stop asyncio.shield() from logging expected cancellation exceptions as unhandled by @brandy975 in https://github.com/waybarrios/vllm-mlx/pull/643\n* fix: honor disabled MTP in VLM text routing by @Thump604 in https://github.com/waybarrios/vllm-mlx/pull/665\n* fix(cache): probe system prefix from normalized messages so agent tool turns cache by @iotashan in https://github.com/waybarrios/vllm-mlx/pull/714\n* fix: enable configured MLLM assistant drafters by default by @Thump604 in https://github.com/waybarrios/vllm-mlx/pull/669\n* fix(reasoning): stop leaking split think tags and rescanning output by @janhilgard in https://github.com/waybarrios/vllm-mlx/pull/677\n* feat: add DeepSeek-V4-Flash support by @janhilgard in https://github.com/waybarrios/vllm-mlx/pull/676\n* fix(scheduler): make --max-kv-size actually bound the KV cache by @janhilgard in https://github.com/waybarrios/vllm-mlx/pull/688\n* fix(simple-engine): close delegated streams by @Thump604 in https://github.com/waybarrios/vllm-mlx/pull/687\n* Fix streaming token loss at --stream-interval>1, DSA CacheList SSD spill, bf16 upcast on mlx 0.31.x, and forced-<think> reasoning by @anisoptera in https://github.com/waybarrios/vllm-mlx/pull/740\n* chore: remove dead _extract_media_from_messages helper by @djacobsmeyer in https://github.com/waybarrios/vllm-mlx/pull/751\n* test(simple-engine): key stream ownership by Thread, not ident by @janhilgard in https://github.com/waybarrios/vllm-mlx/pull/734\n* fix(qwen4-exp): keep text inference on mlx-vlm path by @Thump604 in https://github.com/waybarrios/vllm-mlx/pull/741\n* fix(scheduler): guard MTP install against mlx-lm >= 0.31 BatchGenerator by @azamamirza in https://github.com/waybarrios/vllm-mlx/pull/737\n* Fix integration test opt-in by @Thump604 in https://github.com/waybarrios/vllm-mlx/pull/692\n* fix(cli): honor --prefill-step-size with continuous batching by @Thump604 in https://github.com/waybarrios/vllm-mlx/pull/729\n* Report MLLM prefix caches in registry memory budgets by @waybarrios in https://github.com/waybarrios/vllm-mlx/pull/713\n* fix(registry): populate /metrics engine-state gauges in registry mode by @djacobsmeyer in https://github.com/waybarrios/vllm-mlx/pull/739\n* fix(anthropic): canonicalize system prompts with the shared stripper by @janhilgard in https://github.com/waybarrios/vllm-mlx/pull/767\n* Preserve assistant reasoning when chat history is replayed by @Thump604 in https://github.com/waybarrios/vllm-mlx/pull/771\n* Forward chat reasoning effort to the template by @Thump604 in https://github.com/waybarrios/vllm-mlx/pull/772\n* fix(anthropic): merge non-leading system messages by @Marveliu in https://github.com/waybarrios/vllm-mlx/pull/646\n* fix(server): keep assistant text when a delta also carries tool calls by @janhilgard in https://github.com/waybarrios/vllm-mlx/pull/690\n* fix(mllm): route vision-cache clears through batch_generator.vision_cache by @djacobsmeyer in https://github.com/waybarrios/vllm-mlx/pull/779\n* chore(server): remove dead request/stream helper functions by @djacobsmeyer in https://github.com/waybarrios/vllm-mlx/pull/789\n* Prepare v0.5.0 with reliable prefix cache completion by @waybarrios in https://github.com/waybarrios/vllm-mlx/pull/806\n\n## New Contributors\n* @funkymonkeymonk made their first contribution in https://github.com/waybarrios/vllm-mlx/pull/673\n* @FaintFlower made their first contribution in https://github.com/waybarrios/vllm-mlx/pull/716\n* @djacobsmeyer made their first contribution in https://github.com/waybarrios/vllm-mlx/pull/634\n* @lunarlinger made their first contribution in https://github.com/waybarrios/vllm-mlx/pull/640\n* @rohnach29 made their first contribution in https://github.com/waybarrios/vllm-mlx/pull/642\n* @iotashan made their first contribution in https://github.com/waybarrios/vllm-mlx/pull/714\n* @anisoptera made their first contribution in https://github.com/waybarrios/vllm-mlx/pull/740\n* @azamamirza made their first contribution in https://github.com/waybarrios/vllm-mlx/pull/737\n* @Marveliu made their first contribution in https://github.com/waybarrios/vllm-mlx/pull/646\n\n**Full Changelog**: https://github.com/waybarrios/vllm-mlx/compare/v0.4.1...v0.5.0"
+ },
+ {
+ "repo": "lemonade-sdk/lemonade",
+ "tag_name": "v11.9.0",
+ "name": "v11.9.0",
+ "published_at": "2026-09-02T18:33:53Z",
+ "html_url": "https://github.com/lemonade-sdk/lemonade/releases/tag/v11.9.0",
+ "prerelease": false,
+ "body": "## Headline\n\n- This quality release fixes two top bugs: `allowed_origins` no longer blocks remote use, and the fixed 120s prefill timeout is now a 600s default that is configurable via `global_timeout`.\n- A new experimental `llamacpp-hrx` backend runs AMD's HRX runtime in early access on AMD gfx1100/gfx1151 GPUs on Linux.\n- The pinned llama.cpp backend version was updated to add `Qwen3-Next` support.\n- Models placed in `chat/`, `embeddings/`, and `reranking/` subfolders of `extra_models_dir` are now classified by type and listed individually.\n\n## Breaking Changes\n\n- `extra_models_dir` models in reserved folders (`chat/`, `embeddings/`, `reranking/`) are now listed individually per file instead of as a single `extra.<directory>` model; legacy `extra.chat`/`extra.embeddings`/`extra.reranking` IDs still resolve as hidden aliases to the first file alphabetically.\n- The `LEMONADE_ALLOWED_ORIGINS` environment variable is deprecated in favor of the `allowed_origins` config/CLI option and auto-migrates at startup with a warning; migrate to the new option, as the environment variable is only supported temporarily.\n\n## Lemonade Server\n\n| Operating System | Downloads |\n|------------------|-----------|\n| **Windows** | [lemonade.msi](https://github.com/lemonade-sdk/lemonade/releases/download/v11.9.0/lemonade.msi) |\n| **Ubuntu 24.04+** | [Launchpad PPA](https://launchpad.net/~lemonade-team/+archive/ubuntu/stable) |\n| **Debian 13 (x86_64)** | [lemonade-server_11.9.0-debian13_amd64.deb](https://github.com/lemonade-sdk/lemonade/releases/download/v11.9.0/lemonade-server_11.9.0-debian13_amd64.deb) |\n| **Debian 13 (ARM64)** | [lemonade-server_11.9.0-debian13_arm64.deb](https://github.com/lemonade-sdk/lemonade/releases/download/v11.9.0/lemonade-server_11.9.0-debian13_arm64.deb) |\n| **Fedora 43 (x86_64)** | [lemonade-server-11.9.0-fc43.x86_64.rpm](https://github.com/lemonade-sdk/lemonade/releases/download/v11.9.0/lemonade-server-11.9.0-fc43.x86_64.rpm) |\n| **Fedora 43 (ARM64)** | [lemonade-server-11.9.0-fc43.aarch64.rpm](https://github.com/lemonade-sdk/lemonade/releases/download/v11.9.0/lemonade-server-11.9.0-fc43.aarch64.rpm) |\n| **Fedora 44 (x86_64)** | [lemonade-server-11.9.0-fc44.x86_64.rpm](https://github.com/lemonade-sdk/lemonade/releases/download/v11.9.0/lemonade-server-11.9.0-fc44.x86_64.rpm) |\n| **Fedora 44 (ARM64)** | [lemonade-server-11.9.0-fc44.aarch64.rpm](https://github.com/lemonade-sdk/lemonade/releases/download/v11.9.0/lemonade-server-11.9.0-fc44.aarch64.rpm) |\n| **macOS** | [Lemonade-11.9.0-Darwin.pkg](https://github.com/lemonade-sdk/lemonade/releases/download/v11.9.0/Lemonade-11.9.0-Darwin.pkg) |\n\n> **Other platforms?** See our [Installation Options](https://lemonade-server.ai/docs/guide/install/) for [Docker](https://lemonade-server.ai/docs/guide/install/docker/), [Snap](https://lemonade-server.ai/docs/guide/install/ubuntu/#__tabbed_2_3), [Arch](https://lemonade-server.ai/docs/guide/install/arch/), [Debian](https://lemonade-server.ai/docs/guide/install/), and more.\n\n## Embeddable Lemonade\n\nPortable binaries for bundling into your own installer. Run `lemond ./` as a subprocess.\n\n| Platform | Download |\n|----------|----------|\n| **Ubuntu x64** | [lemonade-embeddable-11.9.0-ubuntu-x64.tar.gz](https://github.com/lemonade-sdk/lemonade/releases/download/v11.9.0/lemonade-embeddable-11.9.0-ubuntu-x64.tar.gz) |\n| **Ubuntu arm64** | [lemonade-embeddable-11.9.0-ubuntu-arm64.tar.gz](https://github.com/lemonade-sdk/lemonade/releases/download/v11.9.0/lemonade-embeddable-11.9.0-ubuntu-arm64.tar.gz) |\n| **Windows x64** | [lemonade-embeddable-11.9.0-windows-x64.zip](https://github.com/lemonade-sdk/lemonade/releases/download/v11.9.0/lemonade-embeddable-11.9.0-windows-x64.zip) |\n| **macOS arm64** | [lemonade-embeddable-11.9.0-macos-arm64.tar.gz](https://github.com/lemonade-sdk/lemonade/releases/download/v11.9.0/lemonade-embeddable-11.9.0-macos-arm64.tar.gz) |\n\n---\n\n## What's Changed\n\nThanks @AaronStGeorge, @Bekhouche, @GabrielReusRodriguez, @Geramy, @abn, @anditherobot, @bitgamma, @fl0rianr, @github-actions, @jeremyfowers, @kenvandine, @superm1, @vaibhav8a, @zaneni6 for your awesome contributions to this release!\n\n<details>\n<summary>Click to expand changelog</summary>\n\n* Update FLM version to 1.0.3 by @zaneni6 in https://github.com/lemonade-sdk/lemonade/pull/3381\n* Add checkpoints to bench output by @bitgamma in https://github.com/lemonade-sdk/lemonade/pull/3359\n* feat(logging): add log rotation and transition default server posture to stdout/stderr by @abn in https://github.com/lemonade-sdk/lemonade/pull/2864\n* server: fix MTP draft companion handling by @fl0rianr in https://github.com/lemonade-sdk/lemonade/pull/3368\n* docs: fix typo maintainance -> maintenance by @vaibhav8a in https://github.com/lemonade-sdk/lemonade/pull/3383\n* fix: recover cache and models lost by 11.8.0 FHS split (#3387) by @superm1 in https://github.com/lemonade-sdk/lemonade/pull/3393\n* ci(sdcpp): run the CPU validation leg only on schedule/workflow_dispatch by @jeremyfowers in https://github.com/lemonade-sdk/lemonade/pull/3406\n* test(gpu-hang): scale request timeouts to the model-operation budget by @jeremyfowers in https://github.com/lemonade-sdk/lemonade/pull/3366\n* feat(backends): hrx backend [experimental] by @AaronStGeorge in https://github.com/lemonade-sdk/lemonade/pull/3374\n* fix(routing): llm classifier no longer treats has_tools as risk by @Bekhouche in https://github.com/lemonade-sdk/lemonade/pull/3330\n* fix(test): pre-download the model in test_gpu_hang_retry before the timed load by @jeremyfowers in https://github.com/lemonade-sdk/lemonade/pull/3408\n* fix(server): classify extra models using reserved directories: chat, embeddings, reranking instead of treating all model as chat by @anditherobot in https://github.com/lemonade-sdk/lemonade/pull/3333\n* fix(server): join the directory watcher before ModelManager members die by @Geramy in https://github.com/lemonade-sdk/lemonade/pull/3453\n* Update llama.cpp to b10723 by @github-actions[bot] in https://github.com/lemonade-sdk/lemonade/pull/3452\n* Fix(#3336) improve validate llamacpp.py by @GabrielReusRodriguez in https://github.com/lemonade-sdk/lemonade/pull/3379\n* fix(ci): resolve the newest llama.cpp nightly tag instead of /releases/latest by @jeremyfowers in https://github.com/lemonade-sdk/lemonade/pull/3450\n* ci(sdcpp): raise lemond global_timeout to 3600s for validation legs by @jeremyfowers in https://github.com/lemonade-sdk/lemonade/pull/3404\n* fix(server): log unauthorized origin rejections by @abn in https://github.com/lemonade-sdk/lemonade/pull/3418\n* ci(distro): retry transient mirror and HF fetch failures by @jeremyfowers in https://github.com/lemonade-sdk/lemonade/pull/3365\n* chore: Version bump to 11.9 by @kenvandine in https://github.com/lemonade-sdk/lemonade/pull/3454\n* fix(server): allow same-origin HTTP requests matching request Host by @abn in https://github.com/lemonade-sdk/lemonade/pull/3419\n* feat(server): add allowed_origins to global configuration schema by @abn in https://github.com/lemonade-sdk/lemonade/pull/3420\n\n</details>\n\n## New Contributors\n* @vaibhav8a made their first contribution in https://github.com/lemonade-sdk/lemonade/pull/3383\n* @AaronStGeorge made their first contribution in https://github.com/lemonade-sdk/lemonade/pull/3374\n\n**Full Changelog**: https://github.com/lemonade-sdk/lemonade/compare/v11.8.1...v11.9.0\n\n---\n\n> Windows installers are signed. Free code signing provided by [SignPath.io](https://signpath.io), certificate by [SignPath Foundation](https://signpath.org). See our [Code Signing Policy](https://github.com/lemonade-sdk/lemonade#code-signing-policy).\n"
+ },
+ {
+ "repo": "EricLBuehler/mistral.rs",
+ "tag_name": "v0.9.3",
+ "name": "v0.9.3",
+ "published_at": "2026-09-07T22:46:13Z",
+ "html_url": "https://github.com/EricLBuehler/mistral.rs/releases/tag/v0.9.3",
+ "prerelease": false,
+ "body": "## What's Changed\r\n* feat(core): concurrency fixes and scheduler improvements by @EricLBuehler in https://github.com/EricLBuehler/mistral.rs/pull/2389\r\n* perf: first-class FP8 and concurrent DFlash decode by @EricLBuehler in https://github.com/EricLBuehler/mistral.rs/pull/2390\r\n* feat(metrics): add mistralrs_sequences_capacity gauge by @EricLBuehler in https://github.com/EricLBuehler/mistral.rs/pull/2391\r\n* perf(core): optimize exact stochastic DFlash serving by @EricLBuehler in https://github.com/EricLBuehler/mistral.rs/pull/2392\r\n* feat(server): align server error codes to openai standard by @EricLBuehler in https://github.com/EricLBuehler/mistral.rs/pull/2393\r\n* feat(logging): expose separate prefill and decode t/s logging by @EricLBuehler in https://github.com/EricLBuehler/mistral.rs/pull/2395\r\n* fix(multimodal): keep Qwen media work out of decode by @EricLBuehler in https://github.com/EricLBuehler/mistral.rs/pull/2396\r\n* fix(ci): green the build for rust 1.98 and metal by @EricLBuehler in https://github.com/EricLBuehler/mistral.rs/pull/2397\r\n* feat(core): report cached prompt tokens in usage by @EricLBuehler in https://github.com/EricLBuehler/mistral.rs/pull/2398\r\n* perf(core): improve hybrid FP8 serving by @EricLBuehler in https://github.com/EricLBuehler/mistral.rs/pull/2399\r\n* feat(server): improve Anthropic Messages and Claude Code support by @matt-minev in https://github.com/EricLBuehler/mistral.rs/pull/2400\r\n* docs: refresh generated reference artifacts by @matt-minev in https://github.com/EricLBuehler/mistral.rs/pull/2401\r\n* fix(website): fix handling for clipboard by @EricLBuehler in https://github.com/EricLBuehler/mistral.rs/pull/2403\r\n* perf(cuda): scale FP8 and DFlash2 serving through C128 by @EricLBuehler in https://github.com/EricLBuehler/mistral.rs/pull/2404\r\n* feat(website): add details about v0.9.3 release by @EricLBuehler in https://github.com/EricLBuehler/mistral.rs/pull/2405\r\n* chore(cuda): clippy by @EricLBuehler in https://github.com/EricLBuehler/mistral.rs/pull/2406\r\n* feat(release): improve reliability, add caching by @EricLBuehler in https://github.com/EricLBuehler/mistral.rs/pull/2408\r\n* fix(core): align tool chat template JSON rendering by @atirna in https://github.com/EricLBuehler/mistral.rs/pull/2371\r\n* fix(lfm2): support legacy full_attn_idxs config field by @tugot17 in https://github.com/EricLBuehler/mistral.rs/pull/2305\r\n* fix(webui): send chat images as data URLs (fix HTTP 500 on image send) by @subin9 in https://github.com/EricLBuehler/mistral.rs/pull/2338\r\n* feat(core): bump cutile to v0.3.0 by @EricLBuehler in https://github.com/EricLBuehler/mistral.rs/pull/2410\r\n* feat(core): fp8 tensor-core gemv/gemm, cutile gdn prefill by @EricLBuehler in https://github.com/EricLBuehler/mistral.rs/pull/2412\r\n* feat(cutile): add fp8 fused moe and a load-time kernel autotuner by @EricLBuehler in https://github.com/EricLBuehler/mistral.rs/pull/2413\r\n* fix(cuda): restore GB10 builds with cudaforge 0.1.6 by @sizzlecar in https://github.com/EricLBuehler/mistral.rs/pull/2415\r\n* feat(fp8): support ModelOpt and compressed-tensors checkpoints with cutile kernels by @EricLBuehler in https://github.com/EricLBuehler/mistral.rs/pull/2416\r\n* feat(quant): add NVFP4 checkpoint loading and cuTile Blackwell acceleration by @EricLBuehler in https://github.com/EricLBuehler/mistral.rs/pull/2418\r\n\r\n## New Contributors\r\n* @atirna made their first contribution in https://github.com/EricLBuehler/mistral.rs/pull/2371\r\n* @tugot17 made their first contribution in https://github.com/EricLBuehler/mistral.rs/pull/2305\r\n* @subin9 made their first contribution in https://github.com/EricLBuehler/mistral.rs/pull/2338\r\n* @sizzlecar made their first contribution in https://github.com/EricLBuehler/mistral.rs/pull/2415\r\n\r\n**Full Changelog**: https://github.com/EricLBuehler/mistral.rs/compare/v0.9.2...v0.9.3"
+ },
+ {
+ "repo": "qualcomm/GenieX",
+ "tag_name": "v0.7.0",
+ "name": "v0.7.0",
+ "published_at": "2026-09-18T18:59:34Z",
+ "html_url": "https://github.com/qualcomm/GenieX/releases/tag/v0.7.0",
+ "prerelease": false,
+ "body": "## Hexagon HTP\n\nMicrosoft-signed HTP catalog (llama.cpp @ `4ff829`). No cert import required on Windows on Snapdragon.\n\n## What's Changed\n* fix(sdk): use resolved device for VLM encoder by @RemiliaForever in https://github.com/qualcomm/GenieX/pull/1437\n* fix(sdk): patch llama.cpp OpenCL add kernel for Adreno by @RemiliaForever in https://github.com/qualcomm/GenieX/pull/1438\n* ci: add Breeze grading workflow by @RemiliaForever in https://github.com/qualcomm/GenieX/pull/1424\n* feat(qairt): enable HTP multicore for QAIRT bundles by @mansiverma897993 in https://github.com/qualcomm/GenieX/pull/1313\n* feat(sdk): load QAIRT system prompt from metadata by @RemiliaForever in https://github.com/qualcomm/GenieX/pull/1443\n* feat(sdk): expose the QAIRT runtime path on the C API by @mengshengwu in https://github.com/qualcomm/GenieX/pull/1441\n* feat(sdk): add VLM compute device override by @RemiliaForever in https://github.com/qualcomm/GenieX/pull/1446\n* fix(server): rebuild full VLM context after a failed generation by @RemiliaForever in https://github.com/qualcomm/GenieX/pull/1442\n* chore(sdk): bump geniex-qairt to merged gemma4 VLM pipeline fix by @Davidqian123 in https://github.com/qualcomm/GenieX/pull/1448\n* fix(sdk): keep bench output when generation exhausts the context by @zhycheng614 in https://github.com/qualcomm/GenieX/pull/1451\n* fix(server): decode base64 audio input by @mansiverma897993 in https://github.com/qualcomm/GenieX/pull/1458\n* fix(go): retry chunk download on any error, not just timeout/EOF by @mengshengwu in https://github.com/qualcomm/GenieX/pull/1459\n* fix(sdk): apply context params to VLM encoder by @RemiliaForever in https://github.com/qualcomm/GenieX/pull/1460\n* feat(qairt): native stop sequences in the geniex-qairt pipeline by @mansiverma897993 in https://github.com/qualcomm/GenieX/pull/1342\n* fix(sdk): defer geniex-bench top-p to bundle/plugin default when unset by @vinovo in https://github.com/qualcomm/GenieX/pull/1461\n* fix(cli): print download cancel hint before progress by @mengshengwu in https://github.com/qualcomm/GenieX/pull/1468\n* fix(qairt): surface QNN backend logs through --log/GENIEX_LOG by @mengshengwu in https://github.com/qualcomm/GenieX/pull/1470\n* feat(sdk): add Qwen3.5-4B to geniex-bench model catalog by @mengshengwu in https://github.com/qualcomm/GenieX/pull/1469\n* feat: unify HTP power-mode control across qairt and llama_cpp by @mengshengwu in https://github.com/qualcomm/GenieX/pull/1447\n* fix(sdk): name supported formats when a repo has no loadable model files by @Hahahaooovvv in https://github.com/qualcomm/GenieX/pull/1467\n\n\n**Full Changelog**: https://github.com/qualcomm/GenieX/compare/v0.6.1...v0.7.0"
+ },
+ {
+ "repo": "mudler/LocalAI",
+ "tag_name": "v4.10.0",
+ "name": "v4.10.0",
+ "published_at": "2026-09-17T19:46:38Z",
+ "html_url": "https://github.com/mudler/LocalAI/releases/tag/v4.10.0",
+ "prerelease": false,
+ "body": "# \ud83c\udf89 LocalAI 4.10.0 Release! \ud83d\ude80\r\n\r\n<h1 align=\"center\">\r\n <br>\r\n <img height=\"300\" src=\"https://raw.githubusercontent.com/mudler/LocalAI/refs/heads/master/core/http/static/logo.png\">\r\n <br>\r\n <br>\r\n</h1>\r\n\r\nLocalAI 4.10.0 is out!\r\n\r\nTwenty-eight days and 280 pull requests. The work landed on three fronts: operating a fleet, feeding it models from private sources, and fixing the backends you depend on. A fleet operations dashboard replaces the flat node list with cluster-wide health, capacity, running models, and bulk lifecycle actions. A credentials file lets one `credentials.yaml` authenticate OCI registries, galleries and direct downloads without scattering tokens across environment variables. And `local-ai benchmark` measures end-to-end latency and throughput from the CLI.\r\n\r\nThe ds4 backend had four fixes that together make it reliable on CUDA: the kernels now build for the target architecture, abandoned inference is cancelled, generation boundaries are enforced, and prefilled reasoning is separated from content. Distributed mode gained seven stabilisation fixes across staging, liveness, virtual models and the control-plane database. Four CVEs were patched, `enable_thinking=false` is now honoured across every text backend, and Apple M5 no longer SIGSEGVs at startup.\r\n\r\n**Highlights:**\r\n\r\n- \ud83d\udcca **Fleet operations dashboard** - a real-time cluster overview replacing the flat node list. Aggregate health bands, capacity gauges for VRAM/RAM/CPU/disk, a running-models tab showing replica placement across workers, bulk drain/resume/remove with bounded concurrency, and a slide-out node inspector. Workers now report CPU telemetry at registration and heartbeat.\r\n- \ud83d\udd11 **Credentials file for private sources** - one `credentials.yaml` authenticates OCI registries, galleries, HTTP downloads and `github:` URIs. Supports basic auth, bearer tokens and custom headers, with secrets read at use time so rotated K8s secret mounts take effect without restart, and full masking in logs and error messages.\r\n- \ud83d\udccf **`local-ai benchmark` CLI command** - measures end-to-end latency and throughput against configured text models. Reports min/mean/max latency and completion tokens/sec across warmup and measured runs, with table and JSON output, redirect blocking, and API-key redaction.\r\n- \ud83d\udd27 **vllm-cpp LoRA fusion at load time** - `lora_adapters` and `lora_scales` are now consumed by the vllm-cpp video engine, fusing LoRA deltas into DiT weights at engine load. Works for both LTX2.5 and MiniMax-H3 through one generic path.\r\n- \ud83c\udf0d **Environment variables per model** - an `env:` map in model config injects variables into the backend process environment, so `CUDA_VISIBLE_DEVICES` or `VLLM_CACHE_DIR` can be pinned per model without wrapping the backend in a script.\r\n- \ud83d\udcac **`template.system_messages_after_first`** - agent frameworks that append system instructions mid-conversation no longer crash Qwen3.x models. Late system turns can be merged into the leading system message or forwarded as user turns.\r\n- \ud83e\udde0 **`context_size` in `/v1/models/capabilities`** - clients can now read each model's effective context window to budget tokens and truncate prompts without probing the model config out-of-band.\r\n- \ud83d\udee1\ufe0f **Four CVEs patched** - ip-address 10.3.1 (CVE-2026-69192), containerd 1.7.32 and 1.7.33 (CVE-2026-46680, CVE-2026-53488), and react-router (GHSA-qwww-vcr4-c8h2).\r\n- \ud83c\udf4e **Apple M5 SIGSEGV fixed** - `go-m1cpu` bumped to v0.2.2, fixing a segmentation fault on Apple M5 hardware at startup.\r\n\r\nPlus a focus mode toggle in chat settings, face enrollment replay for the 1:N registry, a ROCm build of `stablediffusion-ggml`, in-place editing of distributed scheduling rules, OCI layer download resumption, and 140 new gallery entries.\r\n\r\n\r\n\r\n---\r\n\r\n## \ud83d\udcca This release in numbers\r\n\r\n| | |\r\n|---|---|\r\n| Pull requests merged | **280** |\r\n| Commits | 382 |\r\n| Files changed | 633 (**+46,550** / -3,316) |\r\n| Development window | 28 days (2026-08-20 to 2026-09-17) |\r\n| Human contributors | 29, of whom **16 first-time** |\r\n| Gallery entries | 1,707 to **1,847** (+140) |\r\n\r\nWhere the work landed:\r\n\r\n| Area | Change |\r\n|---|---|\r\n| `core/` | +25,773 / -1,993 across 314 files |\r\n| `gallery/` | +6,734 / -417 |\r\n| `backend/` | +4,932 / -433 across 154 files |\r\n| `pkg/` | +4,392 / -91 across 55 files |\r\n| `docs/` | +2,526 / -101 across 48 files |\r\n| `swagger/` | +322 |\r\n\r\n---\r\n\r\n## \ud83d\udccc TL;DR\r\n\r\n| Area | Summary |\r\n|------|---------|\r\n| \ud83d\udcca **Fleet dashboard** | The Nodes page is rewritten as a fleet operations dashboard. A ClusterOverview shows health bands (healthy/draining/attention) and capacity gauges for VRAM, RAM, CPU and disk derived from the single `GET /api/nodes` response. A NodeFleetTable offers sortable, groupable, paginated rows with checkbox selection across filters and pages, and bulk Drain/Resume/Remove with bounded concurrency (limit 8). A Running Models tab shows all loaded replicas on healthy workers grouped by model, with replica count, node count, in-flight requests and backend types. Slide-out inspectors for nodes and models provide detail and drill-down. Workers now report `cpu_logical_cores`, `cpu_usage_percent` and `cpu_load_1` at registration and heartbeat. Lifecycle transitions are atomic: `MarkDraining` flips status and deletes model records in one transaction, `ResumeNode` is conditional (draining to healthy only), and HTTP endpoints return proper 404/409 instead of generic 500. |\r\n| \ud83d\udd11 **Credentials file** | `--credentials-file` / `LOCALAI_CREDENTIALS_FILE` loads a YAML list of rules, each matching a URL prefix and carrying one auth type: basic (`username` + `password`/`password_env`/`password_file`), bearer (`bearer`/`bearer_env`/`bearer_file`), or custom header (`header.name` + `header.value`/`_env`/`_file`). OCI registries consult the store first, then docker config. HTTP downloads, galleries and `github:` URIs authenticate through a `RoundTripper` that clones the request per redirect hop, so a cross-host redirect never carries the credential. Secrets are read at use time, not load time, so a rotated K8s secret mount takes effect without restart. `Credential` and `Store` implement `String`, `GoString` and `LogValue` so no `fmt` verb or `slog` call can print secret material. Unknown YAML keys are errors; match strings with `user:token@`, `?` or `#` are rejected at load. |\r\n| \ud83d\udccf **CLI benchmark** | `local-ai benchmark MODEL [MODEL ...]` sends non-streaming `POST /v1/chat/completions` requests at `temperature:0` with configurable `--runs` (default 3), `--warmup` (default 1), `--max-tokens` (default 128) and `--timeout` (default 5m). Reports min/mean/max latency and end-to-end completion tokens/sec per model. Output is a table by default or `--json` for structured data with per-sample latency and nullable token counts. The HTTP client refuses redirects, the API key is redacted from error messages, and the full report is buffered and only written after all models succeed, so a failure leaves no partial output. |\r\n| \ud83d\udd27 **vllm-cpp LoRA fusion** | The `lora_adapters`, `lora_scales`, `lora_adapter` and `lora_scale` config fields are now consumed by the vllm-cpp video backend. `buildLoraExtras` converts them into indexed `lora_path`/`lora_strength` key-value pairs (first adapter unsuffixed, subsequent ones `_2`, `_3`, ...), packs them via the ABI v18 extras seam, and hands them to `vllmVideoEngineLoad`. vllm.cpp fuses the LoRA deltas into the DiT weights at load. One generic path serves both LTX2.5 and MiniMax-H3. Adapters are always loaded; per-request prompt-activated LoRA is out of scope. |\r\n| \ud83c\udf0d **Per-model env vars** | An `env:` map on the model config is appended to the child process environment after the base env, so model vars override inherited system vars. Delivered through the common process-spawning layer in `pkg/model/process.go`, so every gRPC backend inherits it with no per-backend wiring. The UI renders an editable key/value map. Model vars take effect only when a model spawns its own backend process, not when it reuses a worker-supervised one. |\r\n| \ud83d\udcac **Late system messages** | `template.system_messages_after_first` controls what happens to system-role messages after the leading system block. `\"merge\"` folds them into the first system message with a `\\n\\n` separator; `\"user\"` rewrites them in place to role `user`; unset passes through. Fixes a Jinja exception (`System message must be at the beginning`) raised by Qwen3.x tokenizer templates when agent frameworks append system instructions mid-conversation. Runs after `stripEmptySystemMessages` and before tokenizer-template `system_prompt` injection. |\r\n| \ud83e\udde0 **context_size exposed** | `/v1/models/capabilities` now populates `entry.ContextSize` with `backend.EffectiveContextSize(cfg)` when the result is positive, letting clients budget tokens and truncate prompts without probing the model config out-of-band. |\r\n| \ud83d\udee1\ufe0f **CVE fixes** | ip-address to 10.3.1 (CVE-2026-69192), containerd to 1.7.32 (CVE-2026-46680) then 1.7.33 (CVE-2026-53488), react-router to 7.18.2 / 8.3.0 (GHSA-qwww-vcr4-c8h2). |\r\n| \ud83c\udf4e **Apple M5** | `go-m1cpu` bumped to v0.2.2, fixing a SIGSEGV on Apple M5 at startup. |\r\n| \ud83e\udde0 **Models** | 140 new gallery entries: Qwopus 3.6, OpenResearcher, LFM2.5 DSpark, Qwen3.8 small distills and Flash Next, UI-Mate 27B, Homura 30B, Ornith 1.5 35B and 9B, Granite 4.2, Tiel-Coder 35B, Thomson 1.0 Small, PhoneLLM, WeMM embedding, EXL3 vllm.cpp, Gemma 4 E4B, Orukeet for NeMo speech, and more. |\r\n\r\n---\r\n\r\n## \ud83d\ude80 New Features & Major Enhancements\r\n\r\n### \ud83d\udcca Fleet operations dashboard\r\n\r\nThe Nodes page was a flat list of per-node cards. It showed you what was registered but not what the cluster was doing, and every lifecycle action was one node at a time.\r\n\r\nThe rewrite is a fleet operations dashboard with two tabs and a shared summary poller.\r\n\r\n**ClusterOverview** sits at the top. A health band segments nodes into healthy, draining and attention counts. Capacity gauges show VRAM, RAM, CPU and disk, each reporting how many workers do not report that metric so the number is honest rather than averaged over reporting nodes. An attention queue filters for pending approval, offline/unhealthy, low VRAM, low RAM and low disk, surfacing nodes that need action without double-counting the headline total.\r\n\r\n**NodeFleetTable** is sortable, groupable (by label or type) and paginated at 50 rows per page, with checkbox selection that persists across filters and pages. Bulk Drain, Resume and Remove run with bounded concurrency (limit 8) so a fleet with thousands of registrations is usable. Columns: status pill, address, capacity (VRAM + RAM), CPU, workload and heartbeat. An Approve button appears on pending nodes.\r\n\r\n**Running models tab.** A fleet-wide view of loaded replicas on healthy workers, grouped by model: model name, replica count, node count, in-flight requests, backend types and last used. Per-model Stop shuts down all placements through `POST /backend/shutdown {model}`, which now fires the remote unloader even when a local process exists, and deduplicates nodes before sending stop messages.\r\n\r\n**Inspectors.** A slide-out NodeInspector shows health, labels, resource bars, CPU, loaded model count, backends (lazy-loaded) and in-flight, with Approve/Drain/Resume actions. A ModelInspector shows replica placement grouped by node, with clickable node names that drill into a NodeInspector.\r\n\r\nWorkers now report `cpu_logical_cores`, `cpu_usage_percent` and `cpu_load_1` at registration and heartbeat. The dashboard shows CPU capacity and utilization, with idle cores derived as `cpu_logical_cores * (1 - cpu_usage_percent / 100)`.\r\n\r\nLifecycle transitions are now atomic. `MarkDraining` flips status and deletes model records in one transaction rather than as separate operations that could interleave with `SetNodeModel`. `ResumeNode` is conditional: draining to healthy only, preventing overwriting of pending approval or concurrent state changes. HTTP endpoints return proper 404 (not found) and 409 (status conflict) instead of a generic 500.\r\n\r\n<img width=\"2880\" height=\"1800\" alt=\"ui-fleet-dashboard\" src=\"https://github.com/user-attachments/assets/14311767-9b0e-4c32-82a4-421c76d0f406\" />\r\n\r\n\r\n> \ud83d\udd17 PRs: #12046\r\n\r\n### \ud83d\udd11 Credentials file for private sources\r\n\r\nBackends, models and gallery indexes can live behind private OCI registries, internal HTTP servers or private GitHub repos. Before this, each download path had its own ad-hoc auth mechanism, and tokens were scattered across environment variables.\r\n\r\nOne `credentials.yaml` file, loaded by `--credentials-file` / `LOCALAI_CREDENTIALS_FILE`, authenticates them all. It is a top-level YAML list of rules. Each rule has a `match` URL prefix and exactly one auth type: basic (`username` + `password` / `password_env` / `password_file`), bearer (`bearer` / `bearer_env` / `bearer_file`), or custom header (`header.name` + `header.value` / `header.value_env` / `header.value_file`). Exactly one of the plain, `_env` or `_file` form per secret; unknown keys are errors so a misspelled `pasword_env` fails loudly.\r\n\r\nOCI registries (backend images, `oci://` and `ollama://` models, digest lookup, Range resume, cosign verify) consult the store first, then `authn.DefaultKeychain` (docker config). HTTP downloads, galleries and `github:` URIs authenticate through an `http.RoundTripper` that puts the credential on a clone of the request. net/http rebuilds each redirect hop from the original headers, so a redirect to another host never carries the credential. A caller-supplied credential (e.g. `HF_TOKEN`) takes precedence: the transport leaves an existing `Authorization` header alone.\r\n\r\nSecrets are read at use time, not load time, so a rotated K8s secret mount takes effect without restart. `Credential` and `Store` implement `String`, `GoString` and `LogValue`, so no `fmt` verb or `slog` call, including when nested in unexported fields, can print secret material. Credentials go only to `https`, or to `http` when the rule sets `allow_insecure`. Match strings containing `user:token@`, `?` (where signed URLs carry tokens) or `#` are rejected at load. YAML decode errors are redacted: quoted scalar values (often secrets) are stripped, keeping only line numbers and parser wording.\r\n\r\n> \ud83d\udd17 PRs: #12028\r\n\r\n### \ud83d\udccf `local-ai benchmark` CLI command\r\n\r\nA benchmark command for configured text models, answering \"how fast is this model on this hardware\" without leaving the terminal.\r\n\r\n`local-ai benchmark MODEL [MODEL ...]` sends non-streaming `POST /v1/chat/completions` requests at `temperature:0` with configurable `--runs` (default 3), `--warmup` (default 1), `--max-tokens` (default 128) and `--timeout` (default 5m). It reports min, mean and max latency and end-to-end completion tokens/sec per model.\r\n\r\nThe throughput metric is `sum(completion_tokens) / sum(latencies)` across measured runs. It is `nil`/`N/A` if any sample omits `completion_tokens` from the response usage, or if total latency is zero. The command does not measure decode-only speed or time to first token; it is an end-to-end client benchmark covering HTTP transport, queueing, prompt processing, generation and parse.\r\n\r\nThe HTTP client refuses redirects by default, so a misconfigured endpoint behind a redirect does not silently succeed. The API key is redacted from error messages, and raw transport errors are masked as `HTTP request failed` to avoid credential echo. The full report is buffered in a `bytes.Buffer` and only written via `io.Copy` after all models succeed, so a failure leaves no partial output. Signal handling cancels the active request on SIGINT/SIGTERM.\r\n\r\nOutput is a table by default or `--json` for structured data with per-sample latency and nullable token counts. The API key is not included in the JSON. To compare backends, configure separate model aliases per backend and pass those alias names.\r\n\r\n> \ud83d\udd17 PRs: #12020\r\n\r\n### \ud83d\udd27 vllm-cpp LoRA fusion at load time\r\n\r\nThe `lora_adapters` and `lora_scales` config fields are standard LocalAI fields that list LoRA adapter weight files and per-adapter strength multipliers. The vllm-cpp video backend now consumes them for the first time, fusing LoRA deltas into DiT (diffusion Transformer) weights at engine load.\r\n\r\n`buildLoraExtras` converts the config fields into indexed `lora_path` / `lora_strength` key-value pairs. The first adapter is unsuffixed; subsequent ones get `_2`, `_3`, etc. If the singular `lora_adapter` is set, it is prepended as the first adapter before the plural list. Relative adapter paths are joined to the model path; absolute paths pass through. Strength defaults to 1.0 when no scale is provided.\r\n\r\nThe pairs are packed via the ABI v18 extras seam and handed to `vllmVideoEngineLoad`. vllm.cpp's `ResolveDitLoraSpecs` fuses the LoRA deltas into the DiT weights at load. One generic path serves both LTX2.5 and MiniMax-H3, since the engine auto-detects the DiT family from the checkpoint.\r\n\r\nAdapters are always loaded; there is no per-request activation in this path. Runtime prompt-activated LoRA (punica-style `<lora:name:strength>`) is out of scope and tracked separately.\r\n\r\n> \ud83d\udd17 PRs: #12082\r\n\r\n### \ud83c\udf0d Environment variables per model\r\n\r\nAn `env:` map on the model config injects variables into the backend process environment. The primary use case is pinning `CUDA_VISIBLE_DEVICES` per model to control GPU placement, but it applies to any backend that reads environment variables.\r\n\r\n```yaml\r\nname: vllm-model\r\nbackend: vllm\r\nenv:\r\n CUDA_VISIBLE_DEVICES: \"0,1\"\r\n VLLM_CACHE_DIR: \"/tmp/vllm_cache\"\r\n```\r\n\r\nThe map is delivered through the common process-spawning layer in `pkg/model/process.go`, so every gRPC backend inherits it with no per-backend wiring. Model vars are appended last, after the base env, so they override same-named inherited system vars. The UI renders an editable key/value map.\r\n\r\nModel vars take effect only when a model spawns its own backend process, not when it reuses a worker-supervised one. There is no allowlist or blocklist: model configs are an operator trust boundary, consistent with LocalAI's general posture.\r\n\r\n> \ud83d\udd17 PRs: #10721\r\n\r\n### \ud83d\udcac `template.system_messages_after_first`\r\n\r\nAgent frameworks legitimately append system instructions mid-conversation: a tool-parameter prompt before a call, an adjustment prompt after a result. The Qwen3.x family of tokenizer templates raises `Jinja Exception: System message must be at the beginning` for any system-role message after the first turn, returning HTTP 500. Plain chat looks healthy; every agent task fails.\r\n\r\n`template.system_messages_after_first` controls what happens to system-role messages that appear after the contiguous leading system block:\r\n\r\n- `\"merge\"` folds late system messages into the first system message with a `\\n\\n` separator, dropping them from their original position.\r\n- `\"user\"` rewrites each late system message in place to role `user`, keeping its position and content.\r\n- unset (default) passes through unchanged; the tokenizer template decides.\r\n\r\nThe normalization runs in `ChatEndpoint` after `stripEmptySystemMessages` and before tokenizer-template `system_prompt` injection, so the Jinja template never sees an out-of-place system message. It only touches requests whose model config opts in, so default behavior is unchanged.\r\n\r\n`\"user\"` is the recommended mode for agent workloads: the instruction stays where the framework placed it, and the model follows it.\r\n\r\n> \ud83d\udd17 PRs: #11906\r\n\r\n### \ud83e\udde0 `context_size` in `/v1/models/capabilities`\r\n\r\nThe capabilities endpoint now populates `entry.ContextSize` with `backend.EffectiveContextSize(cfg)` when the result is positive. Clients can read each model's effective context window to budget tokens, truncate prompts and history, and avoid sending more context than the model accepts, without having to probe the model config out-of-band.\r\n\r\n> \ud83d\udd17 PRs: #12083\r\n\r\n### \ud83d\udd75\ufe0f Face enrollment replay\r\n\r\nThe face recognition 1:N registry gains a replay path. `FaceRegisterRequest` now accepts an `Embedding` (pre-computed vector) and `RegisteredAt` (original timestamp), so a client that previously extracted an embedding via `/v1/face/embed` can replay the saved vector, name, labels and timestamp directly, skipping image decoding and the `FaceEmbed` inference call entirely.\r\n\r\nThe store derives a deterministic ID from the embedding bytes and store namespace via `uuid.NewSHA1`, so replaying the same vector after a restart or on another replica returns the same ID without creating duplicates. The server does not persist the registry itself; clients must retain and replay the records to restore the registry after a restart.\r\n\r\n> \ud83d\udd17 PRs: #11908\r\n\r\n### \ud83c\udfaf Focus mode toggle\r\n\r\nFocus mode (which collapses the global app sidebar and slims the header once a conversation has at least one message, with Esc temporarily restoring them) already existed as session-only behavior. This adds a persistent opt-out toggle in the chat settings drawer, backed by `localStorage` under `localai_chat_focus_mode`. It defaults to enabled; with the toggle off, the full layout stays even in an active conversation.\r\n\r\n> \ud83d\udd17 PRs: #11750\r\n\r\n### \ud83e\uddf0 Smaller features worth knowing about\r\n\r\n- **`stablediffusion-ggml` builds a ROCm variant.** A ROCm build of the stable-diffusion ggml backend, extending accelerator coverage beyond CUDA and Metal.\r\n- **Scheduling rules editable in place.** Distributed scheduling rules can now be edited in place in the UI rather than through a round-trip to the config file.\r\n- **Scheduling rules keyed by model alias.** Distributed scheduling rules can be keyed by a model alias, not just a model name, so aliased models participate in scheduling correctly.\r\n- **XIO copy buffer size is configurable.** The copy buffer size for XIO transfers is now configurable rather than fixed.\r\n- **OCI layer download resumption.** Interrupted OCI layer downloads resume rather than restarting from scratch.\r\n- **Progress updates no longer throttle downloads.** Status updates were throttling downloads; the progress path is now bounded so it does not compete with the transfer.\r\n- **Node operation controls restored.** The node detail page's operation controls (drain, resume, remove) are restored after a regression that hid them, and backend actions are tested through their menu.\r\n\r\n> \ud83d\udd17 PRs: #11774, #11667, #11771, #11660, #11688, #11661, #12068, #12069\r\n\r\n---\r\n\r\n## \ud83d\udc1b Bug Fixes (recap)\r\n\r\n**Security / CVE:**\r\n- `fix`: upgrade ip-address to 10.3.1 (CVE-2026-69192) - #11632\r\n- `fix`: upgrade containerd to 1.7.32 (CVE-2026-46680) - #11634\r\n- `fix`: upgrade react-router to 7.18.2, 8.3.0 (GHSA-qwww-vcr4-c8h2) - #11644\r\n- `fix`: upgrade containerd to 1.7.33 (CVE-2026-53488) - #11655\r\n- `fix(auth)`: bypass API-key auth for CORS preflight (OPTIONS) requests - #11113\r\n\r\n**ds4:**\r\n- `fix(ds4)`: separate prefilled reasoning from content - #11802\r\n- `fix(ds4)`: enforce generation boundaries - #11821\r\n- `fix(ds4)`: cancel abandoned inference - #11822\r\n- `fix(ds4)`: build CUDA kernels for the target architecture - #11840\r\n- `fix(ds4)`: resolve Hugging Face repository imports - #11340\r\n\r\n**Distributed mode:**\r\n- `fix(distributed)`: restore node liveness tests - #11694\r\n- `fix(distributed)`: survive a slow control-plane database - #11837\r\n- `fix(distributed)`: stage sound detection audio - #11907\r\n- `fix(distributed)`: finalize stalled model uploads - #11910\r\n- `fix(distributed)`: resolve paths for virtual models - #11911\r\n- `fix(distributed)`: bound ephemeral staging - #11924\r\n- `fix(distributed)`: pass raw JPEG base64 through staging - #11956\r\n- `fix(distributed)`: keep staging operations stable - #11663\r\n- `fix(distributed)`: keep model replicas config-consistent - #11664\r\n\r\n**Realtime:**\r\n- `fix(realtime)`: accept GA WebRTC signaling - #11778\r\n- `fix(realtime)`: skip responses for empty transcripts - #11940\r\n- `fix(realtime)`: resolve pipeline voice profiles - #11942\r\n- `fix(realtime)`: support voice profile switching - #11948\r\n\r\n**Backends:**\r\n- `fix(backends)`: honor enable_thinking=false in sglang and vllm - #11715\r\n- `fix(backends)`: honor enable_thinking=false in mlx and vllm-omni - #11962\r\n- `fix(diffusers)`: auto-detect CUDA instead of defaulting to CPU - #11891\r\n- `fix`: prefer the Intel diffusers venv SYCL runtime at startup - #11971\r\n- `fix(qwen-tts)`: restore CUDA 13 SDPA fallback - #12067\r\n- `fix(faster-whisper)`: manually install ctranslate2 with rocm support - #9839\r\n- `fix(whisper)`: honour positional listen address argument - #11652\r\n- `fix(backends)`: bound temporary scratch files - #11941\r\n- `fix(openresponses)`: support Codex WebSocket warm-up - #11608\r\n\r\n**Gallery and models:**\r\n- `fix(gallery)`: tag MTP-enabled models - #11653\r\n- `fix(gallery)`: remove duplicate Qwopus entries - #11670\r\n- `fix(gallery)`: use published F16 mmproj for qwythos-9b - #11792\r\n- `fix(gallery)`: persist inference defaults where the loader reads them - #11232\r\n- `fix(gallery)`: default audio-cpp models to backend:best - #11892\r\n- `fix(gallery)`: restore entries clobbered by batch PR applies - #12016\r\n- `fix(gallery)`: use native Ministral tool parsing - #12022\r\n- `fix(vram)`: persist remote probe metadata - #11487\r\n- `fix(ollama)`: accept :latest tag on model lookup - #11732\r\n\r\n**UI:**\r\n- `fix(ui)`: omit empty system prompt so model default applies - #11838\r\n- `fix(ui)`: omit GPU recommendations that do not fit - #11945\r\n- `fix(ui)`: preserve percent signs in route parameters - #11883\r\n- `fix(ui)`: restore node operation controls - #12068\r\n- `fix(ui)`: test backend actions through their menu - #12069\r\n- `fix(ui)`: size model fit against the cluster - #11765\r\n\r\n**Other:**\r\n- `fix(deps)`: bump go-m1cpu to v0.2.2 to fix SIGSEGV on Apple M5 - #11736\r\n- `fix(downloader)`: make file:// installs reachable again - #11734\r\n- `fix(detection)`: avoid temporary image files - #11938\r\n- `fix(worker)`: resolve temporary paths in tests - #11944\r\n- `fix(ci)`: bound Discord release summaries - #11695\r\n- `fix(ci)`: stop the e2e teardown from failing a green suite - #11816\r\n- `fix(ci)`: unbreak the e2e build and the darwin vllm-metal pin - #11849\r\n- `fix(progress)`: stop status updates throttling downloads - #11661\r\n- `fix(oci)`: resume interrupted layer downloads - #11688\r\n\r\n---\r\n\r\n## \ud83e\udde0 Models\r\n\r\n140 new gallery entries this cycle, taking the index from 1,707 to 1,847.\r\n\r\nText generation: Qwopus 3.6, OpenResearcher, LFM2.5 DSpark, Qwen3.8 small distills, Qwen3.8 Flash Next, Qwen3.8 OBLITERATED, Qwen3.8 Cold Fusion, Qwen3.8 GSQ-RCO, UI-Mate 27B, Homura 30B, Ornith 1.5 35B and 9B (including OBLITERATED), Granite 4.2, Tiel-Coder 35B, Thomson 1.0 Small, PhoneLLM, and Huihui Qwen3.8 Flash Next.\r\n\r\nEmbeddings: WeMM embedding variants.\r\n\r\nEXL3: vllm.cpp EXL3 models.\r\n\r\nVision: Gemma 4 E4B HauhauCS variants.\r\n\r\nSpeech: Orukeet added to the NeMo speech backend.\r\n\r\n> \ud83d\udd17 PRs: #11662, #11658, #11676, #11675, #11672, #11680, #11691, #11714, #11719, #11723, #11766, #11772, #11763, #11775, #11779, #11754, #11751, #11787, #11803, #11912, #11625, #11903, #11921, #11998, #11716\r\n\r\n---\r\n\r\n## \ud83d\udc52 Dependencies\r\n\r\nSubmodule and pin bumps this cycle:\r\n\r\n| Project | Bumps |\r\n|---|---|\r\n| 0xShug0/audio.cpp | 16 |\r\n| ggml-org/llama.cpp | 12 |\r\n| CrispStrobe/CrispASR | 10 |\r\n| ikawrakow/ik_llama.cpp | 8 |\r\n| leejet/stable-diffusion.cpp | 7 |\r\n| mudler/vllm.cpp | 6 |\r\n| ggml-org/whisper.cpp | 6 |\r\n| antirez/ds4 | 6 |\r\n| ServeurpersoCom/omnivoice.cpp | 5 |\r\n| NVIDIA/NeMo-Speech.cpp | 4 |\r\n| PrismML-Eng/llama.cpp | 3 |\r\n| mudler/depth-anything.cpp | 2 |\r\n| vllm-project/vllm cu130 wheel, vllm-metal (darwin), OpenNMT/CTranslate2 ROCm wheel, TheTom/llama-cpp-turboquant | 1 each |\r\n\r\nPlus vllm 0.26.0 to 0.29.0 in the Python backends, torch 2.13.0+xpu to 2.14.0+xpu, grpcio to 1.83.1 across coqui/rerankers/vllm/template, charset-normalizer to 3.5.1, securego/gosec to 2.29.0, LocalAGI and localrecall to v0.6.5, 16 stale dependabot entries removed, npm_and_yarn group updates, and inference defaults refreshed from unsloth.\r\n\r\n---\r\n\r\n## \ud83d\udcd6 Documentation\r\n\r\nNew pages for the CLI benchmark command and the credentials file / private sources guide. The distributed-mode docs document the fleet dashboard, CPU telemetry and the new endpoint error codes. Model configuration gained the `env:` map, `template.system_messages_after_first` and `context_size` documentation. The DCO guide clarifies that maintainer-operated automation may sign off. Installation paths were simplified, an Apache APISIX reverse proxy example was added, dead anchors and section links were fixed, documented env var and CLI flag names were corrected, and the containers page was fixed for Podman image qualification. A FAQ note covers `OPENAI_BASE_URL` for pointing clients at LocalAI, and Solstone was added to the README and ADOPTERS.md.\r\n\r\n> \ud83d\udd17 PRs: #11631, #11294, #11850, #11700, #11885, #11884, #11886, #11749, #11904, #11310, #12044, #12045\r\n\r\n---\r\n\r\n## \ud83d\ude4c New Contributors\r\n\r\n- @localai-org-bot made their first contribution in #11631\r\n- @DanielSwift1992 made their first contribution in #11686\r\n- @leilei3167 made their first contribution in #11715\r\n- @sz-po made their first contribution in #11736\r\n- @blackd made their first contribution in #11750\r\n- @ginavalent made their first contribution in #11806\r\n- @FiloSpaTeam made their first contribution in #11802\r\n- @aimansour made their first contribution in #11700\r\n- @simpleqt made their first contribution in #11885\r\n- @alex-mazzariol made their first contribution in #11749\r\n- @seven7763 made their first contribution in #11310\r\n- @QiuLsG made their first contribution in #11883\r\n- @SomSamantray made their first contribution in #11652\r\n- @devv-shayan made their first contribution in #11962\r\n- @Nathan-Roll1 made their first contribution in #11998\r\n- @quartzjer made their first contribution in #12045\r\n\r\nThanks also to @mudler, @anupamme, @mvanhorn, @jimmykarily, @Nold360, @Anai-Guo, @SuperMarioYL, @eglia and @walcz-de for their contributions this cycle.\r\n\r\n<!-- Release notes generated using configuration in .github/release.yml at master -->\r\n\r\n## What's Changed\r\n### Bug fixes :bug:\r\n* fix: upgrade ip-address to 10.3.1 (CVE-2026-69192) by @anupamme in https://github.com/mudler/LocalAI/pull/11632\r\n* fix: upgrade github.com/containerd/containerd to 1.7.32 (CVE-2026-46680) by @anupamme in https://github.com/mudler/LocalAI/pull/11634\r\n* fix: upgrade react-router to 7.18.2, 8.3.0 (GHSA-qwww-vcr4-c8h2) by @anupamme in https://github.com/mudler/LocalAI/pull/11644\r\n* fix(gallery): tag MTP-enabled models by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/11653\r\n* fix: upgrade github.com/containerd/containerd to 1.7.33 (CVE-2026-53488) by @anupamme in https://github.com/mudler/LocalAI/pull/11655\r\n* fix(gallery): remove duplicate Qwopus entries by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/11670\r\n* fix(vram): persist remote probe metadata by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/11487\r\n* fix(distributed): restore node liveness tests by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/11694\r\n* fix(backends): honor enable_thinking=false in sglang and vllm by @leilei3167 in https://github.com/mudler/LocalAI/pull/11715\r\n* fix(ollama): accept :latest tag on model lookup by @leilei3167 in https://github.com/mudler/LocalAI/pull/11732\r\n* fix(deps): bump go-m1cpu to v0.2.2 to fix SIGSEGV on Apple M5 by @sz-po in https://github.com/mudler/LocalAI/pull/11736\r\n* fix(realtime): accept GA WebRTC signaling by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/11778\r\n* fix(gallery): use published F16 mmproj for qwythos-9b by @leilei3167 in https://github.com/mudler/LocalAI/pull/11792\r\n* fix(ds4): separate prefilled reasoning from content by @FiloSpaTeam in https://github.com/mudler/LocalAI/pull/11802\r\n* fix(ds4): enforce generation boundaries by @FiloSpaTeam in https://github.com/mudler/LocalAI/pull/11821\r\n* fix(distributed): survive a slow control-plane database by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/11837\r\n* fix(ui): omit empty system prompt so model default applies by @leilei3167 in https://github.com/mudler/LocalAI/pull/11838\r\n* fix(ds4): cancel abandoned inference by @FiloSpaTeam in https://github.com/mudler/LocalAI/pull/11822\r\n* fix(ds4): build CUDA kernels for the target architecture by @FiloSpaTeam in https://github.com/mudler/LocalAI/pull/11840\r\n* fix(gallery): persist inference defaults where the loader reads them by @jimmykarily in https://github.com/mudler/LocalAI/pull/11232\r\n* fix(downloader): make file:// installs reachable again (#11701) by @Anai-Guo in https://github.com/mudler/LocalAI/pull/11734\r\n* fix(distributed): stage sound detection audio by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/11907\r\n* fix(distributed): finalize stalled model uploads by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/11910\r\n* fix(distributed): resolve paths for virtual models by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/11911\r\n* fix(diffusers): auto-detect CUDA instead of defaulting to CPU by @blackd in https://github.com/mudler/LocalAI/pull/11891\r\n* fix(gallery): default audio-cpp models to backend:best by @blackd in https://github.com/mudler/LocalAI/pull/11892\r\n* fix(distributed): bound ephemeral staging by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/11924\r\n* fix(detection): avoid temporary image files by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/11938\r\n* fix(realtime): skip responses for empty transcripts by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/11940\r\n* fix(realtime): resolve pipeline voice profiles by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/11942\r\n* fix(backends): bound temporary scratch files by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/11941\r\n* fix(worker): resolve temporary paths in tests by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/11944\r\n* fix(realtime): support voice profile switching by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/11948\r\n* fix(distributed): pass raw JPEG base64 through staging by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/11956\r\n* fix(ui): omit GPU recommendations that do not fit by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/11945\r\n* fix: prefer the Intel diffusers venv SYCL runtime at startup by @mvanhorn in https://github.com/mudler/LocalAI/pull/11971\r\n* fix(ds4): resolve Hugging Face repository imports by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/11340\r\n* fix(openresponses): support Codex WebSocket warm-up by @aimansour in https://github.com/mudler/LocalAI/pull/11608\r\n* fix(ui): preserve percent signs in route parameters by @QiuLsG in https://github.com/mudler/LocalAI/pull/11883\r\n* fix(whisper): honour positional listen address argument by @SomSamantray in https://github.com/mudler/LocalAI/pull/11652\r\n* fix(auth): bypass API-key auth for CORS preflight (OPTIONS) requests by @SuperMarioYL in https://github.com/mudler/LocalAI/pull/11113\r\n* fix(backends): honor enable_thinking=false in mlx and vllm-omni by @devv-shayan in https://github.com/mudler/LocalAI/pull/11962\r\n* fix(faster-whisper): manually install ctranslate2 with rocm support before installing other dependencies by @eglia in https://github.com/mudler/LocalAI/pull/9839\r\n* fix(gallery): restore entries clobbered by batch PR applies by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/12016\r\n* fix(gallery): use native Ministral tool parsing by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/12022\r\n* fix(ui): restore node operation controls by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/12068\r\n* fix(qwen-tts): restore CUDA 13 SDPA fallback by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/12067\r\n* fix(ui): test backend actions through their menu by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/12069\r\n### Exciting New Features \ud83c\udf89\r\n* feat(ui): add 'Focus mode' option in chat settings to persistently toggle the sidebar auto-collapse by @blackd in https://github.com/mudler/LocalAI/pull/11750\r\n* feat(faces): replay saved face enrollments by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/11908\r\n* [chat] feat: template.system_messages_after_first \u2014 merge or forward late system turns by @walcz-de in https://github.com/mudler/LocalAI/pull/11906\r\n* [model-config] feat: add environment variables support for backends by @Nold360 in https://github.com/mudler/LocalAI/pull/10721\r\n* feat(cli): benchmark configured text models by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/12020\r\n* feat(downloads): authenticate registries, galleries and downloads from a credentials file by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/12028\r\n* feat(nodes): add fleet operations dashboard by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/12046\r\n* feat: expose context_size in /v1/models/capabilities by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/12083\r\n* [vllm-cpp] wire lora_adapters/lora_scales into DiT load-time fusion by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/12082\r\n### \ud83e\udde0 Models\r\n* feat(gallery): add Qwopus 3.6 variants by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/11662\r\n* feat(gallery): add OpenResearcher variants by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/11658\r\n* feat(gallery): add LFM2.5 DSpark variants by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/11676\r\n* feat(gallery): add Qwen3.8 small distills by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/11675\r\n* feat(gallery): add UI-Mate 27B variants by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/11672\r\n* feat(gallery): add Homura 30B by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/11680\r\n* feat(gallery): add Qwen3.8 OBLITERATED variants by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/11691\r\n* feat(gallery): add Ornith 1.5 35B variants by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/11714\r\n* feat(gallery): add Granite 4.2 variants by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/11719\r\n* feat(gallery): add Tiel-Coder 35B variants by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/11723\r\n* feat(gallery): add Thomson 1.0 Small variants by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/11766\r\n* feat(gallery): add PhoneLLM variants by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/11772\r\n* feat(gallery): add Qwen3.8 Flash Next by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/11763\r\n* feat(gallery): add WeMM embedding variants by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/11775\r\n* feat(gallery): add Granite 4.2 variants by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/11779\r\n* feat(gallery): add Qwen3.8 Cold Fusion by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/11754\r\n* feat(gallery): add LFM2.5 8B DSpark variants by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/11751\r\n* feat(gallery): add Qwen3.8 GSQ-RCO variants by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/11787\r\n* feat(gallery): add Ornith 1.5 9B OBLITERATED by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/11803\r\n* feat(gallery): add EXL3 vllm.cpp models by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/11912\r\n* feat(gallery): add Ornith 1.5 35B variants by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/11625\r\n* feat(gallery): add Gemma 4 E4B HauhauCS variants by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/11903\r\n* feat(gallery): add Huihui Qwen3.8 Flash Next by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/11921\r\n* [gallery] feat: add Orukeet to the NeMo speech backend by @Nathan-Roll1 in https://github.com/mudler/LocalAI/pull/11998\r\n### \ud83d\udcd6 Documentation and examples\r\n* docs(website): simplify installation paths by @localai-org-bot in https://github.com/mudler/LocalAI/pull/11631\r\n* docs: add Apache APISIX reverse proxy example by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/11294\r\n* docs(dco): let maintainer-operated automation sign off by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/11850\r\n* docs(llama-cpp): clarify multimodal speculative decoding by @aimansour in https://github.com/mudler/LocalAI/pull/11700\r\n* docs: fix dead anchors and a dead section link by @simpleqt in https://github.com/mudler/LocalAI/pull/11885\r\n* docs: fix dead links in the PR template and backends docs by @simpleqt in https://github.com/mudler/LocalAI/pull/11884\r\n* docs: correct documented env var and CLI flag names by @simpleqt in https://github.com/mudler/LocalAI/pull/11886\r\n* Update containers.md to fix podman image qualification by @alex-mazzariol in https://github.com/mudler/LocalAI/pull/11749\r\n* docs(integrations): add Distribution Packages section by @blackd in https://github.com/mudler/LocalAI/pull/11904\r\n* docs(faq): note OPENAI_BASE_URL env var for pointing clients at LocalAI by @seven7763 in https://github.com/mudler/LocalAI/pull/11310\r\n* Add Solstone app link and image to README by @mudler in https://github.com/mudler/LocalAI/pull/12044\r\n* Add solstone to ADOPTERS.md by @quartzjer in https://github.com/mudler/LocalAI/pull/12045\r\n### \ud83d\udc52 Dependencies\r\n* chore(deps): update charset-normalizer requirement from >=3.4.9 to >=3.5.1 in /backend/python/vllm by @dependabot[bot] in https://github.com/mudler/LocalAI/pull/11637\r\n* chore(deps): remove 16 dependabot entries for directories that no longer exist by @DanielSwift1992 in https://github.com/mudler/LocalAI/pull/11686\r\n* chore(deps): bump vllm from 0.26.0 to 0.28.0 in /backend/python/vllm by @dependabot[bot] in https://github.com/mudler/LocalAI/pull/11752\r\n* chore: :arrow_up: Update leejet/stable-diffusion.cpp to `6b3edaaf32cc19e5bb2d819c788bd557eddc8eba` by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/11793\r\n* chore: :arrow_up: Update ggml-org/whisper.cpp to `eacbd8234c6654cdbf2c377f72b2106875479bdc` by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/11796\r\n* chore: :arrow_up: Update CrispStrobe/CrispASR to `18b3e3f8456748a6380dc4c13817df244b695d39` by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/11799\r\n* chore: :arrow_up: Update ikawrakow/ik_llama.cpp to `3c58ae373a0081c884099f435fb16ca720852bf7` by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/11809\r\n* chore: :arrow_up: Update mudler/vllm.cpp to `6a544bdb89eb5a3512ac922241439e45f24d74d4` by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/11797\r\n* chore: :arrow_up: Update 0xShug0/audio.cpp to `bf3315fe4aaa16dc1125f580c29aff90a8900b36` by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/11794\r\n* chore: :arrow_up: Update mudler/depth-anything.cpp to `02ba082274e001a63e50de5a1eb0ccc50c6af4b1` by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/11810\r\n* chore: :arrow_up: Update CrispStrobe/CrispASR to `78c545eb80409b91291642ddb23b3a6dc044fd34` by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/11811\r\n* chore: :arrow_up: Update 0xShug0/audio.cpp to `3497b7cc44753e2c141d8fe60ac42cec433e3281` by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/11818\r\n* chore: :arrow_up: Update mudler/vllm.cpp to `839ea1ceddb787778b6bd86a38a917a1aab74d8f` by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/11817\r\n* chore: :arrow_up: Update ggml-org/llama.cpp to `3466812d1f06728effe7c0f3c0671117f461672d` by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/11798\r\n* chore: :arrow_up: Update 0xShug0/audio.cpp to `f334cff70a68ea3d2e40d6638733e8c1ec434164` by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/11830\r\n* chore: :arrow_up: Update mudler/vllm.cpp to `6bf3abb580982f4fd2e4525ef37802ee0ce28981` by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/11828\r\n* chore: :arrow_up: Update ikawrakow/ik_llama.cpp to `caf7eae5282d840d77e9f91a56df7d2ef28fa612` by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/11842\r\n* chore: :arrow_up: Update NVIDIA/NeMo-Speech.cpp to `56b60d432f1731d6d5b28a4c5a31cbaf871daba1` by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/11846\r\n* chore: :arrow_up: Update CrispStrobe/CrispASR to `ff3945c94cab9191199a5d531a32c4e9535c094b` by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/11829\r\n* chore: :arrow_up: Update 0xShug0/audio.cpp to `c18b7f737aac0a2855e9f963a427498739ad40fe` by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/11843\r\n* chore: :arrow_up: Update ggml-org/llama.cpp to `9cffdcc801582616250520966699cb5b25d28243` by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/11845\r\n* chore(deps): bump the npm_and_yarn group across 1 directory with 2 updates by @dependabot[bot] in https://github.com/mudler/LocalAI/pull/11835\r\n* chore: :arrow_up: Update ikawrakow/ik_llama.cpp to `fe215a8ccdce6b844d2a3a3bbde08ae76a6284bf` by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/11864\r\n* chore(deps): bump grpcio from 1.83.0 to 1.83.1 in /backend/python/coqui by @dependabot[bot] in https://github.com/mudler/LocalAI/pull/11856\r\n* chore(deps): bump grpcio from 1.82.1 to 1.83.1 in /backend/python/rerankers by @dependabot[bot] in https://github.com/mudler/LocalAI/pull/11852\r\n* chore(deps): bump grpcio from 1.83.0 to 1.83.1 in /backend/python/vllm by @dependabot[bot] in https://github.com/mudler/LocalAI/pull/11853\r\n* chore: :arrow_up: Update 0xShug0/audio.cpp to `8fe1ab805555c72aaf06d83828b523dfc6ed796d` by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/11861\r\n* chore: :arrow_up: Update ServeurpersoCom/omnivoice.cpp to `78ce63c90d36b94a765073e8cbe0d241fc114590` by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/11865\r\n* chore: :arrow_up: Update NVIDIA/NeMo-Speech.cpp to `69d7fd46b253043ed6649bf35d68407448912a8e` by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/11863\r\n* chore: :arrow_up: Update ggml-org/whisper.cpp to `52a939a2a762224e255d366c1182b2af4dd1a032` by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/11877\r\n* chore(deps): bump grpcio from 1.82.1 to 1.83.1 in /backend/python/common/template by @dependabot[bot] in https://github.com/mudler/LocalAI/pull/11858\r\n* chore: :arrow_up: Update 0xShug0/audio.cpp to `cd98afb3264044aeff33618df486c81de1bee7e7` by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/11878\r\n* chore(deps): bump the npm_and_yarn group across 1 directory with 3 updates by @dependabot[bot] in https://github.com/mudler/LocalAI/pull/11869\r\n* chore: :arrow_up: Update ggml-org/llama.cpp to `427291b5b34cd914a31b3fd3b61a68f6184f4b9f` by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/11866\r\n* chore: :arrow_up: Update CrispStrobe/CrispASR to `c6d4265ae2ee2b8931b09d7d25d5c65c75c36a41` by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/11862\r\n* chore(deps): bump torch from 2.13.0+xpu to 2.14.0+xpu in /backend/python/common/template by @dependabot[bot] in https://github.com/mudler/LocalAI/pull/11859\r\n* chore: :arrow_up: Update NVIDIA/NeMo-Speech.cpp to `ffa38cb2408f1e832a36d46fef5e3e1e80d07e6c` by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/11887\r\n* chore: :arrow_up: Update ServeurpersoCom/omnivoice.cpp to `040c8b344d8c670ce1475194751d119b5ef82c78` by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/11890\r\n* chore: :arrow_up: Update 0xShug0/audio.cpp to `c41ffefb44c16345c0ac57a0030f4cd3b5fbc4b8` by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/11888\r\n* chore: :arrow_up: Update ggml-org/llama.cpp to `74a7c897f049c17e7080423aa2111776eff6ebbf` by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/11889\r\n* chore: :arrow_up: Update 0xShug0/audio.cpp to `f6277c1695a83cf388a8282c1c1a8757cf626f18` by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/11899\r\n* chore: :arrow_up: Update ggml-org/llama.cpp to `465e49b9cea78a68b9c244ffb48d0ee24a82873d` by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/11897\r\n* chore: :arrow_up: Update antirez/ds4 to `9ab705347c1775e7599ede7eb81a6255ec7dccb5` by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/11808\r\n* chore: :arrow_up: Update leejet/stable-diffusion.cpp to `d8fb10c02977c8ca999f3fb4e02df9ecf10f7ba6` by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/11898\r\n* chore: :arrow_up: Update ggml-org/llama.cpp to `67672dc5b76f8bc17785a19d3dc6d1463fc2902c` by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/11920\r\n* chore: :arrow_up: Update leejet/stable-diffusion.cpp to `d04e8950c1ec8d30248cbe996682b3182fb1adf6` by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/11919\r\n* chore: :arrow_up: Update mudler/depth-anything.cpp to `14f7461d1f704761a038ac9f50dbde8fdb7275e2` by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/11918\r\n* chore: :arrow_up: Update antirez/ds4 to `f62ca29a308724cde5bc99134ede19104b2a3260` by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/11917\r\n* chore: :arrow_up: Update CrispStrobe/CrispASR to `301acd87b036764973b8bfba71e0a21818036d33` by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/11916\r\n* chore: :arrow_up: Update 0xShug0/audio.cpp to `9c6a282337cc83f227cc10428867a478947706ad` by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/11915\r\n* chore: :arrow_up: Update ggml-org/whisper.cpp to `c44b60b8053bbf2a5c1e014f11323fb3f2485177` by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/11937\r\n* chore: :arrow_up: Update antirez/ds4 to `6289c516273979173abbc062209a81dd3706b804` by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/11936\r\n* chore: :arrow_up: Update NVIDIA/NeMo-Speech.cpp to `a5b6953c4a579a2bbd1c0913ad8a85c2a4d99953` by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/11935\r\n* chore: :arrow_up: Update 0xShug0/audio.cpp to `05e508a70e3600b01454c647cdb122133ba8e64c` by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/11933\r\n* chore: :arrow_up: Update ikawrakow/ik_llama.cpp to `1a2a8604a6c6c6413c06bf9adfc2f64329af4366` by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/11932\r\n* chore: :arrow_up: Update ggml-org/llama.cpp to `f3f1a8f2760f28325a5ec20c05b171e5b7c83a29` by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/11934\r\n* chore(deps): bump the npm_and_yarn group across 1 directory with 2 updates by @dependabot[bot] in https://github.com/mudler/LocalAI/pull/11957\r\n* chore: :arrow_up: Update 0xShug0/audio.cpp to `fa5aaac9266a98c68f8a5c9fcd1ba6ff65875416` by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/11955\r\n* chore: :arrow_up: Update ikawrakow/ik_llama.cpp to `3e416d7f5a9d4cc3195e8171dbf891541ca59c6a` by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/11952\r\n* chore: :arrow_up: Update ggml-org/llama.cpp to `434ddbbc0e30522e897670681e503b797c12b7c1` by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/11953\r\n* chore: :arrow_up: Update vllm-project/vllm cu130 wheel to `0.29.0` by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/11954\r\n* chore: :arrow_up: Update 0xShug0/audio.cpp to `efb04233dab73aeee4b2912042a90e7b36329061` by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/11964\r\n* chore: :arrow_up: Update ikawrakow/ik_llama.cpp to `3bb386eb68ffee0a5dc7db21da0735d594929eeb` by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/11967\r\n* chore: :arrow_up: Update ggml-org/llama.cpp to `df03399b885831b2a1603b3abb0d8c156808e363` by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/11966\r\n* chore: :arrow_up: Update leejet/stable-diffusion.cpp to `b68d58624d227682eb4b95ef8bcf569cd1311eb5` by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/11965\r\n* chore(deps): bump securego/gosec from 2.27.1 to 2.29.0 by @dependabot[bot] in https://github.com/mudler/LocalAI/pull/11854\r\n* chore(deps): bump vllm from 0.28.0 to 0.29.0 in /backend/python/vllm by @dependabot[bot] in https://github.com/mudler/LocalAI/pull/11961\r\n* chore(deps): bump LocalAGI and localrecall to v0.6.5 by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/11985\r\n* chore: :arrow_up: Update leejet/stable-diffusion.cpp to `7f410a3793c5bba8eb198e962ce7a3d6095f9d89` by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/12012\r\n* chore: :arrow_up: Update CrispStrobe/CrispASR to `ab87fba65a4ade1ef30ccaa881458a0914aae557` by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/12011\r\n* chore: :arrow_up: Update 0xShug0/audio.cpp to `5ba81ac54fb071b835680973f8868546b4db372b` by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/12010\r\n* chore: :arrow_up: Update ggml-org/whisper.cpp to `1da4dc82fa7996d4edda05890dca65aeceaafd6d` by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/12009\r\n* chore: :arrow_up: Update OpenNMT/CTranslate2 ROCm wheel to `v4.8.2` by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/12004\r\n* chore: :arrow_up: Update ServeurpersoCom/omnivoice.cpp to `2ebbdcf281ce4b632e326011974da5a8b35c6b27` by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/12006\r\n* chore: :arrow_up: Update vllm-metal (darwin) to `v0.29.0` by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/12007\r\n* chore: :arrow_up: Update CrispStrobe/CrispASR to `a4db74cfd13c8674a7815608baaa505e673ce930` by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/12035\r\n* chore: :arrow_up: Update 0xShug0/audio.cpp to `ff1bcc4555ff99c4383329b0b21b52a18cc8b3cd` by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/12033\r\n* chore: :arrow_up: Update antirez/ds4 to `a04f46fa423e45712c8c7e430eff422479f314a3` by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/12013\r\n* chore: :arrow_up: Update mudler/vllm.cpp to `60990ee784101f74f6d1775575e9e89dfb26f73a` by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/12014\r\n* chore: :arrow_up: Update ggml-org/llama.cpp to `5f436dddb440a288ee5611d7d1eca564a6aca9f4` by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/12008\r\n* chore: :arrow_up: Update leejet/stable-diffusion.cpp to `42d6c0ab92fe6595776b28e3f7c8925db79b31f5` by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/12034\r\n* chore: :arrow_up: Update PrismML-Eng/llama.cpp to `d8f26eec76da6d09bb708bcba51ef64b8cd868a3` by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/12005\r\n* chore: :arrow_up: Update TheTom/llama-cpp-turboquant to `407f3237bfb3eeaff61546797de3d8c1a96be748` by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/12003\r\n* chore: :arrow_up: Update ikawrakow/ik_llama.cpp to `7434a014237e55b2e0d36edc2ace6acfb82fea1c` by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/12052\r\n* chore: :arrow_up: Update CrispStrobe/CrispASR to `97cea0919f1b5abfa2395f5791f88a05a0151223` by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/12054\r\n* chore: :arrow_up: Update 0xShug0/audio.cpp to `582a63ca8b6f76235f1b8636e734930218c78c05` by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/12056\r\n* chore: :arrow_up: Update mudler/vllm.cpp to `67c81102777296fdd506945860eb3030b9fad71a` by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/12053\r\n* chore: :arrow_up: Update antirez/ds4 to `9139e2ae58a41503968a500f36f75895c1ba63fc` by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/12062\r\n* chore: :arrow_up: Update PrismML-Eng/llama.cpp to `6ac5eb087009a762a3f61876f7dc152bc8600bf4` by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/12055\r\n* chore: :arrow_up: Update ggml-org/llama.cpp to `96ffdc41ceb055e1c2d3d96667ae6d9f0ccb710b` by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/12057\r\n* chore: :arrow_up: Update ServeurpersoCom/omnivoice.cpp to `3d341e55d365423476f8739d396b9a03d5277055` by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/12059\r\n* chore: :arrow_up: Update ggml-org/whisper.cpp to `1d549b3cecc2d98d76d4ddc2edca0d1512f5d7a0` by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/12060\r\n* chore: :arrow_up: Update leejet/stable-diffusion.cpp to `59c23bce0d82be3a922023ab811194f05b3e2faa` by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/12061\r\n* chore: :arrow_up: Update ServeurpersoCom/omnivoice.cpp to `3c30bdcfe6cc53121bd7a7b377bf8b55f526efa3` by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/12079\r\n* chore: :arrow_up: Update ikawrakow/ik_llama.cpp to `d85a1d7e91e5bc2f83a796542a8d5c436d32c99f` by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/12078\r\n* chore: :arrow_up: Update CrispStrobe/CrispASR to `89dfbc05d1466f6c7517986c6ff85179507431f8` by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/12077\r\n* chore: :arrow_up: Update 0xShug0/audio.cpp to `4af143229384fb6da3f373dc87de145ae954609b` by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/12073\r\n* chore: :arrow_up: Update ggml-org/whisper.cpp to `da54572229bcf64ba367d96c7ef15770376c4280` by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/12076\r\n* chore: :arrow_up: Update mudler/vllm.cpp to `b54db871e8478d0a3450132fa793d30cc336b321` by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/12074\r\n* chore: :arrow_up: Update ggml-org/llama.cpp to `38a5b42d9a3e82e0a586bcd1caed121f36c87a73` by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/12075\r\n* chore: :arrow_up: Update PrismML-Eng/llama.cpp to `7dffb158de30ebb8ef9d64f33c6b0b2d7c1e6313` by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/12072\r\n* chore: :arrow_up: Update antirez/ds4 to `8db1d1d155cb0400a86a86b9c62d0defb3a6148b` by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/12093\r\n* chore: :arrow_up: Update CrispStrobe/CrispASR to `ba3499e7c7f6013a73738cad530b252d59675f49` by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/12092\r\n### Other Changes\r\n* docs(blog): add the 4.9 release post and its demo clips by @localai-bot in https://github.com/mudler/LocalAI/pull/11629\r\n* docs: :arrow_up: update docs version mudler/LocalAI by @localai-bot in https://github.com/mudler/LocalAI/pull/11643\r\n* chore: :arrow_up: Update ggml-org/llama.cpp to `d59d455fd8ea09e5a2e87ce2a9d668267ffb5ccd` by @localai-bot in https://github.com/mudler/LocalAI/pull/11618\r\n* chore: bump inference defaults from unsloth by @github-actions[bot] in https://github.com/mudler/LocalAI/pull/11654\r\n* chore: :arrow_up: Update 0xShug0/audio.cpp to `43001a7e0f452d80f4588e613f13332940dd4d3a` by @localai-bot in https://github.com/mudler/LocalAI/pull/11649\r\n* chore(model-gallery): :arrow_up: update checksum by @localai-bot in https://github.com/mudler/LocalAI/pull/11645\r\n* feat(xio): make copy buffer size configurable by @localai-bot in https://github.com/mudler/LocalAI/pull/11660\r\n* fix(progress): stop status updates throttling downloads by @localai-bot in https://github.com/mudler/LocalAI/pull/11661\r\n* fix(distributed): keep staging operations stable by @localai-bot in https://github.com/mudler/LocalAI/pull/11663\r\n* feat(ui): edit scheduling rules in place by @localai-bot in https://github.com/mudler/LocalAI/pull/11667\r\n* chore(model-gallery): :arrow_up: update checksum by @localai-bot in https://github.com/mudler/LocalAI/pull/11665\r\n* fix(distributed): keep model replicas config-consistent by @localai-bot in https://github.com/mudler/LocalAI/pull/11664\r\n* chore: :arrow_up: Update CrispStrobe/CrispASR to `74bb374a8cc74284348d76a0a6e944180fbe6b07` by @localai-bot in https://github.com/mudler/LocalAI/pull/11650\r\n* chore: :arrow_up: Update 0xShug0/audio.cpp to `4d383be1bff107e823ffc19120dcb6c78d493c0f` by @localai-bot in https://github.com/mudler/LocalAI/pull/11666\r\n* chore: :arrow_up: Update ggml-org/whisper.cpp to `233fe1fc9b48a09e361d3594520838ca266537fe` by @localai-bot in https://github.com/mudler/LocalAI/pull/11648\r\n* chore(model-gallery): :arrow_up: update checksum by @localai-bot in https://github.com/mudler/LocalAI/pull/11690\r\n* chore: :arrow_up: Update antirez/ds4 to `c1d4597a80e300b803dc642519718f2c999589da` by @localai-bot in https://github.com/mudler/LocalAI/pull/11685\r\n* chore(website): refresh the counters by @localai-bot in https://github.com/mudler/LocalAI/pull/11697\r\n* fix(ci): bound Discord release summaries by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/11695\r\n* fix(oci): resume interrupted layer downloads by @localai-bot in https://github.com/mudler/LocalAI/pull/11688\r\n* chore: :arrow_up: Update CrispStrobe/CrispASR to `ae4474dd8306384a0e697183d863dfc52e69a2fb` by @localai-bot in https://github.com/mudler/LocalAI/pull/11684\r\n* chore: :arrow_up: Update 0xShug0/audio.cpp to `288a2712316470847a730e55db9ac9e5062a2b03` by @localai-bot in https://github.com/mudler/LocalAI/pull/11683\r\n* feat(swagger): update swagger by @localai-bot in https://github.com/mudler/LocalAI/pull/11682\r\n* chore(model gallery): :robot: add 1 new models via gallery agent by @localai-bot in https://github.com/mudler/LocalAI/pull/11692\r\n* chore(model-gallery): :arrow_up: update checksum by @localai-bot in https://github.com/mudler/LocalAI/pull/11707\r\n* chore: :arrow_up: Update ikawrakow/ik_llama.cpp to `0ed847d3140baead542abe3e5e6fe841013e7340` by @localai-bot in https://github.com/mudler/LocalAI/pull/11708\r\n* chore: :arrow_up: Update 0xShug0/audio.cpp to `d25ffac094a9d5a240940b4955ea79ad9b7b4c78` by @localai-bot in https://github.com/mudler/LocalAI/pull/11710\r\n* chore: :arrow_up: Update ggml-org/llama.cpp to `f280b26983ad0fdb705a0d9ebf0503e76f2899b0` by @localai-bot in https://github.com/mudler/LocalAI/pull/11646\r\n* feat(gallery): add Ornith 1.5 397B variants by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/11716\r\n* chore: :arrow_up: Update ikawrakow/ik_llama.cpp to `08b500b958a3f1102e6500e5c425e65517d6fb7e` by @localai-bot in https://github.com/mudler/LocalAI/pull/11726\r\n* chore: :arrow_up: Update 0xShug0/audio.cpp to `c79e58899bf13db4d78fd06372da23cc13f55b28` by @localai-bot in https://github.com/mudler/LocalAI/pull/11722\r\n* chore: :arrow_up: Update ggml-org/whisper.cpp to `978113305b2ead22249b881deafa131dc8884911` by @localai-bot in https://github.com/mudler/LocalAI/pull/11711\r\n* chore(model-gallery): :arrow_up: update checksum by @localai-bot in https://github.com/mudler/LocalAI/pull/11730\r\n* Fix flaky \"tests-apple\" CI job in modeladmin test suite by @mudler with @Copilot in https://github.com/mudler/LocalAI/pull/11717\r\n* chore: :arrow_up: Update ggml-org/llama.cpp to `eab8ee41f889ef7823af517e8098fb8a9b3cf601` by @localai-bot in https://github.com/mudler/LocalAI/pull/11724\r\n* chore: :arrow_up: Update leejet/stable-diffusion.cpp to `50d640568388f876b0d63ee6ddb6bc86d997ec64` by @localai-bot in https://github.com/mudler/LocalAI/pull/11725\r\n* chore: :arrow_up: Update mudler/vllm.cpp to `6738e0b4639199f3ff0998815e4d32bfa7fe5be2` by @localai-bot in https://github.com/mudler/LocalAI/pull/11647\r\n* chore(model-gallery): :arrow_up: update checksum by @localai-bot in https://github.com/mudler/LocalAI/pull/11742\r\n* chore: :arrow_up: Update PrismML-Eng/llama.cpp to `312bb2a93ea2bf798333fa859614fbf913ecb9e2` by @localai-bot in https://github.com/mudler/LocalAI/pull/11740\r\n* chore: :arrow_up: Update vllm-project/vllm cu130 wheel to `0.28.0` by @localai-bot in https://github.com/mudler/LocalAI/pull/11741\r\n* chore: :arrow_up: Update ikawrakow/ik_llama.cpp to `ef40550042973817ac391ca95a2ff041f512257b` by @localai-bot in https://github.com/mudler/LocalAI/pull/11743\r\n* chore: :arrow_up: Update ggml-org/llama.cpp to `925e1179947ea0c0ebfb0032df18af3a729822be` by @localai-bot in https://github.com/mudler/LocalAI/pull/11744\r\n* chore: :arrow_up: Update 0xShug0/audio.cpp to `db21cbdd60f3d2ff62114bc863781ff8073ac39b` by @localai-bot in https://github.com/mudler/LocalAI/pull/11746\r\n* chore: :arrow_up: Update mudler/depth-anything.cpp to `739992d10bf9472c46dcd4622b14d2b20766c58d` by @localai-bot in https://github.com/mudler/LocalAI/pull/11758\r\n* chore: :arrow_up: Update ikawrakow/ik_llama.cpp to `7cff686d3732bfef5ce18bc4a6115fbceda29c14` by @localai-bot in https://github.com/mudler/LocalAI/pull/11757\r\n* chore: :arrow_up: Update leejet/stable-diffusion.cpp to `be0e34480dada95f8ce9a021bbb95c5de85d67c7` by @localai-bot in https://github.com/mudler/LocalAI/pull/11760\r\n* chore: :arrow_up: Update 0xShug0/audio.cpp to `17751c0e8c48a3d56dcf05eeb60464409ecc69ce` by @localai-bot in https://github.com/mudler/LocalAI/pull/11759\r\n* chore: :arrow_up: Update ggml-org/llama.cpp to `e70802a01f03f0ed31a26338a5664796f3824371` by @localai-bot in https://github.com/mudler/LocalAI/pull/11755\r\n* fix(ui): size model fit against the cluster and move node labels into the selector by @localai-bot in https://github.com/mudler/LocalAI/pull/11765\r\n* chore: :arrow_up: Update ikawrakow/ik_llama.cpp to `15dddc60b3fc937a9e2a210359ecce392ccdf446` by @localai-bot in https://github.com/mudler/LocalAI/pull/11767\r\n* chore: :arrow_up: Update antirez/ds4 to `8db89fe083ae4d17c9a2428ccd29803d3ae8f577` by @localai-bot in https://github.com/mudler/LocalAI/pull/11768\r\n* chore: :arrow_up: Update ggml-org/llama.cpp to `d7bd3bfcad3e29c7e49fd26f38c79ee3e9a3fd6b` by @localai-bot in https://github.com/mudler/LocalAI/pull/11770\r\n* feat(distributed): key scheduling rules by a model alias by @localai-bot in https://github.com/mudler/LocalAI/pull/11771\r\n* chore: bump inference defaults from unsloth by @github-actions[bot] in https://github.com/mudler/LocalAI/pull/11773\r\n* chore: :arrow_up: Update 0xShug0/audio.cpp to `89a0e9803380880305e9e1b83c93614f9df2c893` by @localai-bot in https://github.com/mudler/LocalAI/pull/11769\r\n* chore: :arrow_up: Update mudler/vllm.cpp to `150b37852c123f7855fb219b37347572ca9427e7` by @localai-bot in https://github.com/mudler/LocalAI/pull/11745\r\n* feat(stablediffusion-ggml): build a ROCm variant by @localai-bot in https://github.com/mudler/LocalAI/pull/11774\r\n* chore(model gallery): :robot: add 1 new models via gallery agent by @localai-bot in https://github.com/mudler/LocalAI/pull/11777\r\n* refactor(worker): iterate node label pairs with SplitSeq by @ginavalent in https://github.com/mudler/LocalAI/pull/11806\r\n* fix(ci): stop the e2e teardown from failing a green suite by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/11816\r\n* chore(model-gallery): :arrow_up: update checksum by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/11831\r\n* fix(ci): unbreak the e2e build and the darwin vllm-metal pin by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/11849\r\n* chore(model-gallery): :arrow_up: update checksum by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/11868\r\n* chore(model gallery): :robot: add 1 new models via gallery agent by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/11880\r\n* feat(swagger): update swagger by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/11914\r\n* chore(website): refresh the counters by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/11902\r\n* chore(model-gallery): :arrow_up: update checksum by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/11939\r\n* chore(model-gallery): :arrow_up: update checksum by @localai-org-maint-bot in https://github.com/mudler/LocalAI/pull/11968\r\n\r\n## New Contributors\r\n* @localai-org-bot made their first contribution in https://github.com/mudler/LocalAI/pull/11631\r\n* @DanielSwift1992 made their first contribution in https://github.com/mudler/LocalAI/pull/11686\r\n* @leilei3167 made their first contribution in https://github.com/mudler/LocalAI/pull/11715\r\n* @sz-po made their first contribution in https://github.com/mudler/LocalAI/pull/11736\r\n* @blackd made their first contribution in https://github.com/mudler/LocalAI/pull/11750\r\n* @ginavalent made their first contribution in https://github.com/mudler/LocalAI/pull/11806\r\n* @FiloSpaTeam made their first contribution in https://github.com/mudler/LocalAI/pull/11802\r\n* @aimansour made their first contribution in https://github.com/mudler/LocalAI/pull/11700\r\n* @simpleqt made their first contribution in https://github.com/mudler/LocalAI/pull/11885\r\n* @alex-mazzariol made their first contribution in https://github.com/mudler/LocalAI/pull/11749\r\n* @seven7763 made their first contribution in https://github.com/mudler/LocalAI/pull/11310\r\n* @QiuLsG made their first contribution in https://github.com/mudler/LocalAI/pull/11883\r\n* @SomSamantray made their first contribution in https://github.com/mudler/LocalAI/pull/11652\r\n* @devv-shayan made their first contribution in https://github.com/mudler/LocalAI/pull/11962\r\n* @Nathan-Roll1 made their first contribution in https://github.com/mudler/LocalAI/pull/11998\r\n* @quartzjer made their first contribution in https://github.com/mudler/LocalAI/pull/12045\r\n\r\n**Full Changelog**: https://github.com/mudler/LocalAI/compare/v4.9.0...v4.10.0\r\n"
+ },
+ {
+ "repo": "janhq/jan",
+ "tag_name": "v0.8.4",
+ "name": "0.8.4",
+ "published_at": "2026-07-23T06:29:15Z",
+ "html_url": "https://github.com/janhq/jan/releases/tag/v0.8.4",
+ "prerelease": false,
+ "body": "## Migration\n\n**Settings and credentials now live in a backend-managed store.**\n\nProvider settings and credentials previously persisted in the webview's `localStorage`. In 0.8.4, Jan reads and writes them from a backend-managed store instead (secrets go to the OS keyring). Your existing settings and API keys are copied over automatically on first launch - no manual action is needed, and from this version on all new changes are saved to the backend store.\n\nYour old `localStorage` data is **not deleted** - it's kept as a snapshot so you can downgrade to a pre-0.8.4 build and still have your previous settings. Any changes you make in 0.8.4+ live only in the backend store, so a downgrade sees the older snapshot, not your latest settings.\n\n---\n\n## What's Changed\n* chore(flatpak): add 0.8.3 release notes to appdata by @qnixsynapse in https://github.com/janhq/jan/pull/8343\n* docs: update changelogs for v0.8.3 and update documentation by @qnixsynapse in https://github.com/janhq/jan/pull/8336\n* Merge release/v0.8.3 to main by @qnixsynapse in https://github.com/janhq/jan/pull/8344\n* i18n(ja): update Japanese translations (#8264) by @mahirhir in https://github.com/janhq/jan/pull/8348\n* i18n(ja): complete common.json translations and fix terminology (#8264) by @mahirhir in https://github.com/janhq/jan/pull/8349\n* i18n(ja): complete settings.json translations (#8264) by @mahirhir in https://github.com/janhq/jan/pull/8352\n* fix(markdown): force code blocks LTR in RTL language context by @qnixsynapse in https://github.com/janhq/jan/pull/8350\n* feat(chat): add toggle for folding interim text into reasoning trace by @qnixsynapse in https://github.com/janhq/jan/pull/8363\n* fix(rag): repair embeddings, stop chat-model reloads, and parse more PDFs by @qnixsynapse in https://github.com/janhq/jan/pull/8360\n* ci: close issues inactive for over 180 days by @qnixsynapse in https://github.com/janhq/jan/pull/8366\n* ci: grant actions:write to stale workflow for state persistence by @qnixsynapse in https://github.com/janhq/jan/pull/8377\n* fix(chat): stop persisting detached assistant replies in branched threads by @qnixsynapse in https://github.com/janhq/jan/pull/8376\n* feat(settings): move settings/credentials off localStorage to backend store by @qnixsynapse in https://github.com/janhq/jan/pull/8388\n* fix(params): coerce numeric assistant params from string to number by @qnixsynapse in https://github.com/janhq/jan/pull/8386\n* fix: use snake case for ADL enum in get_gpu_usage by @nkulshre-amd in https://github.com/janhq/jan/pull/8390\n* fix(chat): stop MCP tool-approval popup from hanging; move transient approval state off disk by @qnixsynapse in https://github.com/janhq/jan/pull/8391\n* API key rotation for Gemini by @jjchen01 in https://github.com/janhq/jan/pull/8408\n* fix(security): bump handlebars to 4.7.9 to clear RCE advisory by @qnixsynapse in https://github.com/janhq/jan/pull/8395\n* fix(chat): ignore custom SSE event types in OpenAI-compatible streams by @qnixsynapse in https://github.com/janhq/jan/pull/8392\n* feat: OpenAI-compatible translating gateway + Responses API + unified reasoning by @qnixsynapse in https://github.com/janhq/jan/pull/8423\n* fix: Stabilize chat prefix caching, restore cloud providers, unify logging (v0.8.4) by @qnixsynapse in https://github.com/janhq/jan/pull/8425\n* fix(mcp): guard against duplicate serve() on streamable-http MCP servers by @thinhlpg in https://github.com/janhq/jan/pull/8435\n* fix(providers): send anthropic-version on model-list fetch by @qnixsynapse in https://github.com/janhq/jan/pull/8433\n* fix(desktop): macOS hide-on-close; hide-to-tray when Local API Server runs by @qnixsynapse in https://github.com/janhq/jan/pull/8436\n* feat(extensions): bundle built-in extensions into the app binary and update tests by @qnixsynapse in https://github.com/janhq/jan/pull/8437\n* feat(llamacpp): per-model chat template kwargs by @qnixsynapse in https://github.com/janhq/jan/pull/8441\n* feat: token counter for remote/MLX providers + chat & inference fixes by @qnixsynapse in https://github.com/janhq/jan/pull/8456\n* fix(chat): preserve AbortError identity when stopping mid model-load by @thinhlpg in https://github.com/janhq/jan/pull/8443\n* fix(chat): stop hiding the llama.cpp token counter during chat by @qnixsynapse in https://github.com/janhq/jan/pull/8461\n* feat(web-search): native web_search/web_fetch via tauri-plugin-websearch by @qnixsynapse in https://github.com/janhq/jan/pull/8464\n* fix: persist chain-of-thought duration in thread metadata by @qnixsynapse in https://github.com/janhq/jan/pull/8474\n* refactor: centralize llamacpp extension lookup with scoped-name fallback by @qnixsynapse in https://github.com/janhq/jan/pull/8487\n\n## New Contributors\n* @mahirhir made their first contribution in https://github.com/janhq/jan/pull/8348\n* @nkulshre-amd made their first contribution in https://github.com/janhq/jan/pull/8390\n\n**Full Changelog**: https://github.com/janhq/jan/compare/v0.8.3...v0.8.4\n"
+ },
+ {
+ "repo": "mozilla-ai/llamafile",
+ "tag_name": "0.10.6",
+ "name": "llamafile v0.10.6",
+ "published_at": "2026-09-15T16:02:33Z",
+ "html_url": "https://github.com/mozilla-ai/llamafile/releases/tag/0.10.6",
+ "prerelease": false,
+ "body": "## What's Changed\r\n\r\n* Update llama.cpp to b10441 by @aittalam in https://github.com/mozilla-ai/llamafile/pull/1043\r\n* llamafile_open_zip: validate the zip entry window against the actual file size by @professor-moody in https://github.com/mozilla-ai/llamafile/pull/1039\r\n* fix(zip): guard extra-field header read before bounds are known by @shoemoney in https://github.com/mozilla-ai/llamafile/pull/1044\r\n* fix(build): add missing link recipes for llama-bench, quantize, imatrix and perplexity by @dexhunter in https://github.com/mozilla-ai/llamafile/pull/1034\r\n* fix(build): compile the ggml objects with -DNDEBUG by @aittalam in https://github.com/mozilla-ai/llamafile/pull/1050\r\n* Enable -DGGML_CUDA_USE_GRAPHS / -DGGML_HIP_GRAPHS by @aittalam in https://github.com/mozilla-ai/llamafile/pull/1051\r\n* fix(ggml): use `ggml_time_init()` in `ggml_time_ms` and `ggml_time_us` so that `timer_freq` is initialised by @aittalam in https://github.com/mozilla-ai/llamafile/pull/1054\r\n* Vulkan toolchain by @aittalam in https://github.com/mozilla-ai/llamafile/pull/1055\r\n\r\n## New Contributors\r\n* @professor-moody made their first contribution in https://github.com/mozilla-ai/llamafile/pull/1039\r\n* @shoemoney made their first contribution in https://github.com/mozilla-ai/llamafile/pull/1044\r\n* @dexhunter made their first contribution in https://github.com/mozilla-ai/llamafile/pull/1034\r\n\r\n**Full Changelog**: https://github.com/mozilla-ai/llamafile/compare/0.10.5...0.10.6"
+ },
+ {
+ "repo": "mlc-ai/mlc-llm",
+ "error": "HTTP Error 404: Not Found"
+ },
+ {
+ "repo": "mostlygeek/llama-swap",
+ "tag_name": "v256",
+ "name": "v256",
+ "published_at": "2026-09-17T07:14:28Z",
+ "html_url": "https://github.com/mostlygeek/llama-swap/releases/tag/v256",
+ "prerelease": false,
+ "body": "Maintainer note: This is the v256 Big-F-Release. I've been very busy and many small improvements have been squatting in main waiting to be unleashed. A few highlights: \r\n\r\n## Playground Chat UI improvements \r\n\r\n- tightened up the margins and cleaned up the design to be more mobile friendly\r\n- more timings and stats than you can shake a stick at\r\n- improved system prompt and added more hyper param controls (do who uses these anymore?) \r\n\r\n<img width=\"359\" height=\"625\" alt=\"image\" src=\"https://github.com/user-attachments/assets/06d526f3-688b-4a35-8479-17d047dd4055\" />\r\n<img width=\"358\" height=\"616\" alt=\"image\" src=\"https://github.com/user-attachments/assets/4e5a5c8e-3aa4-4aa9-bae7-9a97c339939b\" />\r\n\r\nThanks to contributors: \r\n\r\n - JoC - Started the model idle-time window after models become ready or restart [PR #1095](https://github.com/mostlygeek/llama-swap/pull/1095).\r\n - Kris - Added live generation statistics to Chat [PR #1099](https://github.com/mostlygeek/llama-swap/pull/1099).\r\n\r\n## Changelog\r\n* 6701d0d9d9dce7d698bcb6738ccacb75b0c27731 changelog: v256\r\n* 2409272b824156080a6af3b4997f147d5a337db5 CHANGELOG.md,docs: fix pull request link\r\n* b98a8ff76c89e88b338fc3754da3fc4b4e837126 changelog: add scripts and automation to generate changelog entries (#1125)\r\n* 40b36d9571eb1e865f78db34a59b4a676d4854a6 ui: make the playground chat responsive and lighter (#1120)\r\n* 0e1f7977bf5d607ce356873ac29d3751a93c78e5 internal/server: reduce zstd pool memory consumption (#1123)\r\n* aecf92e2e89b31eacdd1b778d83918a2445aa7db internal/store: split repository interfaces from the sqlite backend (#1122)\r\n* 21bc145a6fa6198c4d856fbb0830845c472b0ae4 process: start TTL idle window when model becomes ready (#1095)\r\n* 41ec321b6216d838488b2a7d936274ed227c0c5e internal/server: add global concurrency semaphore (#1110)\r\n* e1526c848060f702e402d75c6aa50a069f9dfd9c cmd/vllm-wrapper: use default timeouts\r\n* 8fa85899e424d47b81fa60aff08b238a793e2e2b internal/server: add tabbyapi usage data extraction (#1104)\r\n* 168632699383d440a5282b9da3612e741dd6cb7a ui: show chat stats in Help agent\r\n* acb77599647e2f508d681e23cefc898bae3258af ui: show live generation stats in Chat (#1099)\r\n* 6b181151549d80c49fedcaf408392d2309fa0edd ui: keep Work status bar stable while streaming (#1101)\r\n\r\n"
+ },
+ {
+ "repo": "LostRuins/koboldcpp",
+ "tag_name": "v1.121",
+ "name": "koboldcpp-1.121",
+ "published_at": "2026-09-15T15:35:48Z",
+ "html_url": "https://github.com/LostRuins/koboldcpp/releases/tag/v1.121",
+ "prerelease": false,
+ "body": "# koboldcpp-1.121\r\n\r\nhttps://github.com/user-attachments/assets/b0663a91-ba3f-49c0-b7b3-1090aca62445\r\n\r\n_Music generated by @Dampfinchen, clips for music video created entirely in KoboldCpp_\r\n\r\n- **NEW: Media reference support added for Minimax H3!** - You will now be able to attach audio clips and multiple reference images to generate minimax H3 videos. The music video attached to this video was created entirely in KoboldCpp (in segments of 7s each)\r\n- Fixed a race condition during streaming.\r\n- Fixed LTX Audio conditioning bug\r\n- Added support for video LoRAs (thanks @wbruna)\r\n- Swapped to CUDA 11.4 for OldPC release binaries (tested and verified by @henk717)\r\n- Fixed a bug in grammar memoization (thanks @Reithan)\r\n- Fixed tool calling support for Kimi models and Deepseek V4 Flash models\r\n- Handle invalid numeric API parameters (thanks @yhz5613813)\r\n- Fixed WebSearch UTF-8 handling (thanks @Erquint)\r\n- `--analyze` feature can now analyze `.safetensor` file metadata in addition to `.gguf` files.\r\n- Hide cache slots in GUI launcher if smartcache is off\r\n- Made help menu in GUI launcher more accessible\r\n- Added a flag `--ffncpu` which behaves like llama.cpp `--n-cpu-ffn`, an alternative way to partial offload layers\r\n- Optimized Adaptive-P sampler\r\n- Prevent segfault on model load failure\r\n- Reworked GUI launcher's SaveDataFile selection UI\r\n- Add support for xhigh reasoning effort\r\n- Improved huggingface model downloader\r\n- Jinja tools requests with tools in the payload have a lower default temperature\r\n- **NEW: Reworked MusicUI (Music Generation UI)** for a more intuitive user experience.\r\n- Multiple Stable-UI (SDUI) GUI Improvements:\r\n - **NEW: Runtime LoRA selector in included stable-ui (http://localhost:5001/sdui)** - Thanks to @wbruna, you can now easily view, add and remove image generation LoRAs from your selected LoRA directory at runtime in SDUI with a nice GUI selector.\r\n - Fixed SDUI Image carousel selection\r\n - Image Generation Recovery - Enable \"Poor connection mode\" in SDUI to allow recovering generated images even if you disconnect or accidentally close your browser.\r\n - Improved cloudflare tunnel stability in SDUI - Sends a keepalive to help prevent tunnel collapse during long requests (e.g. video generation).\r\n - Added box for sending extra custom fields in SDUI, increased max video frames to 240 (default) or 480 (extended toggle), show elapsed time\r\n - Set SDUI dynamic LoRA default strength to 1.\r\n - Prevent viewing generation progress status if genkey is incorrect\r\n- Updated Kobold Lite, multiple fixes and improvements\r\n - Attached vision images now render in correct turns.\r\n - Improved autoscrolling behavior\r\n - Fixed `<t2i>` tag handling in thinking blocks.\r\n - Added mini toggle buttons for enabling TTS and voice recognition if detected.\r\n- Merged fixes, new model support, and improvements from upstream\r\n\r\nIn addition to the above our docker image now has KCPP_LORA support for use with cloud providers, use this variable along with direct links to the lora's you wish to download, they can be separated by a comma.\r\n\r\nDownload and run the **koboldcpp.exe** (Windows) or **koboldcpp-linux-x64** (Linux), which is a one-file pyinstaller for NVIDIA GPU users. \r\nIf you have an older CPU or older NVIDIA GPU and **koboldcpp** does not work, try **oldpc** version instead (Cuda11 + AVX1). \r\nIf you don't have an NVIDIA GPU, or do not need CUDA, you can use the **nocuda** version which is smaller. \r\nIf you're using AMD, we recommend trying the **Vulkan** option in the nocuda build first, for best support. Alternatively, you can [download our rolling ROCm binary here](https://koboldai.org/cpplinuxrocm) if you use Linux.\r\nIf you're on a modern MacOS (M-Series) you can use the **koboldcpp-mac-arm64** MacOS binary. \r\n[Click here for .gguf conversion and quantization tools](https://kcpptools.concedo.workers.dev/)\r\nNewer **rolling experimental builds** can be [found here](https://github.com/LostRuins/koboldcpp/releases/tag/rolling), these are auto-updated and may be unstable.\r\n\r\nRun it from the command line with the desired launch parameters (see `--help`), or manually select the model in the GUI.\r\nand then once loaded, you can connect like this (or use the full koboldai client):\r\n**http://localhost:5001**\r\n\r\nFor more information, be sure to run the program from command line with the `--help` flag. You can also refer to the [readme](https://github.com/LostRuins/koboldcpp/blob/concedo/README.md) and the [wiki](https://github.com/LostRuins/koboldcpp/wiki)."
+ },
+ {
+ "repo": "turboderp-org/exllamav3",
+ "tag_name": "v1.5.0",
+ "name": "1.5.0",
+ "published_at": "2026-09-13T14:00:46Z",
+ "html_url": "https://github.com/turboderp-org/exllamav3/releases/tag/v1.5.0",
+ "prerelease": false,
+ "body": "- Fix NemotronHForCausalLM (support Nemotron-3-Super)\r\n- Faster quantization\r\n- Experimental zero-copy pinned arena mode for CPU offload (Linux only for now)\r\n- Faster MoE prefill and decode (see below)\r\n- Improved speculative decoding for MoE models\r\n- Faster dense model prefill on some architectures\r\n- Remove some sources of nondeterminism\r\n- Bugfixes and cleanup\r\n\r\n| Test case | 4k prefill, t/s | decode, t/s, S=1 |\r\n|:-------------------------------------------------------------------------------------------------------------------|-------------------------:|---------------------------:|\r\n| Qwen3.8-Flash-Next, 3.05bpw<br>**Pro 6000**, ngram_ram | 5788 \u2192 7327 (**+27%**) | 101.04 \u2192 115.27 (**+14%**) |\r\n| Qwen3.8-Flash-Next, 3.05bpw<br>**Pro 6000 (gen5 x16) + TR 7960X**, 80% CPU offload, pinned arena, ngram_ram | 2801 \u2192 4323 (**+54%**) | 55.66 \u2192 66.65 (**+20%**) |\r\n| Qwen3.8-Flash-Next, 3.05bpw<br>**5090 (gen5 x8) + TR 7960X**, 80% CPU offload, pinned arena, ngram_ram | 2615 \u2192 3256 (**+25%**) | 53.42 \u2192 63.27 (**+18%**) |\r\n| Qwen3.8-Flash-Next, 3.05bpw<br>**4090 (gen4 x4) + TR 7960X**, 80% CPU offload, pinned arena, ngram_ram | 1165 \u2192 1204 (**+3%**) | 56.07 \u2192 59.52 (**+6%**) |\r\n| Qwen3.8-Flash-Next, 3.05bpw<br>**3090Ti+3090 (gen4 x4) + AVX2, 8 cores**, 20% CPU offload, pinned arena, ngram_ram | 1572 \u2192 1841 (**+17%**) | 58.40 \u2192 62.77 (**+7%**) |\r\n| Gemma4-26B-A3B, 3.10bpw<br>**Pro 6000** | 15626 \u2192 18679 (**+20%**) | 185.45 \u2192 202.32 (**+9%**) |\r\n| Gemma4-26B-A3B, 3.10bpw<br>**5090** | 12066 \u2192 14695 (**+22%**) | 180.77 \u2192 199.55 (**+10%**) |\r\n| Gemma4-26B-A3B, 3.10bpw<br>**4090** | 8779 \u2192 10605 (**+21%**) | 172.92 \u2192 181.72 (**+5%**) |\r\n| Gemma4-26B-A3B, 3.10bpw<br>**3090** | 4742 \u2192 5980 (**+26%**) | 129.77 \u2192 134.82 (**+4%**) |\r\n| GLM5.3-Flash, 2.05bpw<br>**Pro 6000** | 3446 \u2192 3938 (**+14%**) | 90.18 \u2192 93.65 (**+4%**) | \r\n| GLM5.3-Flash, 3.05bpw<br>**5090 (gen5 x8) + TR 7960X**, 80% CPU offload, pinned arena | 809 \u2192 1056 (**+30%**) | 30.53 \u2192 30.61 (**+0%**) | \r\n| GLM5.3-Flash, 3.05bpw<br>**5090 (gen5 x8) + AVX2, 8 cores**, 80% CPU offload, pinned arena | 841 \u2192 1068 (**+27%**) | 9.78 \u2192 10.30 (**+5%**) |\r\n| Qwen3.8-27B, 4.00bpw<br>**Pro 6000** | 5549 \u2192 5547 (**-0%**) | 76.88 \u2192 76.53 (**-0%**) |\r\n| Qwen3.8-27B, 4.00bpw<br>**5090** | 3766 \u2192 4651 (**+24%**) | 78.54 \u2192 78.58 (**+0%**) |\r\n| Qwen3.8-27B, 4.00bpw<br>**4090** | 2636 \u2192 2819 (**+7%**) | 54.60 \u2192 54.61 (**+0%**) |\r\n| Qwen3.8-27B, 4.00bpw<br>**3090Ti** | 1442 \u2192 1552 (**+8%**) | 48.55 \u2192 48.66 (**+0%**) |\r\n| gpt-oss-20b, 3.00bpw<br>**Pro 6000** | 22229 \u2192 22249 (**+0%**) | 263.77 \u2192 303.27 (**+15%**) |\r\n| gpt-oss-20b, 3.00bpw<br>**5090** | 16579 \u2192 17936 (**+8%**) | 255.68 \u2192 287.21 (**+12%**) |\r\n| gpt-oss-20b, 3.00bpw<br>**4090** | 12520 \u2192 12703 (**+1%**) | 222.59 \u2192 242.71 (**+9%**) |\r\n| gpt-oss-20b, 3.00bpw<br>**3090Ti** | 6455 \u2192 7087 (**+10%**) | 158.09 \u2192 177.93 (**+13%**) |\r\n\r\n**Full Changelog**: https://github.com/turboderp-org/exllamav3/compare/v1.4.9...v1.5.0"
+ },
+ {
+ "repo": "kvcache-ai/ktransformers",
+ "tag_name": "v0.7.1",
+ "name": "KTransformers v0.7.1",
+ "published_at": "2026-09-15T10:17:55Z",
+ "html_url": "https://github.com/kvcache-ai/ktransformers/releases/tag/v0.7.1",
+ "prerelease": false,
+ "body": "# KTransformers v0.7.1: Qwen VLM and Kimi LoRA Fine-Tuning\r\n\r\nKTransformers v0.7.1 brings together **Qwen VLM** and **Kimi K2.5 / K2.6** LoRA fine-tuning through LLaMA-Factory. CPU\u2013GPU heterogeneous execution combines host memory with GPU acceleration for multimodal adaptation and personalized dialogue.\r\n\r\n## \ud83d\ude80 Core Highlights\r\n\r\n- **Qwen VLM MoE fine-tuning**: BF16 image\u2013text LoRA for Qwen3-VL-30B-A3B-Instruct and Qwen3.5-35B-A3B, covering vision, language, and routed-expert modules. Contributed by **Bowen Wu (@Illumination111)** in [#2156](https://github.com/kvcache-ai/ktransformers/pull/2156).\r\n- **Native RAWINT4 Kimi fine-tuning**: text LoRA for Kimi K2.5 / K2.6 using the original packed expert weights, avoiding full-model BF16 expansion.\r\n\r\n## \ud83d\udce6 Installation\r\n\r\nFollow the model-specific guides for the complete dependency setup, data preparation, and training commands:\r\n\r\n- [Qwen VLM LoRA Fine-Tuning Guide](https://github.com/kvcache-ai/ktransformers/blob/main/doc/en/SFT/KTransformers-VLM-LoRA-Fine-Tuning-Guide.md)\r\n- [Kimi LoRA Training and Inference Guide](https://github.com/kvcache-ai/ktransformers/blob/main/.github/release/examples/kimi-k25/README.md)\r\n\r\n## \ud83c\udf1f Contributors\r\n\r\nThanks to @JimmyPeilinLi, @Illumination111, @yyj6666667, and everyone who contributed code, documentation, and testing.\r\n\r\nSpecial thanks to @05yuki for valuable insights into MXFP4 kernels and performance optimization.\r\n"
+ },
+ {
+ "repo": "NVIDIA/TensorRT-LLM",
+ "tag_name": "v1.2.1",
+ "name": "v1.2.1",
+ "published_at": "2026-04-20T11:51:33Z",
+ "html_url": "https://github.com/NVIDIA/TensorRT-LLM/releases/tag/v1.2.1",
+ "prerelease": false,
+ "body": "## Highlights\r\n\r\n- **Fixed Issue**\r\n - Fixed an issue that caused KV cache corruption (#12770)\r\n\r\n- **Infrastructure Changes**\r\n - Upgraded xgrammar and flashinfer (#12811)\r\n"
+ }
+ ]
+}
diff --git a/docs/research/local-ai-harnesses/shortlist.json b/docs/research/local-ai-harnesses/shortlist.json
new file mode 100644
index 000000000..579c2d78d
--- /dev/null
+++ b/docs/research/local-ai-harnesses/shortlist.json
@@ -0,0 +1,125 @@
+{
+ "status": "three_harnesses_implemented_direct_engines_prioritized",
+ "research_date": "2026-09-19",
+ "timezone": "America/New_York",
+ "confirmed_scope": {
+ "modalities": [
+ "language_models"
+ ],
+ "hardware_priority": "current_apple_silicon_mac",
+ "machine": {
+ "chip": "Apple M2 Max",
+ "memory_gib": 64,
+ "macos": "26.6.2"
+ }
+ },
+ "core": [
+ {
+ "id": "ollama",
+ "name": "Ollama",
+ "source": "https://github.com/ollama/ollama"
+ },
+ {
+ "id": "llamacpp",
+ "name": "llama.cpp",
+ "source": "https://github.com/ggml-org/llama.cpp"
+ },
+ {
+ "id": "lmstudio",
+ "name": "LM Studio / llmster",
+ "source": "https://lmstudio.ai/docs/developer/rest"
+ },
+ {
+ "id": "mlx_lm",
+ "name": "MLX-LM",
+ "source": "https://github.com/ml-explore/mlx-lm"
+ },
+ {
+ "id": "omlx",
+ "name": "oMLX",
+ "source": "https://github.com/jundot/omlx"
+ },
+ {
+ "id": "vllm",
+ "name": "vLLM",
+ "source": "https://github.com/vllm-project/vllm-metal",
+ "qualification": "vLLM/vLLM Metal 0.29.0 qualified with Qwen3 0.6B 4-bit on M2 Max; no CUDA feature parity implied.",
+ "backend": "vllm-metal"
+ }
+ ],
+ "experimental": [
+ {
+ "id": "vllm_mlx",
+ "name": "vllm-mlx",
+ "source": "https://github.com/waybarrios/vllm-mlx",
+ "qualification": "Independent of upstream vLLM; use separate adapter and isolated environment."
+ },
+ {
+ "id": "sglang_mlx",
+ "name": "SGLang MLX",
+ "source": "https://github.com/sgl-project/sglang",
+ "qualification": "Validate Mac backend installation, supported models, and metrics; do not assume CUDA feature parity."
+ }
+ ],
+ "next_candidates": [
+ "mistral.rs",
+ "exo",
+ "LocalAI",
+ "Lemonade",
+ "llamafile",
+ "Jan"
+ ],
+ "research_only": [
+ "Flash-MoE",
+ "Parallax",
+ "ik_llama.cpp",
+ "LiteRT-LM",
+ "MLC-LLM"
+ ],
+ "other_hardware": [
+ "vLLM CUDA/ROCm",
+ "SGLang CUDA/ROCm",
+ "KTransformers",
+ "TensorRT-LLM",
+ "ExLlamaV3",
+ "GenieX"
+ ],
+ "orchestration_integrations": [
+ "llama-swap",
+ "RamaLama"
+ ],
+ "excluded": [
+ {
+ "name": "Hugging Face TGI",
+ "reason": "Archived repository; maintainers recommend other serving engines."
+ }
+ ],
+ "benchmarked": true,
+ "implementation_started": true,
+ "decision": {
+ "first_harness": "ollama",
+ "reason": "User requested one working pilot before choosing the rest. Ollama was already installed and provides model management plus local inference.",
+ "remaining_list_locked": false,
+ "second_harness": "mlx_lm",
+ "updated_at": "2026-09-20",
+ "product_direction": "Harness plus the conversational agent provides model management. Prioritize direct engines and valuable advanced capabilities over additional overlapping manager applications.",
+ "next_recommended": "llamacpp",
+ "advanced_targets": [
+ "vllm",
+ "sglang_mlx"
+ ],
+ "deprioritized": [
+ "lmstudio",
+ "omlx",
+ "vllm_mlx"
+ ],
+ "third_harness": "vllm",
+ "framework_backend_policy": "One harness per framework; vLLM Metal is the Apple Silicon backend inside vLLM."
+ },
+ "implemented": [
+ "ollama",
+ "mlx_lm",
+ "vllm"
+ ],
+ "benchmark_scope": "Separate Ollama, MLX-LM and vLLM Metal small-model warm measurements on this M2 Max. vLLM also measures bounded concurrent client requests. Different timing/cache definitions; no controlled cross-framework ranking."
+}
diff --git a/docs/research/local-ai-harnesses/store-category.png b/docs/research/local-ai-harnesses/store-category.png
new file mode 100644
index 000000000..befc1dc82
Binary files /dev/null and b/docs/research/local-ai-harnesses/store-category.png differ
diff --git a/docs/research/local-ai-harnesses/store-verification.json b/docs/research/local-ai-harnesses/store-verification.json
new file mode 100644
index 000000000..0b5479891
--- /dev/null
+++ b/docs/research/local-ai-harnesses/store-verification.json
@@ -0,0 +1,94 @@
+{
+ "sourceCommit": "05fcd5a6185f757259699f93e6ad0fa922a65892",
+ "platform": "Apple M2 Max, 64 GiB, macOS 26.6.2",
+ "checks": [
+ "fresh sparse installs into a temporary registry",
+ "isolated setup and doctor hooks, including native Metal cache kernel",
+ "independent viewer and assets",
+ "natural-language runner deployment of cached Qwen3 demo model",
+ "real arithmetic answers and token metrics",
+ "test viewer cleanup"
+ ],
+ "note": "vLLM answered with the expression 2 + 2 = 4; an initial exact-number assertion was too narrow. The successful stored job, correct answer, metrics and process exit were then checked.",
+ "results": [
+ {
+ "package": "autonomous/ollama",
+ "answer": "4",
+ "metrics": {
+ "model": "qwen3:0.6b",
+ "tokens": 90,
+ "promptTokens": 24,
+ "tokensPerSecond": 290.84234406002986,
+ "promptTokensPerSecond": 1017.5959296162816,
+ "loadMs": 0.631708,
+ "evalMs": 309.446,
+ "doneReason": "stop",
+ "firstTokenMs": 36.63487500000019,
+ "firstOutputMs": 334.022375,
+ "elapsedMs": 338.58695799999987,
+ "context": 4096,
+ "maxTokens": 512,
+ "temperature": 0.3,
+ "seed": 42,
+ "thinking": true
+ },
+ "jobStatus": "succeeded",
+ "testViewerStopped": true
+ },
+ {
+ "package": "autonomous/mlx-lm",
+ "answer": "4",
+ "metrics": {
+ "model": "mlx-community/Qwen3-0.6B-4bit",
+ "tokens": 249,
+ "promptTokens": 22,
+ "tokensPerSecond": 365.3933375725497,
+ "promptTokensPerSecond": 15.211015247795467,
+ "nativeFirstTokenMs": 1701.4684170717373,
+ "nativeFirstOutputMs": 2380.1687500672415,
+ "peakMemoryBytes": 462946604,
+ "thinking": true,
+ "doneReason": "stop",
+ "wallMs": 2390.8956249943003,
+ "measurement": "MLX-LM native generation_tps; TTFT includes local IPC; fresh prompt cache",
+ "firstTokenMs": 1713.313583,
+ "firstOutputMs": 2391.9136660000004
+ },
+ "jobStatus": "succeeded",
+ "testViewerStopped": true
+ },
+ {
+ "package": "autonomous/vllm",
+ "answer": "2 + 2 = 4",
+ "metrics": {
+ "model": "mlx-community/Qwen3-0.6B-4bit",
+ "tokens": 8,
+ "promptTokens": 26,
+ "tokensPerSecond": 28.04624175995718,
+ "firstTokenMs": 226.6821259999997,
+ "firstOutputMs": 226.72441799999797,
+ "totalMs": 285.2432090000002,
+ "thinking": false,
+ "doneReason": "stop",
+ "metric": "client_end_to_end",
+ "serving": {
+ "context": 4096,
+ "maxSequences": 4,
+ "memoryFraction": 0.1,
+ "prefixCaching": true
+ }
+ },
+ "jobStatus": "succeeded",
+ "testViewerStopped": true
+ }
+ ],
+ "verifiedAt": "2026-09-20T07:00:19.653951+00:00",
+ "automatedChecks": {
+ "harnessTests": 24,
+ "desktopTests": 17,
+ "storeRegistryAndCatalogTests": 448,
+ "targetedDartAnalysis": "passed",
+ "catalogPackages": 49,
+ "manifestConformance": "all three passed"
+ }
+}
diff --git a/docs/research/local-ai-harnesses/vllm/benchmarks.json b/docs/research/local-ai-harnesses/vllm/benchmarks.json
new file mode 100644
index 000000000..5370df0f5
--- /dev/null
+++ b/docs/research/local-ai-harnesses/vllm/benchmarks.json
@@ -0,0 +1,1046 @@
+{
+ "exportedAt": "2026-09-20T05:46:19.730Z",
+ "system": {
+ "platform": "darwin",
+ "arch": "arm64",
+ "chip": "Apple M2 Max",
+ "totalMemory": 68719476736,
+ "cores": 12
+ },
+ "benchmarks": [
+ {
+ "id": "9453d1cf-1778-4b73-99e3-e95ab9cb70ad",
+ "model": "mlx-community/Qwen3-0.6B-4bit",
+ "digest": "73e3e38d981303bc594367cd910ea6eb48349da8",
+ "at": "2026-09-20T05:45:33.267Z",
+ "runtime": {
+ "name": "vLLM",
+ "version": "0.29.0+cpu",
+ "mlxVersion": "0.32.1"
+ },
+ "hardware": {
+ "platform": "darwin",
+ "arch": "arm64",
+ "chip": "Apple M2 Max",
+ "totalMemory": 68719476736,
+ "cores": 12
+ },
+ "settings": {
+ "context": 2048,
+ "maxTokens": 128,
+ "seed": 42,
+ "temperature": 0,
+ "thinking": false
+ },
+ "prompt": "Explain how a bicycle works to a curious beginner. Discuss the pedals, chain, gears, wheels, brakes, and balance. Write a clear explanation of at least 250 words without a heading.",
+ "promptHash": "e3aff106a75867995dfbb42b73d60cb031d6d1d9511c155329d29033854e829a",
+ "mode": "warm server; repeated prompt may use prefix cache; client end-to-end token throughput, including prompt processing and HTTP",
+ "samples": [
+ {
+ "model": "mlx-community/Qwen3-0.6B-4bit",
+ "tokens": 128,
+ "promptTokens": 54,
+ "tokensPerSecond": 188.87424931468553,
+ "firstTokenMs": 22.569000999999844,
+ "firstOutputMs": 22.569750999999087,
+ "totalMs": 677.6995829999978,
+ "thinking": false,
+ "doneReason": "length",
+ "metric": "client_end_to_end",
+ "serving": {
+ "context": 2048,
+ "maxSequences": 4,
+ "memoryFraction": 0.1,
+ "prefixCaching": true
+ }
+ },
+ {
+ "model": "mlx-community/Qwen3-0.6B-4bit",
+ "tokens": 128,
+ "promptTokens": 54,
+ "tokensPerSecond": 203.99530673097718,
+ "firstTokenMs": 20.764125000001513,
+ "firstOutputMs": 20.764500000001135,
+ "totalMs": 627.4654159999991,
+ "thinking": false,
+ "doneReason": "length",
+ "metric": "client_end_to_end",
+ "serving": {
+ "context": 2048,
+ "maxSequences": 4,
+ "memoryFraction": 0.1,
+ "prefixCaching": true
+ }
+ },
+ {
+ "model": "mlx-community/Qwen3-0.6B-4bit",
+ "tokens": 128,
+ "promptTokens": 54,
+ "tokensPerSecond": 207.23715033537437,
+ "firstTokenMs": 19.830541999999696,
+ "firstOutputMs": 19.83129199999894,
+ "totalMs": 617.6498749999992,
+ "thinking": false,
+ "doneReason": "length",
+ "metric": "client_end_to_end",
+ "serving": {
+ "context": 2048,
+ "maxSequences": 4,
+ "memoryFraction": 0.1,
+ "prefixCaching": true
+ }
+ }
+ ],
+ "medianTokensPerSecond": 203.99530673097718,
+ "medianFirstTokenMs": 20.764125000001513,
+ "medianFirstOutputMs": 20.764500000001135,
+ "residentBytes": 2444935168
+ },
+ {
+ "id": "fd7bf760-1a51-499d-ab6e-e1b85f0b09cd",
+ "model": "mlx-community/Qwen3-0.6B-4bit",
+ "digest": "73e3e38d981303bc594367cd910ea6eb48349da8",
+ "at": "2026-09-20T05:44:06.282Z",
+ "runtime": {
+ "name": "vLLM",
+ "version": "0.29.0+cpu",
+ "mlxVersion": "0.32.1"
+ },
+ "hardware": {
+ "platform": "darwin",
+ "arch": "arm64",
+ "chip": "Apple M2 Max",
+ "totalMemory": 68719476736,
+ "cores": 12
+ },
+ "settings": {
+ "context": 2048,
+ "maxTokens": 128,
+ "seed": 42,
+ "temperature": 0,
+ "thinking": false
+ },
+ "prompt": "Explain how a bicycle works to a curious beginner. Discuss the pedals, chain, gears, wheels, brakes, and balance. Write a clear explanation of at least 250 words without a heading.",
+ "promptHash": "e3aff106a75867995dfbb42b73d60cb031d6d1d9511c155329d29033854e829a",
+ "mode": "warm server; repeated prompt may use prefix cache; client end-to-end token throughput, including prompt processing and HTTP",
+ "samples": [
+ {
+ "model": "mlx-community/Qwen3-0.6B-4bit",
+ "tokens": 128,
+ "promptTokens": 54,
+ "tokensPerSecond": 223.0120888389493,
+ "firstTokenMs": 19.631916999998793,
+ "firstOutputMs": 19.632666999998037,
+ "totalMs": 573.9599170000001,
+ "thinking": false,
+ "doneReason": "length",
+ "metric": "client_end_to_end",
+ "serving": {
+ "context": 2048,
+ "maxSequences": 4,
+ "memoryFraction": 0.1,
+ "prefixCaching": true
+ }
+ },
+ {
+ "model": "mlx-community/Qwen3-0.6B-4bit",
+ "tokens": 128,
+ "promptTokens": 54,
+ "tokensPerSecond": 219.83205734191475,
+ "firstTokenMs": 19.45866599999863,
+ "firstOutputMs": 19.458957999999257,
+ "totalMs": 582.262666999999,
+ "thinking": false,
+ "doneReason": "length",
+ "metric": "client_end_to_end",
+ "serving": {
+ "context": 2048,
+ "maxSequences": 4,
+ "memoryFraction": 0.1,
+ "prefixCaching": true
+ }
+ },
+ {
+ "model": "mlx-community/Qwen3-0.6B-4bit",
+ "tokens": 128,
+ "promptTokens": 54,
+ "tokensPerSecond": 212.27963031834062,
+ "firstTokenMs": 19.63141700000415,
+ "firstOutputMs": 19.631875000002765,
+ "totalMs": 602.9782500000001,
+ "thinking": false,
+ "doneReason": "length",
+ "metric": "client_end_to_end",
+ "serving": {
+ "context": 2048,
+ "maxSequences": 4,
+ "memoryFraction": 0.1,
+ "prefixCaching": true
+ }
+ }
+ ],
+ "medianTokensPerSecond": 219.83205734191475,
+ "medianFirstTokenMs": 19.63141700000415,
+ "medianFirstOutputMs": 19.631875000002765,
+ "residentBytes": 2445688832
+ }
+ ],
+ "loadTests": [
+ {
+ "id": "19bf2bc2-11ef-4895-8733-e88c24376d3a",
+ "at": "2026-09-20T05:45:41.616Z",
+ "model": "mlx-community/Qwen3-0.6B-4bit",
+ "digest": "73e3e38d981303bc594367cd910ea6eb48349da8",
+ "runtime": {
+ "name": "vLLM",
+ "version": "0.29.0+cpu",
+ "backend": "Metal",
+ "backendVersion": "0.29.0"
+ },
+ "hardware": {
+ "platform": "darwin",
+ "arch": "arm64",
+ "chip": "Apple M2 Max",
+ "totalMemory": 68719476736,
+ "cores": 12
+ },
+ "concurrency": 4,
+ "requests": 8,
+ "succeeded": 8,
+ "failed": 0,
+ "totalMs": 1746.6910419999986,
+ "aggregateTokensPerSecond": 586.2513606456114,
+ "medianLatencyMs": 872.0753120000008,
+ "p95LatencyMs": 910.880333000001,
+ "medianFirstTokenMs": 46.82056250000278,
+ "settings": {
+ "context": 2048,
+ "maxTokens": 128,
+ "seed": 42,
+ "temperature": 0,
+ "thinking": false
+ },
+ "serving": {
+ "context": 2048,
+ "maxSequences": 4,
+ "memoryFraction": 0.1,
+ "prefixCaching": true,
+ "status": "serving",
+ "model": "mlx-community/Qwen3-0.6B-4bit",
+ "processRssBytes": 2445279232,
+ "runningRequests": 0,
+ "waitingRequests": 0,
+ "cacheUsage": 0,
+ "prefixQueries": 773,
+ "prefixHits": 560
+ },
+ "prompt": "Explain how a bicycle works to a curious beginner. Discuss the pedals, chain, gears, wheels, brakes, and balance. Write a clear explanation of at least 250 words without a heading.",
+ "promptVariation": "Reference number: {1-based request index}.",
+ "mode": "one discarded warmup; repeated prompt prefixes may be cached; client end-to-end measurements",
+ "samples": [
+ {
+ "index": 0,
+ "status": "succeeded",
+ "model": "mlx-community/Qwen3-0.6B-4bit",
+ "tokens": 128,
+ "promptTokens": 60,
+ "tokensPerSecond": 153.38036698349822,
+ "firstTokenMs": 40.155791000001045,
+ "firstOutputMs": 40.15649900000062,
+ "totalMs": 834.5266249999986,
+ "thinking": false,
+ "doneReason": "length",
+ "metric": "client_end_to_end",
+ "serving": {
+ "context": 2048,
+ "maxSequences": 4,
+ "memoryFraction": 0.1,
+ "prefixCaching": true
+ }
+ },
+ {
+ "index": 1,
+ "status": "succeeded",
+ "model": "mlx-community/Qwen3-0.6B-4bit",
+ "tokens": 128,
+ "promptTokens": 60,
+ "tokensPerSecond": 153.22307130587717,
+ "firstTokenMs": 46.95591699999932,
+ "firstOutputMs": 46.95604200000162,
+ "totalMs": 835.383332999998,
+ "thinking": false,
+ "doneReason": "length",
+ "metric": "client_end_to_end",
+ "serving": {
+ "context": 2048,
+ "maxSequences": 4,
+ "memoryFraction": 0.1,
+ "prefixCaching": true
+ }
+ },
+ {
+ "index": 2,
+ "status": "succeeded",
+ "model": "mlx-community/Qwen3-0.6B-4bit",
+ "tokens": 128,
+ "promptTokens": 60,
+ "tokensPerSecond": 153.17860571694376,
+ "firstTokenMs": 46.82866700000159,
+ "firstOutputMs": 46.82900000000154,
+ "totalMs": 835.6258330000019,
+ "thinking": false,
+ "doneReason": "length",
+ "metric": "client_end_to_end",
+ "serving": {
+ "context": 2048,
+ "maxSequences": 4,
+ "memoryFraction": 0.1,
+ "prefixCaching": true
+ }
+ },
+ {
+ "index": 3,
+ "status": "succeeded",
+ "model": "mlx-community/Qwen3-0.6B-4bit",
+ "tokens": 128,
+ "promptTokens": 60,
+ "tokensPerSecond": 153.10183730302418,
+ "firstTokenMs": 46.81245800000397,
+ "firstOutputMs": 46.8126250000023,
+ "totalMs": 836.0448329999999,
+ "thinking": false,
+ "doneReason": "length",
+ "metric": "client_end_to_end",
+ "serving": {
+ "context": 2048,
+ "maxSequences": 4,
+ "memoryFraction": 0.1,
+ "prefixCaching": true
+ }
+ },
+ {
+ "index": 4,
+ "status": "succeeded",
+ "model": "mlx-community/Qwen3-0.6B-4bit",
+ "tokens": 128,
+ "promptTokens": 60,
+ "tokensPerSecond": 140.9527406042054,
+ "firstTokenMs": 41.461708000002545,
+ "firstOutputMs": 41.46200000000317,
+ "totalMs": 908.1057910000018,
+ "thinking": false,
+ "doneReason": "length",
+ "metric": "client_end_to_end",
+ "serving": {
+ "context": 2048,
+ "maxSequences": 4,
+ "memoryFraction": 0.1,
+ "prefixCaching": true
+ }
+ },
+ {
+ "index": 5,
+ "status": "succeeded",
+ "model": "mlx-community/Qwen3-0.6B-4bit",
+ "tokens": 128,
+ "promptTokens": 60,
+ "tokensPerSecond": 140.52339847807414,
+ "firstTokenMs": 47.61737500000163,
+ "firstOutputMs": 47.617542000003596,
+ "totalMs": 910.880333000001,
+ "thinking": false,
+ "doneReason": "length",
+ "metric": "client_end_to_end",
+ "serving": {
+ "context": 2048,
+ "maxSequences": 4,
+ "memoryFraction": 0.1,
+ "prefixCaching": true
+ }
+ },
+ {
+ "index": 6,
+ "status": "succeeded",
+ "model": "mlx-community/Qwen3-0.6B-4bit",
+ "tokens": 128,
+ "promptTokens": 60,
+ "tokensPerSecond": 140.59506862796826,
+ "firstTokenMs": 47.375374999995984,
+ "firstOutputMs": 47.375499999994645,
+ "totalMs": 910.4159999999974,
+ "thinking": false,
+ "doneReason": "length",
+ "metric": "client_end_to_end",
+ "serving": {
+ "context": 2048,
+ "maxSequences": 4,
+ "memoryFraction": 0.1,
+ "prefixCaching": true
+ }
+ },
+ {
+ "index": 7,
+ "status": "succeeded",
+ "model": "mlx-community/Qwen3-0.6B-4bit",
+ "tokens": 128,
+ "promptTokens": 60,
+ "tokensPerSecond": 140.63358369575838,
+ "firstTokenMs": 46.70749899999646,
+ "firstOutputMs": 46.707832999996754,
+ "totalMs": 910.1666659999973,
+ "thinking": false,
+ "doneReason": "length",
+ "metric": "client_end_to_end",
+ "serving": {
+ "context": 2048,
+ "maxSequences": 4,
+ "memoryFraction": 0.1,
+ "prefixCaching": true
+ }
+ }
+ ]
+ },
+ {
+ "id": "f136d3c6-189d-4158-af01-37c69fad10c4",
+ "at": "2026-09-20T05:45:39.385Z",
+ "model": "mlx-community/Qwen3-0.6B-4bit",
+ "digest": "73e3e38d981303bc594367cd910ea6eb48349da8",
+ "runtime": {
+ "name": "vLLM",
+ "version": "0.29.0+cpu",
+ "backend": "Metal",
+ "backendVersion": "0.29.0"
+ },
+ "hardware": {
+ "platform": "darwin",
+ "arch": "arm64",
+ "chip": "Apple M2 Max",
+ "totalMemory": 68719476736,
+ "cores": 12
+ },
+ "concurrency": 1,
+ "requests": 8,
+ "succeeded": 8,
+ "failed": 0,
+ "totalMs": 5719.136291999999,
+ "aggregateTokensPerSecond": 179.04801489560307,
+ "medianLatencyMs": 718.4712499999987,
+ "p95LatencyMs": 741.7076670000024,
+ "medianFirstTokenMs": 27.92120799999975,
+ "settings": {
+ "context": 2048,
+ "maxTokens": 128,
+ "seed": 42,
+ "temperature": 0,
+ "thinking": false
+ },
+ "serving": {
+ "context": 2048,
+ "maxSequences": 4,
+ "memoryFraction": 0.1,
+ "prefixCaching": true,
+ "status": "serving",
+ "model": "mlx-community/Qwen3-0.6B-4bit",
+ "processRssBytes": 2444935168,
+ "runningRequests": 0,
+ "waitingRequests": 0,
+ "cacheUsage": 0,
+ "prefixQueries": 239,
+ "prefixHits": 144
+ },
+ "prompt": "Explain how a bicycle works to a curious beginner. Discuss the pedals, chain, gears, wheels, brakes, and balance. Write a clear explanation of at least 250 words without a heading.",
+ "promptVariation": "Reference number: {1-based request index}.",
+ "mode": "one discarded warmup; repeated prompt prefixes may be cached; client end-to-end measurements",
+ "samples": [
+ {
+ "index": 0,
+ "status": "succeeded",
+ "model": "mlx-community/Qwen3-0.6B-4bit",
+ "tokens": 128,
+ "promptTokens": 60,
+ "tokensPerSecond": 176.83349917230174,
+ "firstTokenMs": 26.26654200000121,
+ "firstOutputMs": 26.267625000000407,
+ "totalMs": 723.8447500000002,
+ "thinking": false,
+ "doneReason": "length",
+ "metric": "client_end_to_end",
+ "serving": {
+ "context": 2048,
+ "maxSequences": 4,
+ "memoryFraction": 0.1,
+ "prefixCaching": true
+ }
+ },
+ {
+ "index": 1,
+ "status": "succeeded",
+ "model": "mlx-community/Qwen3-0.6B-4bit",
+ "tokens": 128,
+ "promptTokens": 60,
+ "tokensPerSecond": 172.57472949918903,
+ "firstTokenMs": 36.065999999998894,
+ "firstOutputMs": 36.06683300000077,
+ "totalMs": 741.7076670000024,
+ "thinking": false,
+ "doneReason": "length",
+ "metric": "client_end_to_end",
+ "serving": {
+ "context": 2048,
+ "maxSequences": 4,
+ "memoryFraction": 0.1,
+ "prefixCaching": true
+ }
+ },
+ {
+ "index": 2,
+ "status": "succeeded",
+ "model": "mlx-community/Qwen3-0.6B-4bit",
+ "tokens": 128,
+ "promptTokens": 60,
+ "tokensPerSecond": 177.9406738147618,
+ "firstTokenMs": 36.62258400000064,
+ "firstOutputMs": 36.623875999997836,
+ "totalMs": 719.340874999998,
+ "thinking": false,
+ "doneReason": "length",
+ "metric": "client_end_to_end",
+ "serving": {
+ "context": 2048,
+ "maxSequences": 4,
+ "memoryFraction": 0.1,
+ "prefixCaching": true
+ }
+ },
+ {
+ "index": 3,
+ "status": "succeeded",
+ "model": "mlx-community/Qwen3-0.6B-4bit",
+ "tokens": 128,
+ "promptTokens": 60,
+ "tokensPerSecond": 178.3719483634114,
+ "firstTokenMs": 29.575873999998294,
+ "firstOutputMs": 29.576498999998876,
+ "totalMs": 717.6016249999993,
+ "thinking": false,
+ "doneReason": "length",
+ "metric": "client_end_to_end",
+ "serving": {
+ "context": 2048,
+ "maxSequences": 4,
+ "memoryFraction": 0.1,
+ "prefixCaching": true
+ }
+ },
+ {
+ "index": 4,
+ "status": "succeeded",
+ "model": "mlx-community/Qwen3-0.6B-4bit",
+ "tokens": 128,
+ "promptTokens": 60,
+ "tokensPerSecond": 182.6634650461023,
+ "firstTokenMs": 25.506625000001804,
+ "firstOutputMs": 25.50733300000138,
+ "totalMs": 700.742209,
+ "thinking": false,
+ "doneReason": "length",
+ "metric": "client_end_to_end",
+ "serving": {
+ "context": 2048,
+ "maxSequences": 4,
+ "memoryFraction": 0.1,
+ "prefixCaching": true
+ }
+ },
+ {
+ "index": 5,
+ "status": "succeeded",
+ "model": "mlx-community/Qwen3-0.6B-4bit",
+ "tokens": 128,
+ "promptTokens": 60,
+ "tokensPerSecond": 188.17859839654844,
+ "firstTokenMs": 25.137167000000773,
+ "firstOutputMs": 25.13770800000202,
+ "totalMs": 680.2048750000031,
+ "thinking": false,
+ "doneReason": "length",
+ "metric": "client_end_to_end",
+ "serving": {
+ "context": 2048,
+ "maxSequences": 4,
+ "memoryFraction": 0.1,
+ "prefixCaching": true
+ }
+ },
+ {
+ "index": 6,
+ "status": "succeeded",
+ "model": "mlx-community/Qwen3-0.6B-4bit",
+ "tokens": 128,
+ "promptTokens": 60,
+ "tokensPerSecond": 173.38913618853095,
+ "firstTokenMs": 24.84412499999962,
+ "firstOutputMs": 24.84533399999782,
+ "totalMs": 738.2238749999997,
+ "thinking": false,
+ "doneReason": "length",
+ "metric": "client_end_to_end",
+ "serving": {
+ "context": 2048,
+ "maxSequences": 4,
+ "memoryFraction": 0.1,
+ "prefixCaching": true
+ }
+ },
+ {
+ "index": 7,
+ "status": "succeeded",
+ "model": "mlx-community/Qwen3-0.6B-4bit",
+ "tokens": 128,
+ "promptTokens": 60,
+ "tokensPerSecond": 183.87159977275672,
+ "firstTokenMs": 31.890665999999328,
+ "firstOutputMs": 31.89179100000183,
+ "totalMs": 696.1379579999993,
+ "thinking": false,
+ "doneReason": "length",
+ "metric": "client_end_to_end",
+ "serving": {
+ "context": 2048,
+ "maxSequences": 4,
+ "memoryFraction": 0.1,
+ "prefixCaching": true
+ }
+ }
+ ]
+ },
+ {
+ "id": "f0435c5b-f9ed-4af3-9b63-1574412cce25",
+ "at": "2026-09-20T05:44:13.929Z",
+ "model": "mlx-community/Qwen3-0.6B-4bit",
+ "digest": "73e3e38d981303bc594367cd910ea6eb48349da8",
+ "runtime": {
+ "name": "vLLM",
+ "version": "0.29.0+cpu",
+ "backend": "Metal",
+ "backendVersion": "0.29.0"
+ },
+ "hardware": {
+ "platform": "darwin",
+ "arch": "arm64",
+ "chip": "Apple M2 Max",
+ "totalMemory": 68719476736,
+ "cores": 12
+ },
+ "concurrency": 4,
+ "requests": 8,
+ "succeeded": 8,
+ "failed": 0,
+ "totalMs": 1733.193542000001,
+ "aggregateTokensPerSecond": 590.8168794688479,
+ "medianLatencyMs": 866.1290204999987,
+ "p95LatencyMs": 881.4543749999975,
+ "medianFirstTokenMs": 55.44614549999642,
+ "settings": {
+ "context": 2048,
+ "maxTokens": 128,
+ "seed": 42,
+ "temperature": 0,
+ "thinking": false
+ },
+ "serving": {
+ "context": 2048,
+ "maxSequences": 4,
+ "memoryFraction": 0.1,
+ "prefixCaching": true,
+ "status": "serving",
+ "model": "mlx-community/Qwen3-0.6B-4bit",
+ "processRssBytes": 2446032896,
+ "runningRequests": 0,
+ "waitingRequests": 0,
+ "cacheUsage": 0,
+ "prefixQueries": 773,
+ "prefixHits": 560
+ },
+ "prompt": "Explain how a bicycle works to a curious beginner. Discuss the pedals, chain, gears, wheels, brakes, and balance. Write a clear explanation of at least 250 words without a heading.",
+ "promptVariation": "Reference number: {1-based request index}.",
+ "mode": "one discarded warmup; repeated prompt prefixes may be cached; client end-to-end measurements",
+ "samples": [
+ {
+ "index": 0,
+ "status": "succeeded",
+ "model": "mlx-community/Qwen3-0.6B-4bit",
+ "tokens": 128,
+ "promptTokens": 60,
+ "tokensPerSecond": 145.2397942248954,
+ "firstTokenMs": 55.5712489999969,
+ "firstOutputMs": 55.57145799999853,
+ "totalMs": 881.3011659999975,
+ "thinking": false,
+ "doneReason": "length",
+ "metric": "client_end_to_end",
+ "serving": {
+ "context": 2048,
+ "maxSequences": 4,
+ "memoryFraction": 0.1,
+ "prefixCaching": true
+ }
+ },
+ {
+ "index": 1,
+ "status": "succeeded",
+ "model": "mlx-community/Qwen3-0.6B-4bit",
+ "tokens": 128,
+ "promptTokens": 60,
+ "tokensPerSecond": 145.38861122371307,
+ "firstTokenMs": 48.734040999999706,
+ "firstOutputMs": 48.734582999997656,
+ "totalMs": 880.3990829999966,
+ "thinking": false,
+ "doneReason": "length",
+ "metric": "client_end_to_end",
+ "serving": {
+ "context": 2048,
+ "maxSequences": 4,
+ "memoryFraction": 0.1,
+ "prefixCaching": true
+ }
+ },
+ {
+ "index": 2,
+ "status": "succeeded",
+ "model": "mlx-community/Qwen3-0.6B-4bit",
+ "tokens": 128,
+ "promptTokens": 60,
+ "tokensPerSecond": 145.335077543502,
+ "firstTokenMs": 55.288832999998704,
+ "firstOutputMs": 55.28924999999799,
+ "totalMs": 880.7233750000014,
+ "thinking": false,
+ "doneReason": "length",
+ "metric": "client_end_to_end",
+ "serving": {
+ "context": 2048,
+ "maxSequences": 4,
+ "memoryFraction": 0.1,
+ "prefixCaching": true
+ }
+ },
+ {
+ "index": 3,
+ "status": "succeeded",
+ "model": "mlx-community/Qwen3-0.6B-4bit",
+ "tokens": 128,
+ "promptTokens": 60,
+ "tokensPerSecond": 145.2145495335483,
+ "firstTokenMs": 55.32104199999594,
+ "firstOutputMs": 55.321375999996235,
+ "totalMs": 881.4543749999975,
+ "thinking": false,
+ "doneReason": "length",
+ "metric": "client_end_to_end",
+ "serving": {
+ "context": 2048,
+ "maxSequences": 4,
+ "memoryFraction": 0.1,
+ "prefixCaching": true
+ }
+ },
+ {
+ "index": 4,
+ "status": "succeeded",
+ "model": "mlx-community/Qwen3-0.6B-4bit",
+ "tokens": 128,
+ "promptTokens": 60,
+ "tokensPerSecond": 150.42366136741038,
+ "firstTokenMs": 45.1540419999983,
+ "firstOutputMs": 45.154458000000886,
+ "totalMs": 850.9299589999973,
+ "thinking": false,
+ "doneReason": "length",
+ "metric": "client_end_to_end",
+ "serving": {
+ "context": 2048,
+ "maxSequences": 4,
+ "memoryFraction": 0.1,
+ "prefixCaching": true
+ }
+ },
+ {
+ "index": 5,
+ "status": "succeeded",
+ "model": "mlx-community/Qwen3-0.6B-4bit",
+ "tokens": 128,
+ "promptTokens": 60,
+ "tokensPerSecond": 150.25961609949977,
+ "firstTokenMs": 58.28095899999971,
+ "firstOutputMs": 58.281041999998706,
+ "totalMs": 851.8589580000007,
+ "thinking": false,
+ "doneReason": "length",
+ "metric": "client_end_to_end",
+ "serving": {
+ "context": 2048,
+ "maxSequences": 4,
+ "memoryFraction": 0.1,
+ "prefixCaching": true
+ }
+ },
+ {
+ "index": 6,
+ "status": "succeeded",
+ "model": "mlx-community/Qwen3-0.6B-4bit",
+ "tokens": 128,
+ "promptTokens": 60,
+ "tokensPerSecond": 150.37047814371138,
+ "firstTokenMs": 57.71675100000357,
+ "firstOutputMs": 57.717209000002185,
+ "totalMs": 851.2309170000008,
+ "thinking": false,
+ "doneReason": "length",
+ "metric": "client_end_to_end",
+ "serving": {
+ "context": 2048,
+ "maxSequences": 4,
+ "memoryFraction": 0.1,
+ "prefixCaching": true
+ }
+ },
+ {
+ "index": 7,
+ "status": "succeeded",
+ "model": "mlx-community/Qwen3-0.6B-4bit",
+ "tokens": 128,
+ "promptTokens": 60,
+ "tokensPerSecond": 150.3561414856606,
+ "firstTokenMs": 57.47833299999547,
+ "firstOutputMs": 57.47845799999777,
+ "totalMs": 851.3120829999971,
+ "thinking": false,
+ "doneReason": "length",
+ "metric": "client_end_to_end",
+ "serving": {
+ "context": 2048,
+ "maxSequences": 4,
+ "memoryFraction": 0.1,
+ "prefixCaching": true
+ }
+ }
+ ]
+ },
+ {
+ "id": "80fba206-1ab3-4762-b340-3b70aa46a441",
+ "at": "2026-09-20T05:44:11.759Z",
+ "model": "mlx-community/Qwen3-0.6B-4bit",
+ "digest": "73e3e38d981303bc594367cd910ea6eb48349da8",
+ "runtime": {
+ "name": "vLLM",
+ "version": "0.29.0+cpu",
+ "backend": "Metal",
+ "backendVersion": "0.29.0"
+ },
+ "hardware": {
+ "platform": "darwin",
+ "arch": "arm64",
+ "chip": "Apple M2 Max",
+ "totalMemory": 68719476736,
+ "cores": 12
+ },
+ "concurrency": 1,
+ "requests": 8,
+ "succeeded": 8,
+ "failed": 0,
+ "totalMs": 5138.068124999998,
+ "aggregateTokensPerSecond": 199.2966957790192,
+ "medianLatencyMs": 631.5969165000024,
+ "p95LatencyMs": 715.6627919999992,
+ "medianFirstTokenMs": 23.968166500002553,
+ "settings": {
+ "context": 2048,
+ "maxTokens": 128,
+ "seed": 42,
+ "temperature": 0,
+ "thinking": false
+ },
+ "serving": {
+ "context": 2048,
+ "maxSequences": 4,
+ "memoryFraction": 0.1,
+ "prefixCaching": true,
+ "status": "serving",
+ "model": "mlx-community/Qwen3-0.6B-4bit",
+ "processRssBytes": 2445688832,
+ "runningRequests": 0,
+ "waitingRequests": 0,
+ "cacheUsage": 0,
+ "prefixQueries": 239,
+ "prefixHits": 144
+ },
+ "prompt": "Explain how a bicycle works to a curious beginner. Discuss the pedals, chain, gears, wheels, brakes, and balance. Write a clear explanation of at least 250 words without a heading.",
+ "promptVariation": "Reference number: {1-based request index}.",
+ "mode": "one discarded warmup; repeated prompt prefixes may be cached; client end-to-end measurements",
+ "samples": [
+ {
+ "index": 0,
+ "status": "succeeded",
+ "model": "mlx-community/Qwen3-0.6B-4bit",
+ "tokens": 128,
+ "promptTokens": 60,
+ "tokensPerSecond": 206.68862408193544,
+ "firstTokenMs": 23.63654200000383,
+ "firstOutputMs": 23.637125000004744,
+ "totalMs": 619.2890420000003,
+ "thinking": false,
+ "doneReason": "length",
+ "metric": "client_end_to_end",
+ "serving": {
+ "context": 2048,
+ "maxSequences": 4,
+ "memoryFraction": 0.1,
+ "prefixCaching": true
+ }
+ },
+ {
+ "index": 1,
+ "status": "succeeded",
+ "model": "mlx-community/Qwen3-0.6B-4bit",
+ "tokens": 128,
+ "promptTokens": 60,
+ "tokensPerSecond": 178.85518351777068,
+ "firstTokenMs": 24.39208399999916,
+ "firstOutputMs": 24.397000000000844,
+ "totalMs": 715.6627919999992,
+ "thinking": false,
+ "doneReason": "length",
+ "metric": "client_end_to_end",
+ "serving": {
+ "context": 2048,
+ "maxSequences": 4,
+ "memoryFraction": 0.1,
+ "prefixCaching": true
+ }
+ },
+ {
+ "index": 2,
+ "status": "succeeded",
+ "model": "mlx-community/Qwen3-0.6B-4bit",
+ "tokens": 128,
+ "promptTokens": 60,
+ "tokensPerSecond": 187.77378729303157,
+ "firstTokenMs": 70.14762400000109,
+ "firstOutputMs": 70.14837400000033,
+ "totalMs": 681.6712909999987,
+ "thinking": false,
+ "doneReason": "length",
+ "metric": "client_end_to_end",
+ "serving": {
+ "context": 2048,
+ "maxSequences": 4,
+ "memoryFraction": 0.1,
+ "prefixCaching": true
+ }
+ },
+ {
+ "index": 3,
+ "status": "succeeded",
+ "model": "mlx-community/Qwen3-0.6B-4bit",
+ "tokens": 128,
+ "promptTokens": 60,
+ "tokensPerSecond": 200.74512260945863,
+ "firstTokenMs": 26.628207000001566,
+ "firstOutputMs": 26.62870699999985,
+ "totalMs": 637.624458000002,
+ "thinking": false,
+ "doneReason": "length",
+ "metric": "client_end_to_end",
+ "serving": {
+ "context": 2048,
+ "maxSequences": 4,
+ "memoryFraction": 0.1,
+ "prefixCaching": true
+ }
+ },
+ {
+ "index": 4,
+ "status": "succeeded",
+ "model": "mlx-community/Qwen3-0.6B-4bit",
+ "tokens": 128,
+ "promptTokens": 60,
+ "tokensPerSecond": 203.2448384611681,
+ "firstTokenMs": 22.722499999999854,
+ "firstOutputMs": 22.723042000001442,
+ "totalMs": 629.7822910000032,
+ "thinking": false,
+ "doneReason": "length",
+ "metric": "client_end_to_end",
+ "serving": {
+ "context": 2048,
+ "maxSequences": 4,
+ "memoryFraction": 0.1,
+ "prefixCaching": true
+ }
+ },
+ {
+ "index": 5,
+ "status": "succeeded",
+ "model": "mlx-community/Qwen3-0.6B-4bit",
+ "tokens": 128,
+ "promptTokens": 60,
+ "tokensPerSecond": 202.08030879235176,
+ "firstTokenMs": 23.09137500000361,
+ "firstOutputMs": 23.091958000004524,
+ "totalMs": 633.4115420000016,
+ "thinking": false,
+ "doneReason": "length",
+ "metric": "client_end_to_end",
+ "serving": {
+ "context": 2048,
+ "maxSequences": 4,
+ "memoryFraction": 0.1,
+ "prefixCaching": true
+ }
+ },
+ {
+ "index": 6,
+ "status": "succeeded",
+ "model": "mlx-community/Qwen3-0.6B-4bit",
+ "tokens": 128,
+ "promptTokens": 60,
+ "tokensPerSecond": 210.60627447064996,
+ "firstTokenMs": 24.299791000001278,
+ "firstOutputMs": 24.300415999998222,
+ "totalMs": 607.7691669999986,
+ "thinking": false,
+ "doneReason": "length",
+ "metric": "client_end_to_end",
+ "serving": {
+ "context": 2048,
+ "maxSequences": 4,
+ "memoryFraction": 0.1,
+ "prefixCaching": true
+ }
+ },
+ {
+ "index": 7,
+ "status": "succeeded",
+ "model": "mlx-community/Qwen3-0.6B-4bit",
+ "tokens": 128,
+ "promptTokens": 60,
+ "tokensPerSecond": 209.28292394405048,
+ "firstTokenMs": 23.36149999999907,
+ "firstOutputMs": 23.361958999998024,
+ "totalMs": 611.612250000002,
+ "thinking": false,
+ "doneReason": "length",
+ "metric": "client_end_to_end",
+ "serving": {
+ "context": 2048,
+ "maxSequences": 4,
+ "memoryFraction": 0.1,
+ "prefixCaching": true
+ }
+ }
+ ]
+ }
+ ]
+}
\ No newline at end of file
diff --git a/docs/research/local-ai-harnesses/vllm/findings.md b/docs/research/local-ai-harnesses/vllm/findings.md
new file mode 100644
index 000000000..ef25113e9
--- /dev/null
+++ b/docs/research/local-ai-harnesses/vllm/findings.md
@@ -0,0 +1,117 @@
+# vLLM Metal qualification and harness
+
+September 20, 2026 · Apple M2 Max · 64 GiB · macOS 26.6.2.
+
+## Product decision
+
+The user chose to build vLLM after clarifying the overlap between frameworks and
+management applications. **One harness per framework** is the accepted structure:
+vLLM Metal is the Mac backend within **vLLM**, not another harness or store entry.
+MLX is Apple's array framework, MLX-LM is a language-model library, and vLLM Metal
+uses MLX underneath vLLM's serving machinery. They expose different operational
+capabilities despite sharing lower layers. SGLang remains future work.
+
+The working vLLM harness exposes useful serving complexity: context, parallel
+sequences, Metal memory budget, prefix caching, concurrent requests and measurements.
+The queue, commands, inventory and base viewer are shared with the earlier harnesses;
+the adapter, serving panels, branding and metric definitions belong to vLLM.
+
+## Qualified runtime
+
+- Official vLLM 0.29.0 macOS arm64 Python 3.12 core wheel (`0.29.0+cpu`).
+- Official vLLM Metal 0.29.0 macOS 15 arm64 wheel.
+- MLX/MLX Metal 0.32.1, required by the compiled plugin ABI.
+- MLX-LM from commit `9e6acca691e64d6d8bb808c328fcdea459099cca` (reports 0.32.0).
+- Isolated environment with 159 pinned dependencies. Existing MLX-LM and system Python
+ are unchanged. Official wheel hashes are retained in requirements and lock files.
+- Runtime selection confirms **MetalPlatform**. The doctor executes and evaluates
+ a native reshape-and-cache Metal operation; real inference also succeeded.
+
+The `+cpu` core-wheel suffix is packaging, not the selected inference device. The
+plugin supplies native Metal paged-attention kernels and MLX execution. This avoids
+a local vLLM source build for the tested stable release.
+
+Pinned 0.29.0 still supports `VLLM_METAL_MEMORY_FRACTION`, although newer main-branch
+configuration documentation describes its removal. Set it to `auto` so the CLI
+`--gpu-memory-utilization` controls memory planning. The budget is a fraction of
+Metal's recommended working set, not a percentage of total system RAM. Default 0.1
+worked for the small demo model; larger models are not implied to fit that budget.
+
+## Real evidence
+
+Tested model: `mlx-community/Qwen3-0.6B-4bit`, existing cached snapshot
+`73e3e38d981303bc594367cd910ea6eb48349da8`. We reused its weights; no duplicate model
+download was needed. Live checks verified an actual answer, three measured warm
+requests, concurrent serving, applying 1,024/2/no-prefix-cache and 2,048/4/prefix-cache
+configurations, preserving settings during chat, cancellation, recovery and unloading.
+
+The latest completed integration run recorded:
+
+| Workload | Throughput | p95 complete-request latency | Errors |
+| --- | ---: | ---: | ---: |
+| 8 requests, concurrency 1 | 179.0 aggregate tokens/s | 742 ms | 0 |
+| 8 requests, concurrency 4 | 586.3 aggregate tokens/s | 911 ms | 0 |
+
+Both used 2,048 context, four allowed sequences, 0.1 memory fraction, prefix cache on,
+up to 128 output tokens, seed 42, temperature 0 and reasoning disabled. An earlier
+pair measured 199.3 and 590.8 tokens/s. These are exploratory warm tests on a working
+Mac; native app build work and other applications were active. Prefix-cache history
+was not reset between tests. Do not infer an isolated 3.3× scaling guarantee.
+
+Single-request benchmark in the latest run: **204.0 client end-to-end tokens/s**, with
+**21 ms** median time to first token, across three warm samples. This includes prompt
+processing and HTTP; it is not comparable to MLX-LM's native generation rate. The
+small model's answers establish functionality, not quality.
+
+Raw samples, timestamps, settings, model revision and hardware are in
+[benchmarks.json](benchmarks.json); checks and a real answer are in
+[verification.json](verification.json). Earlier successful measurement records are
+preserved even when a later lifecycle check in that development run failed.
+
+## Validation and limits
+
+- 21 Node checks cover shared behavior, bounded controls, prompt/operation separation,
+ split UTF-8 SSE, native token counts, partial stream errors, real request overlap,
+ failure samples and persisted concurrency records.
+- Native API checks reject missing credentials, cross-origin requests and hostile
+ Host headers, including health and metrics routes. Every native route is guarded.
+- Viewer, scripts, stylesheet, official icon and favicon returned HTTP 200.
+- Manifest conformance, isolated setup, installed doctor and native Metal check passed.
+- 15 targeted native category/identity tests passed. The icon build was compiled,
+ locally signed, verified and installed with the previous app preserved.
+
+The backend inference and control path were tested. Browser visual/interaction QA
+was not requested. Native desktop control failed to start in this session, so a new
+native vLLM workspace could not be opened automatically. The local viewer remains
+available on port 4312 with its measured results. Native app navigation is
+**Local AI → vLLM → New project**.
+
+Only the named Qwen3 model has live qualification here. Uncached download/resume,
+other architectures, large-model memory fitting, remote/GPU servers, structured
+generation and training remain outside this qualification. Settings persist while
+the viewer runs and reset on its restart; recordings retain the settings used.
+
+Additional handoff check: cancellation was exercised after **4 of 32 measured
+requests** completed at concurrency 4. The server returned to idle, then a fresh
+server produced a real answer. This checks overlapping-request cancellation beyond
+the earlier warmup cancellation. [handoff.json](handoff.json) records the evidence
+and the native app handoff limitation. The agent-facing `serve` command also passed.
+
+## Primary sources
+
+- [vLLM Metal installation](https://docs.vllm.ai/projects/vllm-metal/en/latest/installation/)
+ — hardware, Python and prebuilt installation path.
+- [vLLM Metal repository](https://github.com/vllm-project/vllm-metal)
+ — platform integration, Metal kernels and serving architecture.
+- [Pinned 0.29.0 package metadata](https://github.com/vllm-project/vllm-metal/blob/v0.29.0/pyproject.toml)
+ — ABI pins and exact MLX-LM dependency.
+- [vLLM 0.29.0 release](https://github.com/vllm-project/vllm/releases/tag/v0.29.0)
+ and [vLLM Metal 0.29.0 release](https://github.com/vllm-project/vllm-metal/releases/tag/v0.29.0)
+ — downloaded official wheels.
+- [Model support matrix](https://docs.vllm.ai/projects/vllm-metal/en/latest/supported_models/)
+ — architecture support is backend-specific.
+- Installed 0.29.0 `vllm_metal/config.py`, `v1/cache_policy.py`, and the native CLI/API
+ sources were used to verify behavior where current main documentation differs.
+
+Public release metadata is preserved in the parent research directory. Refer to the
+primary source links above for upstream installation and configuration files.
diff --git a/docs/research/local-ai-harnesses/vllm/handoff.json b/docs/research/local-ai-harnesses/vllm/handoff.json
new file mode 100644
index 000000000..0b4d4e4f5
--- /dev/null
+++ b/docs/research/local-ai-harnesses/vllm/handoff.json
@@ -0,0 +1,51 @@
+{
+ "at": "2026-09-20T05:58:11.404Z",
+ "package": "local/vllm",
+ "viewer": "http://127.0.0.1:4312/?view=grid",
+ "workspace": "<pilot-checkout>/vllm",
+ "nativeAppInstalled": true,
+ "nativeWorkspaceOpened": false,
+ "nativeHandoffLimitation": "Desktop control failed to start; embedded browser unavailable.",
+ "concurrentCancellation": {
+ "jobId": "fa5914b0-affc-4f21-9fc1-945a2c4cc26c",
+ "status": "cancelled",
+ "observedDuringMeasurements": true,
+ "serverReleased": true
+ },
+ "recovery": {
+ "jobId": "01292d33-24fb-485e-9ff3-4645e41d27b9",
+ "output": "Bicycle brakes are essential for safety and control, preventing skidding and ensuring smooth movement on the road.",
+ "metrics": {
+ "model": "mlx-community/Qwen3-0.6B-4bit",
+ "tokens": 22,
+ "promptTokens": 25,
+ "tokensPerSecond": 87.40616253175972,
+ "firstTokenMs": 84.27175000001444,
+ "firstOutputMs": 84.2723330000299,
+ "totalMs": 251.6984999999986,
+ "thinking": false,
+ "doneReason": "stop",
+ "metric": "client_end_to_end",
+ "serving": {
+ "context": 2048,
+ "maxSequences": 4,
+ "memoryFraction": 0.1,
+ "prefixCaching": true
+ }
+ }
+ },
+ "serving": {
+ "context": 2048,
+ "maxSequences": 4,
+ "memoryFraction": 0.1,
+ "prefixCaching": true,
+ "status": "serving",
+ "model": "mlx-community/Qwen3-0.6B-4bit",
+ "processRssBytes": 2444001280,
+ "runningRequests": 0,
+ "waitingRequests": 0,
+ "cacheUsage": 0,
+ "prefixQueries": 25,
+ "prefixHits": 0
+ }
+}
\ No newline at end of file
diff --git a/docs/research/local-ai-harnesses/vllm/verification.json b/docs/research/local-ai-harnesses/vllm/verification.json
new file mode 100644
index 000000000..a01c1da83
--- /dev/null
+++ b/docs/research/local-ai-harnesses/vllm/verification.json
@@ -0,0 +1,152 @@
+{
+ "at": "2026-09-20T05:46:19.731Z",
+ "checks": [
+ "serve",
+ "private API auth, host and origin",
+ "viewer and official icon HTTP assets",
+ "chat",
+ "benchmark",
+ "load_test",
+ "load_test",
+ "serve",
+ "chat",
+ "serving settings apply and chat preserves them",
+ "active cancellation frees owned server",
+ "serve",
+ "chat",
+ "recovery after cancellation",
+ "unload_all"
+ ],
+ "runtime": {
+ "id": "vllm",
+ "name": "vLLM",
+ "online": true,
+ "version": "0.29.0+cpu",
+ "backend": "Metal",
+ "backendVersion": "0.29.0",
+ "mlxVersion": "0.32.1",
+ "models": [
+ {
+ "id": "HuggingFaceTB/SmolLM2-135M-Instruct",
+ "name": "SmolLM2-135M-Instruct",
+ "path": "<home>/.cache/huggingface/hub/models--HuggingFaceTB--SmolLM2-135M-Instruct/snapshots/12fd25f77366fa6b3b4b768ec3050bf629380bac",
+ "digest": "12fd25f77366fa6b3b4b768ec3050bf629380bac",
+ "size": 272437573,
+ "family": "llama",
+ "parameters": null,
+ "quantization": "bfloat16",
+ "supportsThinking": false,
+ "running": false,
+ "resident": null
+ },
+ {
+ "id": "Qwen/Qwen2.5-1.5B-Instruct",
+ "name": "Qwen2.5-1.5B-Instruct",
+ "path": "<home>/.cache/huggingface/hub/models--Qwen--Qwen2.5-1.5B-Instruct/snapshots/989aa7980e4cf806f80c7fef2b1adb7bc71aa306",
+ "digest": "989aa7980e4cf806f80c7fef2b1adb7bc71aa306",
+ "size": 3098955668,
+ "family": "qwen2",
+ "parameters": null,
+ "quantization": "bfloat16",
+ "supportsThinking": false,
+ "running": false,
+ "resident": null
+ },
+ {
+ "id": "mlx-community/Qwen2.5-0.5B-Instruct-4bit",
+ "name": "Qwen2.5-0.5B-Instruct-4bit",
+ "path": "<home>/.cache/huggingface/hub/models--mlx-community--Qwen2.5-0.5B-Instruct-4bit/snapshots/a5339a4131f135d0fdc6a5c8b5bbed2753bbe0f3",
+ "digest": "a5339a4131f135d0fdc6a5c8b5bbed2753bbe0f3",
+ "size": 289598797,
+ "family": "qwen2",
+ "parameters": null,
+ "quantization": "4-bit",
+ "supportsThinking": false,
+ "running": false,
+ "resident": null
+ },
+ {
+ "id": "mlx-community/Qwen2.5-1.5B-Instruct-4bit",
+ "name": "Qwen2.5-1.5B-Instruct-4bit",
+ "path": "<home>/.cache/huggingface/hub/models--mlx-community--Qwen2.5-1.5B-Instruct-4bit/snapshots/8b403126fc14f14cfc99bb4cfa72ecbc129ea677",
+ "digest": "8b403126fc14f14cfc99bb4cfa72ecbc129ea677",
+ "size": 880169797,
+ "family": "qwen2",
+ "parameters": null,
+ "quantization": "4-bit",
+ "supportsThinking": false,
+ "running": false,
+ "resident": null
+ },
+ {
+ "id": "mlx-community/Qwen2.5-7B-Instruct-4bit",
+ "name": "Qwen2.5-7B-Instruct-4bit",
+ "path": "<home>/.cache/huggingface/hub/models--mlx-community--Qwen2.5-7B-Instruct-4bit/snapshots/c26a38f6a37d0a51b4e9a1eb3026530fa35d9fed",
+ "digest": "c26a38f6a37d0a51b4e9a1eb3026530fa35d9fed",
+ "size": 4295887610,
+ "family": "qwen2",
+ "parameters": null,
+ "quantization": "4-bit",
+ "supportsThinking": false,
+ "running": false,
+ "resident": null
+ },
+ {
+ "id": "mlx-community/Qwen3-0.6B-4bit",
+ "name": "Qwen3-0.6B-4bit",
+ "path": "<home>/.cache/huggingface/hub/models--mlx-community--Qwen3-0.6B-4bit/snapshots/73e3e38d981303bc594367cd910ea6eb48349da8",
+ "digest": "73e3e38d981303bc594367cd910ea6eb48349da8",
+ "size": 351383618,
+ "family": "qwen3",
+ "parameters": null,
+ "quantization": "4-bit",
+ "supportsThinking": true,
+ "running": false,
+ "resident": null
+ },
+ {
+ "id": "mlx-community/SmolLM2-135M-Instruct",
+ "name": "SmolLM2-135M-Instruct",
+ "path": "<home>/.cache/huggingface/hub/models--mlx-community--SmolLM2-135M-Instruct/snapshots/422de227b90002f443a21a58b1087f6ee7632731",
+ "digest": "422de227b90002f443a21a58b1087f6ee7632731",
+ "size": 273874876,
+ "family": "llama",
+ "parameters": null,
+ "quantization": "bfloat16",
+ "supportsThinking": false,
+ "running": false,
+ "resident": null
+ }
+ ],
+ "serving": {
+ "context": 2048,
+ "maxSequences": 4,
+ "memoryFraction": 0.1,
+ "prefixCaching": true,
+ "status": "idle",
+ "model": null,
+ "processRssBytes": null
+ }
+ },
+ "chat": {
+ "output": "A bicycle needs brakes to stop the bicycle while moving in a traffic at a speed that is higher than the maximum speed of the bicycle.",
+ "metrics": {
+ "model": "mlx-community/Qwen3-0.6B-4bit",
+ "tokens": 28,
+ "promptTokens": 23,
+ "tokensPerSecond": 96.30713721875337,
+ "firstTokenMs": 60.79104099999677,
+ "firstOutputMs": 60.86099899999681,
+ "totalMs": 290.73649999999907,
+ "thinking": false,
+ "doneReason": "stop",
+ "metric": "client_end_to_end",
+ "serving": {
+ "context": 2048,
+ "maxSequences": 4,
+ "memoryFraction": 0.1,
+ "prefixCaching": true
+ }
+ }
+ }
+}
\ No newline at end of file
diff --git a/store/agents/autonomous-grid/harness.json b/store/agents/autonomous-grid/harness.json
index b96f00c8b..08f6a4faf 100644
--- a/store/agents/autonomous-grid/harness.json
+++ b/store/agents/autonomous-grid/harness.json
@@ -2,7 +2,7 @@
"spec": 1,
"id": "autonomous/autonomous-grid",
"name": "Grid",
- "category": "Compute",
+ "category": "Local AI",
"author": "Autonomous",
"description": "Talk to your fleet. Deploy open-weight models, find the right machine for each workload, and watch your Grid come alive with a live map of engines, models, memory, temperature and performance.",
"engine": "codex",
diff --git a/store/agents/mlx-lm/.gitignore b/store/agents/mlx-lm/.gitignore
new file mode 100644
index 000000000..bfeee06c9
--- /dev/null
+++ b/store/agents/mlx-lm/.gitignore
@@ -0,0 +1,10 @@
+.harness/
+.venv/
+.runtime/
+__pycache__/
+*.pyc
+*.log
+node_modules/
+.DS_Store
+!/dist/
+!/dist/**
diff --git a/store/agents/mlx-lm/LICENSE b/store/agents/mlx-lm/LICENSE
new file mode 100644
index 000000000..a499cd17e
--- /dev/null
+++ b/store/agents/mlx-lm/LICENSE
@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Autonomous, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
diff --git a/store/agents/mlx-lm/README.md b/store/agents/mlx-lm/README.md
new file mode 100644
index 000000000..7d8027558
--- /dev/null
+++ b/store/agents/mlx-lm/README.md
@@ -0,0 +1,87 @@
+# MLX-LM harness
+
+Operate MLX-LM directly by talking to Codex. The viewer shows cached models, the active
+model, memory allocation, live jobs, generated text and measured performance. It shares
+the Ollama pilot's control plane and viewer logic, with MLX-specific identity and behavior.
+
+Install with `harness dsh install autonomous/mlx-lm` in **Local AI**. Open MLX-LM in Harness and create a
+workspace, or start `./viewer.sh` for the standalone viewer at
+`http://127.0.0.1:4311/?view=grid`. Omit `?view=grid` to see the optional bounded command
+panel. General natural-language operation comes from the Harness agent.
+
+Try “run a lightweight model”, “benchmark it”, “compare it with Qwen 2.5 0.5B”, or
+“free up memory”. No vendor CLI knowledge is required.
+
+## Direct integration
+
+The harness owns a private Python worker using official `mlx_lm.load` and `stream_generate`
+APIs. Its isolated environment pins MLX-LM 0.31.3 and MLX 0.32.2, with all dependencies in
+`requirements.lock`. It runs locally on Apple Silicon macOS. No other model-management
+application or inference API key is needed.
+
+The agent uses `toolchain/local-ai`; `toolchain/mlx-lm` exposes the vendor CLI for diagnosis.
+The viewer uses a loopback-only control service with Host, Origin and control-token checks.
+It does not expose an OpenAI-compatible serving endpoint. Codex itself continues to use
+its configured provider; this harness does not make the operator's reasoning local.
+
+## Models and lifecycle
+
+- Hugging Face `owner/repository` IDs identify models. We reuse the existing HF cache and
+ keep a workspace revision registry; other applications' cache references are untouched.
+- Downloads are pinned to the fetched repository revision, use safetensors and tokenizer/
+ configuration files, and never enable remote model Python. Partial downloads can resume.
+- The first demonstration model is `mlx-community/Qwen3-0.6B-4bit`, about 351 MB including
+ tokenizer/configuration data at the verified revision. Small models have limited quality.
+- The worker keeps one model loaded at a time. Switching releases the old model; a
+ 15-minute idle timeout or closing the viewer also releases it. Downloaded weights remain.
+- Cancelling a running request stops this workspace's worker and frees its model. The
+ worker restarts automatically for subsequent requests. Other applications are unaffected.
+- Discovery lists complete cached MLX-community text snapshots, MLX-named repositories,
+ and repositories explicitly downloaded by this workspace. It is not an inventory of
+ every possible MLX-compatible model or of other applications' resident models.
+- This pilot does not delete models, fine-tune, quantize, load adapters, serve multiple
+ concurrent requests, or run multimodal models. These require deliberate future work.
+
+Downloads larger than 80% of physical RAM are rejected and require 2 GiB of disk headroom.
+This is a conservative download check, not a complete model-fit estimator.
+
+## Measurements
+
+Chat uses a 4,096-token context budget and up to 512 generated tokens including reasoning.
+Benchmarks discard one 32-token warm-up, then measure three runs with a 2,048-token context,
+128-token output budget, seed 42, temperature zero and thinking disabled where supported.
+The model stays warm; each sample uses a fresh prompt cache.
+
+Generation speed is MLX-LM's native token rate. First-token time is observed by the harness
+and includes local IPC. First visible answer time excludes hidden reasoning and leading
+whitespace. Native prompt rate, generated token count, finish reason and peak allocation
+are retained. MLX allocation is neither process RSS nor a separate physical VRAM pool.
+
+The included measurements are short warm inference samples on this M2 Max, not quality
+scores or controlled cross-engine rankings. Ollama benchmarks can reuse the prompt cache
+and use different timing definitions. See [raw results](../../../docs/research/local-ai-harnesses/mlx-lm/benchmarks.json)
+and [live validation](../../../docs/research/local-ai-harnesses/mlx-lm/verification.json).
+
+## Development
+
+Shared sources live in `store/tools/local-ai/src`, `bin` and `dist`. Run `npm run build:mlx` from
+`store/tools/local-ai` to regenerate this package's copies and its branded viewer. Edit `worker.py`,
+assets, manifest, agent instructions and toolchain here. Do not edit generated copies.
+
+Run `toolchain/setup.sh` to install, `toolchain/doctor.sh` to inspect readiness,
+`npm test` from `store/tools/local-ai` for regression checks, and `npm run test:mlx-live` for opt-in
+GPU checks against this package's running standalone viewer. Live checks load and unload
+the two small baseline models, benchmark them and exercise cancellation.
+
+`.harness/` stores endpoint discovery, jobs, generated output, measurements, the diagnostic
+log and selected HF revisions. Chat prompts are not persisted by the controller, but model
+output is. Benchmarks and logs belong to the workspace. Framework workspaces must remain
+separate; their common state format does not authorize concurrent writers.
+
+## Credit and stewardship
+
+The native framework and official mark belong to [Apple’s MLX and MLX-LM projects](https://github.com/ml-explore/mlx-lm).
+OpenHarness contributors maintain this independent integration, its agent instructions
+and viewer. The harness is MIT-licensed; see [LICENSE](LICENSE). Upstream notices
+and license texts are recorded in [THIRD_PARTY.md](THIRD_PARTY.md) and `licenses/`.
+Model weights are downloaded separately under their respective model licenses.
diff --git a/store/agents/mlx-lm/THIRD_PARTY.md b/store/agents/mlx-lm/THIRD_PARTY.md
new file mode 100644
index 000000000..82bcf9c19
--- /dev/null
+++ b/store/agents/mlx-lm/THIRD_PARTY.md
@@ -0,0 +1,10 @@
+# Attribution
+
+This independently maintained OpenHarness integration is MIT-licensed; see LICENSE.
+It is not an official application of the upstream framework. Upstream names and
+marks identify the tools being operated and remain their owners’ marks.
+
+Official logo source and retrieval date are recorded in assets/README.md.
+Upstream framework license texts are included in licenses/. The setup scripts use
+the upstream distributions; Python and runtime dependencies retain their own licenses.
+Model weights are not included and retain their individual model licenses.
diff --git a/store/agents/mlx-lm/agent/AGENTS.md b/store/agents/mlx-lm/agent/AGENTS.md
new file mode 100644
index 000000000..4adb5eea3
--- /dev/null
+++ b/store/agents/mlx-lm/agent/AGENTS.md
@@ -0,0 +1,53 @@
+# MLX-LM model operator
+
+These instructions apply to a workspace whose `local-ai.json` names `mlx-lm`.
+You are its natural-language interface; the left viewer shows real state and measurements.
+Use `$LOCAL_AI` for normal operations so progress and results appear in that viewer.
+
+```
+"$LOCAL_AI" status
+"$LOCAL_AI" action deploy mlx-community/Qwen3-0.6B-4bit
+"$LOCAL_AI" action chat mlx-community/Qwen3-0.6B-4bit "Explain unified memory in three sentences."
+"$LOCAL_AI" action benchmark mlx-community/Qwen3-0.6B-4bit mlx-community/Qwen2.5-0.5B-Instruct-4bit
+"$LOCAL_AI" action unload_all
+"$LOCAL_AI" cancel JOB_ID
+```
+
+Use typed actions for requests you have interpreted; do not generate arbitrary shell commands.
+Model identifiers are Hugging Face `owner/repository` names, not Ollama tags. Start with
+status. If the viewer is unavailable, open it or run this package's viewer command with
+`HARNESS_WORKSPACE` set to the workspace. Do not start conflicting writers in `.harness`.
+If the environment is missing, run the package's setup command; it installs pinned packages
+inside its own `.venv`, leaving system Python unchanged.
+
+This harness starts a private Python worker and calls the official `mlx_lm.load` and
+`stream_generate` APIs. `$MLX_LM generate --help` exposes the vendor's native CLI for
+diagnosis. Managed operations must go through `$LOCAL_AI` so the viewer observes them.
+Do not build another agent or conversation pane in the embedded viewer.
+
+The pilot keeps **one model loaded per workspace**. Switching releases the old model;
+unload and the 15-minute idle timeout free its memory and keep downloaded weights.
+Other MLX apps and workspaces are outside this worker's memory/inventory ownership.
+Cancellation stops this worker, frees its loaded model and keeps resumable downloads.
+No model deletion, training, adapters, multimodal models, remote serving or cloud inference
+is exposed. Never enable `trust_remote_code` to make a model work.
+
+For an unspecified first model use `mlx-community/Qwen3-0.6B-4bit`, a small demonstration
+model. Complete its first deployment with a real chat before declaring it working.
+Check actual output and report failures honestly. Small models have limited capability.
+
+Chat has a 4,096-token context and a 512-token output budget, including reasoning.
+Benchmarks use a discarded 32-token warmup and three measured 128-token runs, the same
+prompt, 2,048-token context, seed 42, temperature 0 and thinking off when supported.
+Each MLX run starts a fresh prompt cache; the model remains warm. Report native generation
+tokens/s, observed first-token latency (including local IPC), actual token counts, model
+revision, MLX-LM/MLX versions and native peak allocated memory. Do not equate these with
+answer quality or an isolated lab comparison. Ollama may reuse its prompt cache, so its
+results have different cache conditions and timing definitions.
+
+MLX active allocation uses shared Apple Silicon memory and is not total process RSS or
+a separate physical VRAM pool. Missing telemetry is unknown. Keep failed jobs as failed;
+never edit measurements or the verdict to turn a failure into a success.
+
+Weights remain in the standard Hugging Face cache. `.harness` stores this workspace's
+jobs, benchmarks, runtime log and endpoint record. Record lasting decisions in Markdown.
diff --git a/store/agents/mlx-lm/assets/README.md b/store/agents/mlx-lm/assets/README.md
new file mode 100644
index 000000000..eb813d8c6
--- /dev/null
+++ b/store/agents/mlx-lm/assets/README.md
@@ -0,0 +1,3 @@
+# MLX identity
+
+Official MLX wordmark from https://github.com/ml-explore/mlx/blob/main/docs/logo/mlx_logo.svg (retrieved 2026-09-20). Centered unchanged on a light square tile for the viewer and native app. This is a local third-party harness, not an official MLX application.
diff --git a/store/agents/mlx-lm/assets/mlx-icon.png b/store/agents/mlx-lm/assets/mlx-icon.png
new file mode 100644
index 000000000..384b343dd
Binary files /dev/null and b/store/agents/mlx-lm/assets/mlx-icon.png differ
diff --git a/store/agents/mlx-lm/assets/mlx-icon.svg b/store/agents/mlx-lm/assets/mlx-icon.svg
new file mode 100644
index 000000000..5effa238c
--- /dev/null
+++ b/store/agents/mlx-lm/assets/mlx-icon.svg
@@ -0,0 +1,34 @@
+<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" width="512" height="512" viewBox="0 0 512 512"><rect width="512" height="512" rx="112" fill="#f8f9fc"/><g transform="translate(39 186)"><defs>
+<g>
+<g id="glyph-0-0">
+<path d="M 9.6875 -171.1875 L 188.609375 -171.1875 L 188.609375 -188.8125 L 9.6875 -188.8125 Z M 9.6875 -127.421875 L 188.609375 -127.421875 L 188.609375 -145.046875 L 9.6875 -145.046875 Z M 9.6875 -83.5625 L 188.609375 -83.5625 L 188.609375 -101.1875 L 9.6875 -101.1875 Z M 9.6875 -39.796875 L 188.609375 -39.796875 L 188.609375 -57.421875 L 9.6875 -57.421875 Z M 9.6875 4.0625 L 188.609375 4.0625 L 188.609375 -13.5625 L 9.6875 -13.5625 Z M 9.6875 47.828125 L 188.609375 47.828125 L 188.609375 30.203125 L 9.6875 30.203125 Z M 9.6875 47.828125 "/>
+</g>
+<g id="glyph-0-1">
+<path d="M 13.9375 0 L 42.3125 0 L 42.3125 -91.796875 L 43.671875 -91.796875 L 78.71875 0 L 97.984375 0 L 133.03125 -91.796875 L 134.390625 -91.796875 L 134.390625 0 L 162.765625 0 L 162.765625 -139.71875 L 125.96875 -139.71875 L 88.984375 -42.015625 L 87.8125 -42.015625 L 50.734375 -139.71875 L 13.9375 -139.71875 Z M 13.9375 0 "/>
+</g>
+<g id="glyph-0-2">
+<path d="M 13.9375 0 L 106.21875 0 L 106.21875 -26.046875 L 45.984375 -26.046875 L 45.984375 -139.71875 L 13.9375 -139.71875 Z M 13.9375 0 "/>
+</g>
+<g id="glyph-0-3">
+<path d="M 6.296875 0 L 40.5625 0 L 69.515625 -47.34375 L 70.484375 -47.34375 L 99.625 0 L 135.84375 0 L 91.109375 -69.8125 L 91.109375 -70.296875 L 136.328125 -139.71875 L 100.703125 -139.71875 L 73 -90.71875 L 71.84375 -90.71875 L 43.953125 -139.71875 L 6.484375 -139.71875 L 49.96875 -70.6875 L 49.96875 -70.296875 Z M 6.296875 0 "/>
+</g>
+</g>
+<clipPath id="clip-0">
+<path clip-rule="nonzero" d="M 13 0 L 283 0 L 283 139.71875 L 13 139.71875 Z M 13 0 "/>
+</clipPath>
+<clipPath id="clip-1">
+<path clip-rule="nonzero" d="M 296 0 L 427 0 L 427 139.71875 L 296 139.71875 Z M 296 0 "/>
+</clipPath>
+</defs>
+<g clip-path="url(#clip-0)">
+<g fill="rgb(0%, 0%, 0%)" fill-opacity="1">
+<use xlink:href="#glyph-0-1" x="0" y="139.72"/>
+<use xlink:href="#glyph-0-2" x="176.682092" y="139.72"/>
+</g>
+</g>
+<g clip-path="url(#clip-1)">
+<g fill="rgb(82.998657%, 82.998657%, 82.998657%)" fill-opacity="1">
+<use xlink:href="#glyph-0-3" x="290.57" y="139.72"/>
+</g>
+</g>
+</g></svg>
diff --git a/store/agents/mlx-lm/bin/harness.mjs b/store/agents/mlx-lm/bin/harness.mjs
new file mode 100755
index 000000000..16b63570d
--- /dev/null
+++ b/store/agents/mlx-lm/bin/harness.mjs
@@ -0,0 +1,54 @@
+#!/usr/bin/env node
+import { readFile } from 'node:fs/promises';
+import { resolve, join, dirname } from 'node:path';
+import { fileURLToPath } from 'node:url';
+const args = process.argv.slice(2);
+const runtimeName = process.env.LOCAL_AI_NAME || 'Ollama';
+const runtime = process.env.LOCAL_AI_RUNTIME || 'ollama';
+if (!args.length) { console.log(`Talk to the ${runtimeName} agent, or use:\n local-ai "run a lightweight model"\n local-ai status\n local-ai action deploy MODEL\n local-ai action benchmark MODEL [MODEL…]\n local-ai action chat MODEL "PROMPT"\n local-ai action unload MODEL\n local-ai action-json '{"type":"serve","model":"OWNER/REPO","maxSequences":4}' (vLLM)\n local-ai cancel JOB_ID`); process.exit(0); }
+const packageRoot = dirname(dirname(fileURLToPath(import.meta.url)));
+const workspace = resolve(process.env.HARNESS_WORKSPACE || packageRoot);
+let port = Number(process.env.HARNESS_PORT || ({ 'mlx-lm': 4311, vllm: 4312 }[runtime] || 4310));
+try { const endpoint = JSON.parse(await readFile(join(workspace, '.harness', 'endpoint.json'), 'utf8')); if (Number.isInteger(endpoint.port) && endpoint.port > 0 && endpoint.port <= 65535 && endpoint.workspace === workspace) port = endpoint.port; } catch {}
+const base = `http://127.0.0.1:${port}`;
+const get = async path => { const response = await fetch(`${base}${path}`, { signal: AbortSignal.timeout(15000) }); const value = await response.json(); if (!response.ok) throw new Error(value.error || `HTTP ${response.status}`); return value; };
+let currentJob;
+try {
+ const health = await get('/api/health');
+ if (health.application !== 'local-ai-harnesses' || health.workspace !== workspace || (health.runtime || 'ollama') !== runtime) throw new Error(`This endpoint belongs to another workspace or framework. Open the ${runtimeName} viewer for this workspace.`);
+ const state = await get('/api/state');
+ const post = async (path, body) => {
+ const response = await fetch(`${base}${path}`, { method: 'POST', headers: { 'Content-Type': 'application/json', 'X-Harness-Token': state.token }, body: JSON.stringify(body), signal: AbortSignal.timeout(15000) });
+ const value = await response.json(); if (!response.ok) throw new Error(value.error || `HTTP ${response.status}`); return value;
+ };
+ if (args[0] === 'status' && args.length === 1) {
+ const { token, ...publicState } = state; console.log(JSON.stringify(publicState, null, 2)); process.exit(0);
+ }
+ if (args[0] === 'cancel') { const { job } = await post('/api/cancel', { id: args[1] }); console.log(JSON.stringify({ id: job.id, status: job.status, message: job.message }, null, 2)); process.exit(0); }
+ let result;
+ if (args[0] === 'action-json') {
+ if (args.length !== 2) throw new Error('Pass one JSON action object.');
+ result = await post('/api/action', JSON.parse(args[1]));
+ } else if (args[0] === 'action') {
+ const type = args[1];
+ const body = type === 'benchmark' ? { type, models: args.slice(2) } : type === 'chat' ? { type, model: args[2], prompt: args.slice(3).join(' ') } : { type, model: args[2] };
+ result = await post('/api/action', body);
+ } else result = await post('/api/command', { text: args.join(' ') });
+ if (!result.job) { console.log(result.message || JSON.stringify(result, null, 2)); if (result.models) console.log(result.models.map(m => `${m.id}: ${m.description}`).join('\n')); process.exit(0); }
+ currentJob = result.job.id;
+ console.log(`Job ${currentJob}`);
+ process.once('SIGINT', async () => { try { await post('/api/cancel', { id: currentJob }); console.log('Cancellation requested.'); } finally { process.exit(130); } });
+ let last;
+ while (true) {
+ const { jobs } = await get('/api/jobs');
+ const job = jobs.find(j => j.id === currentJob);
+ if (!job) throw new Error('The job could not be found. Check the viewer.');
+ if (job.message !== last) { console.log(job.message); last = job.message; }
+ if (!['queued', 'running'].includes(job.status)) {
+ if (job.output) console.log(job.output);
+ if (job.result?.metrics) console.log(JSON.stringify(job.result.metrics, null, 2));
+ process.exit(job.status === 'succeeded' ? 0 : 1);
+ }
+ await new Promise(resolve => setTimeout(resolve, 1000));
+ }
+} catch (error) { console.error(error.cause?.code === 'ECONNREFUSED' ? `The viewer is not running. Open the ${runtimeName} harness viewer.` : `${error.message}${currentJob ? ` (job ${currentJob}; check its status before retrying)` : ''}`); process.exitCode = 1; }
diff --git a/store/agents/mlx-lm/dist/app.js b/store/agents/mlx-lm/dist/app.js
new file mode 100644
index 000000000..533f5cded
--- /dev/null
+++ b/store/agents/mlx-lm/dist/app.js
@@ -0,0 +1,209 @@
+const $ = selector => document.querySelector(selector);
+const escapeHTML = value => String(value ?? '').replace(/[&<>"']/g, c => ({ '&': '&amp;', '<': '&lt;', '>': '&gt;', '"': '&quot;', "'": '&#39;' }[c]));
+const gib = bytes => ((bytes || 0) / 1024 ** 3).toFixed(1);
+const number = (value, digits = 1) => Number.isFinite(value) ? value.toFixed(digits) : '—';
+const active = job => ['queued', 'running'].includes(job.status);
+let state;
+let selectedModel = sessionStorage.getItem('harness-model') || '';
+let pending = false;
+let modelSignature = '';
+let benchmarkSignature = '';
+let initialized = false;
+let lastReceived = Date.now();
+const jobElements = new Map();
+const jobSignatures = new Map();
+const chatHistory = new Map();
+const chatRequests = new Map();
+const completed = new Set();
+const embedded = new URLSearchParams(location.search).get('view') === 'grid';
+document.body.classList.toggle('embedded-viewer', embedded);
+
+function toast(text) { const el = $('#toast'); el.textContent = text; el.hidden = false; clearTimeout(toast.timer); toast.timer = setTimeout(() => { el.hidden = true; }, 4500); }
+function scrollConversation() { const el = $('#conversation'); el.scrollTop = el.scrollHeight; }
+function appendMessage(text, user = false) {
+ const el = document.createElement('div'); el.className = user ? 'user-message' : 'assistant-message';
+ if (user) el.textContent = text;
+ else el.innerHTML = `<span class="message-label">MLX-LM</span><p class="message-text">${escapeHTML(text)}</p>`;
+ $('#conversation').append(el); scrollConversation(); return el;
+}
+async function api(path, body) {
+ if (!state?.token) await refresh();
+ const response = await fetch(path, { method: 'POST', headers: { 'Content-Type': 'application/json', 'X-Harness-Token': state?.token || '' }, body: JSON.stringify(body) });
+ const result = await response.json();
+ if (!response.ok) { if (response.status === 403) void refresh(); throw new Error(result.error || 'The request failed.'); }
+ return result;
+}
+async function refresh() {
+ try {
+ const response = await fetch('/api/state'); if (!response.ok) throw new Error('Could not reach the MLX-LM viewer');
+ applyState(await response.json());
+ } catch (error) { $('#connection-status').textContent = 'MLX-LM viewer disconnected'; $('#operator-status').textContent = 'Reconnect to send a request'; return false; }
+ return true;
+}
+function applyState(next) {
+ lastReceived = Date.now(); state = { ...state, ...next };
+ if (selectedModel && !state.runtime.models.some(m => m.id === selectedModel)) selectedModel = '';
+ if (!selectedModel) selectedModel = state.runtime.models.find(m => m.running)?.id || '';
+ render();
+ if (!initialized) {
+ [...state.jobs].slice(0, 4).reverse().forEach(job => showJob(job, true)); initialized = true;
+ }
+ for (const job of state.jobs) if (jobElements.has(job.id) || active(job)) showJob(job);
+ renderActiveJob();
+ renderActivity();
+}
+function selectModel(model) {
+ selectedModel = model; sessionStorage.setItem('harness-model', model);
+ $('#selected-model').value = model;
+ $('#operator-status').textContent = model || 'MLX-LM harness';
+ document.querySelectorAll('[data-model]').forEach(el => el.classList.toggle('selected', el.dataset.model === model));
+}
+function render() {
+ const { runtime, system, benchmarks, runs } = state;
+ const models = runtime.models || [];
+ const loaded = models.filter(m => m.running);
+ const lastJob = state.jobs[0];
+ const notice = $('#runtime-notice');
+ notice.hidden = lastJob?.status !== 'failed';
+ if (!notice.hidden) {
+ const affected = lastJob.activeModel || lastJob.model || lastJob.models?.join(', ') || 'MLX-LM';
+ const explanation = /tensor.*size overflow/i.test(lastJob.message)
+ ? "MLX-LM could not read this model’s weights. Other models can still run."
+ : 'The last operation failed. Check the details before retrying.';
+ notice.innerHTML = `<strong>${escapeHTML(affected)} · ${escapeHTML(lastJob.type)} failed</strong><p>${escapeHTML(explanation)}</p><details><summary>Error details</summary><pre>${escapeHTML(lastJob.message)}</pre></details>`;
+ }
+ $('#connection-status').textContent = runtime.online ? 'MLX-LM connected' : 'MLX-LM offline';
+ $('#model-count').textContent = runtime.online ? models.length : '—';
+ $('#models-total').textContent = models.length;
+ $('#running-count').textContent = runtime.online ? loaded.length : '—';
+ $('#model-memory').innerHTML = `${runtime.online ? gib(loaded.reduce((sum, m) => sum + (m.resident?.size || 0), 0)) : '—'} <small>GiB</small>`;
+ const recent = runs[0];
+ $('#last-speed').innerHTML = `${number(recent?.tokensPerSecond)} <small>tok/s</small>`;
+ $('#last-speed').title = recent ? `${recent.model} · ${new Date(recent.at).toLocaleString()}` : 'A chat response will appear here';
+ $('#map-status').textContent = runtime.online ? 'LIVE CONNECTION' : 'READY TO START';
+ $('#map-status').classList.toggle('online', runtime.online);
+ $('#machine-info').textContent = `${system.chip} · ${Math.round(system.totalMemory / 1024 ** 3)} GiB`;
+ const signature = JSON.stringify({ online: runtime.online, version: runtime.version, models, selectedModel });
+ if (signature !== modelSignature) {
+ modelSignature = signature;
+ $('#runtime-map').innerHTML = `<div class="runtime-node ${runtime.online ? '' : 'offline'}"><img class="node-icon" src="/assets/mlx-icon.svg" width="42" height="42" alt=""><strong>MLX-LM</strong><small>${runtime.online ? `v${escapeHTML(runtime.version)}` : 'Not running'}</small></div><div class="model-nodes">${models.length ? models.map(m => `<button class="model-node ${m.running ? 'active' : ''} ${selectedModel === m.id ? 'selected' : ''}" data-model="${escapeHTML(m.id)}"><strong>${escapeHTML(m.name)}</strong><span>${m.running ? 'In memory' : 'On disk'} · ${gib(m.size)} GiB</span></button>`).join('') : `<div class="map-empty empty-node"><strong>Your first MLX-LM model starts here.</strong>Ask the operator to run your first model.<br>Its connection will appear here.</div>`}</div>`;
+ $('#models-list').innerHTML = models.length ? models.map(m => `<button class="model-row ${selectedModel === m.id ? 'selected' : ''}" data-model="${escapeHTML(m.id)}"><span class="model-title"><strong>${escapeHTML(m.name)}</strong><small>${escapeHTML([m.family, m.parameters, m.quantization].filter(Boolean).join(' · '))}</small></span><span class="model-status ${m.running ? 'loaded' : ''}">${m.running ? 'In memory' : 'On disk'}</span><span class="model-numeric">${gib(m.size)} GiB</span><span class="model-numeric">MLX-LM</span></button>`).join('') : `<div class="empty-small"><p>${runtime.online ? 'No models downloaded yet.' : 'Start MLX-LM to discover your models.'}</p><span>Say “run a lightweight model” to get started.</span></div>`;
+ $('#selected-model').innerHTML = `<option value="">Auto · running model</option>${models.map(m => `<option value="${escapeHTML(m.id)}">${escapeHTML(m.name)}</option>`).join('')}`;
+ $('#selected-model').value = selectedModel;
+ $('#operator-status').textContent = selectedModel || 'MLX-LM harness';
+ }
+ const benchSignature = JSON.stringify(benchmarks.map(b => b.id));
+ if (benchSignature !== benchmarkSignature) { benchmarkSignature = benchSignature; renderBenchmarks(); }
+ $('#export-benchmarks').hidden = benchmarks.length === 0;
+}
+function renderBenchmarks() {
+ const latest = [...new Map([...state.benchmarks].reverse().map(b => [b.model, b])).values()].sort((a, b) => b.medianTokensPerSecond - a.medianTokensPerSecond);
+ if (!latest.length) return;
+ const maxSpeed = Math.max(...latest.map(b => b.medianTokensPerSecond), 1);
+ const maxLatency = Math.max(...latest.map(b => b.medianFirstTokenMs), 1);
+ const bars = (key, max, unit) => latest.map(b => `<div class="chart-row"><span>${escapeHTML(b.model.split("/").at(-1))}</span><div class="bar-track"><div class="bar-fill ${key === 'medianFirstTokenMs' ? 'latency' : ''}" style="width:${Math.max(0, b[key] / max * 100)}%"></div></div><span class="bar-value">${number(b[key], key === 'medianFirstTokenMs' ? 0 : 1)} <small>${unit}</small></span></div>`).join('');
+ $('#performance-content').innerHTML = `<div class="chart-heading"><h3>Generation speed</h3><span>tokens/s · higher is faster</span></div><div class="chart-bars" role="group" aria-label="Measured median generation speed">${bars('medianTokensPerSecond', maxSpeed, '')}</div><div class="chart-heading second-chart"><h3>Time to first token</h3><span>milliseconds · lower is faster</span></div><div class="chart-bars" role="group" aria-label="Measured median time to first token">${bars('medianFirstTokenMs', maxLatency, '')}</div><p class="benchmark-meta">Latest benchmark per model · median of 3 warm runs<br>Same prompt · 2,048-token context · up to 128 generated tokens · seed 42<br>Thinking off when supported; actual mode is recorded per run.<br>Each sample starts with a fresh prompt cache. Speed measures inference, not answer quality.</p><details class="benchmark-details"><summary>Samples and measurement details</summary>${latest.map(b => `<div class="sample-record"><strong>${escapeHTML(b.model)}</strong><span>${new Date(b.at).toLocaleString()} · MLX-LM ${escapeHTML(b.runtime.version)}</span><span>Speed: ${b.samples.map(s => number(s.tokensPerSecond)).join(' / ')} tokens/s</span><span>First token: ${b.samples.map(s => number(s.firstTokenMs, 0)).join(' / ')} ms</span><span>Tokens generated: ${b.samples.map(s => s.tokens).join(' / ')} · thinking: ${escapeHTML(String(b.samples[0].thinking ?? 'not reported'))} · allocation: ${gib(b.residentBytes)} GiB</span></div>`).join('')}</details>`;
+}
+function showJob(job, historical = false) {
+ let el = jobElements.get(job.id);
+ if (!el) {
+ el = document.createElement('div'); el.className = 'assistant-message job-message';
+ jobElements.set(job.id, el); $('#conversation').append(el);
+ }
+ const signature = JSON.stringify(job);
+ if (jobSignatures.get(job.id) === signature) return;
+ jobSignatures.set(job.id, signature);
+ const failed = ['failed', 'cancelled', 'interrupted'].includes(job.status);
+ const metrics = job.result?.metrics;
+ const label = job.type === 'chat' ? job.model : 'MLX-LM';
+ el.innerHTML = `<span class="message-label">${escapeHTML(label)} <span class="job-state ${failed ? 'error-text' : ''}">${escapeHTML(job.status)}</span></span><p class="message-text ${failed ? 'error-text' : ''}">${escapeHTML(job.message)}</p>${job.output ? `<div class="stream-output message-text">${escapeHTML(job.output)}</div>` : ''}${metrics ? `<p class="run-metrics">${number(metrics.tokensPerSecond)} tok/s · ${number(metrics.firstTokenMs, 0)} ms first token · ${metrics.tokens ?? '—'} tokens${metrics.thinking ? ` · ${number(metrics.firstOutputMs, 0)} ms to visible output · reasoning on` : ''}${metrics.peakMemoryBytes ? ` · ${gib(metrics.peakMemoryBytes)} GiB peak allocation` : ''}${metrics.doneReason === 'length' ? ' · output limit reached' : ''}</p>` : ''}${job.persistenceError ? '<p class="error-text">Could not save this result to disk.</p>' : ''}${job.status === 'succeeded' && job.type === 'deploy' ? `<div class="actions"><button data-prompt="Benchmark ${escapeHTML(job.model)}">Benchmark it</button><button data-chat="${escapeHTML(job.model)}">Ask a question</button></div>` : ''}${job.status === 'failed' ? '<p class="secondary">You can retry the request after addressing the error.</p>' : ''}`;
+ if (job.status === 'succeeded' && !completed.has(job.id)) {
+ completed.add(job.id);
+ if (!historical && ['deploy', 'chat', 'benchmark'].includes(job.type) && job.result?.model) selectModel(job.result.model);
+ const request = chatRequests.get(job.id);
+ if (request) {
+ const history = chatHistory.get(job.model) || [];
+ history.push({ role: 'user', content: request }, { role: 'assistant', content: job.output });
+ chatHistory.set(job.model, history.slice(-6).map(m => ({ ...m, content: m.content.slice(0, 6000) }))); chatRequests.delete(job.id);
+ }
+ }
+ if (!historical) scrollConversation();
+}
+function renderActiveJob() {
+ const running = state.jobs.find(j => j.status === 'running') || state.jobs.find(j => j.status === 'queued');
+ const panel = $('#job-panel'); panel.hidden = !running;
+ if (!running) return;
+ const queued = state.jobs.filter(j => j.status === 'queued').length;
+ panel.innerHTML = `<div class="job-top"><strong>${escapeHTML(running.type === 'chat' ? 'Generating' : running.type.replaceAll('_', ' '))}</strong><button data-cancel="${escapeHTML(running.id)}">Cancel</button></div><div class="progress-track" role="progressbar" aria-label="Job progress" aria-valuemin="0" aria-valuemax="100" aria-valuenow="${Math.round(running.progress)}"><div class="progress-fill" style="width:${running.progress}%"></div></div><p>${escapeHTML(running.message)}${queued ? ` · ${queued} queued` : ''}</p>`;
+}
+function receiveJob(job) {
+ if (!state) return;
+ state.jobs = [job, ...state.jobs.filter(j => j.id !== job.id)];
+ showJob(job); renderActiveJob(); renderActivity();
+}
+function renderActivity() {
+ const jobs = state.jobs.slice(0, 6);
+ $('#activity-content').innerHTML = jobs.length ? jobs.map(job => `<div class="activity-row"><span class="activity-state ${job.status === 'failed' ? 'error-text' : ''}">${escapeHTML(job.status)}</span><div><strong>${escapeHTML(job.model || job.models.join(', ') || 'MLX-LM')}</strong><p>${escapeHTML(job.message)}</p></div>${active(job) ? `<button data-cancel="${escapeHTML(job.id)}">Cancel</button>` : '<span class="activity-time">' + new Date(job.completedAt || job.createdAt).toLocaleTimeString([], { hour: '2-digit', minute: '2-digit' }) + '</span>'}</div>`).join('') : '<p class="secondary">Ask the agent to run a model. Its progress will appear here.</p>';
+}
+async function sendCommand(text) {
+ if (pending || !text.trim()) return;
+ pending = true; $('#send-button').disabled = true; $('#suggestions').hidden = true;
+ appendMessage(text, true); $('#command-input').value = '';
+ try {
+ const result = await api('/api/command', { text, selectedModel: selectedModel || undefined, history: chatHistory.get(selectedModel) || [] });
+ if (result.job) {
+ if (result.job.type === 'chat') { const prompt = text.match(/^(?:ask|tell|chat with)\s+.+?\s*:\s*([\s\S]+)$/i)?.[1] || text; chatRequests.set(result.job.id, prompt); }
+ receiveJob(result.job);
+ } else {
+ const el = appendMessage(result.message || 'Choose a model to get started.');
+ if (result.models) el.insertAdjacentHTML('beforeend', result.models.map(m => `<div class="catalog-card"><strong>${escapeHTML(m.name)}</strong><p>${escapeHTML(m.description)}</p><p class="secondary">About ${number(m.downloadGB, 2)} GB to download</p><button data-action="deploy" data-target="${escapeHTML(m.id)}">Run this model</button></div>`).join(''));
+ }
+ } catch (error) { appendMessage(error.message).classList.add('error-text'); }
+ finally { pending = false; $('#send-button').disabled = false; scrollConversation(); }
+}
+async function runAction(type, model) {
+ $('#model-dialog').close();
+ const action = type === 'benchmark' ? { type, models: [model] } : { type, model };
+ appendMessage(`${type === 'deploy' ? 'Run' : type[0].toUpperCase() + type.slice(1)} ${model}`, true);
+ try { const result = await api('/api/action', action); receiveJob(result.job); return { jobId: result.job.id, status: result.job.status }; }
+ catch (error) { appendMessage(error.message).classList.add('error-text'); throw error; }
+}
+function inspectModel(id) {
+ const model = state.runtime.models.find(m => m.id === id); if (!model) return;
+ selectModel(id);
+ const resident = model.resident;
+ $('#model-detail').innerHTML = `<p class="eyebrow">MLX-LM / MODEL DETAILS</p><h2 id="detail-title">${escapeHTML(model.name)}</h2><p class="secondary">${model.running ? 'Loaded and ready for inference' : 'Downloaded and available to load'}</p><div class="detail-grid"><div><span>Model parameters</span><strong>${escapeHTML(model.parameters || 'Unknown')}</strong></div><div><span>Quantization</span><strong>${escapeHTML(model.quantization || 'Unknown')}</strong></div><div><span>Size on disk</span><strong>${gib(model.size)} GiB</strong></div><div><span>Runtime allocation</span><strong>${resident ? `${gib(resident.size)} GiB` : 'Not loaded'}</strong></div><div><span>Harness context budget</span><strong>${resident?.context_length?.toLocaleString() || 'Set when loaded'}</strong></div><div><span>Idle expiry</span><strong>${resident?.expires_at ? new Date(resident.expires_at).toLocaleTimeString() : '—'}</strong></div></div><p class="secondary">MLX reports active allocations for this workspace, excluding its reusable memory cache. On Apple Silicon, CPU and GPU share unified memory; this is not a separate physical VRAM pool.</p><div class="actions"><button class="primary" data-action="deploy" data-target="${escapeHTML(id)}">${model.running ? 'Keep ready for 15m' : 'Load model'}</button><button data-action="benchmark" data-target="${escapeHTML(id)}">Benchmark</button><button data-chat="${escapeHTML(id)}">Chat</button>${model.running ? `<button data-action="unload" data-target="${escapeHTML(id)}">Unload</button>` : ''}</div>`;
+ $('#model-dialog').showModal();
+}
+
+$('#refresh').addEventListener('click', async () => { if (await refresh()) toast('Model status refreshed'); });
+$('#selected-model').addEventListener('change', event => selectModel(event.target.value));
+$('#command-form').addEventListener('submit', event => { event.preventDefault(); void sendCommand($('#command-input').value); });
+$('#command-input').addEventListener('keydown', event => { if (event.key === 'Enter' && !event.shiftKey && !event.isComposing) { event.preventDefault(); $('#command-form').requestSubmit(); } });
+$('.dialog-close').addEventListener('click', () => $('#model-dialog').close());
+document.addEventListener('click', event => {
+ const button = event.target.closest('button'); if (!button) return;
+ if (button.dataset.prompt) void sendCommand(button.dataset.prompt);
+ if (button.dataset.model) inspectModel(button.dataset.model);
+ if (button.dataset.action) void runAction(button.dataset.action, button.dataset.target).catch(() => {});
+ if (button.dataset.chat) { selectModel(button.dataset.chat); $('#model-dialog').close(); $('#command-input').value = 'Ask it: '; $('#command-input').focus(); }
+ if (button.dataset.cancel) void api('/api/cancel', { id: button.dataset.cancel }).then(result => receiveJob(result.job)).catch(error => toast(error.message));
+});
+
+await refresh();
+const events = new EventSource('/api/events');
+events.addEventListener('state', event => { try { applyState(JSON.parse(event.data)); } catch { void refresh(); } });
+events.addEventListener('job', event => { lastReceived = Date.now(); receiveJob(JSON.parse(event.data)); });
+events.addEventListener('error', () => { $('#connection-status').textContent = 'Reconnecting…'; });
+setInterval(() => { if (Date.now() - lastReceived > 12000) void refresh(); }, 10000);
+
+// The same visible UI actions are available to supported agent browsers.
+const lifecycle = new AbortController();
+if (document.modelContext?.registerTool) {
+ const register = tool => document.modelContext.registerTool(tool, { signal: lifecycle.signal });
+ try {
+ await register({ name: 'harness_inventory', title: 'Inspect local models', description: 'Read MLX-LM models, residency, recent benchmarks and job status from this viewer.', inputSchema: { type: 'object', properties: {}, additionalProperties: false }, annotations: { readOnlyHint: true, untrustedContentHint: true }, async execute() { if (!await refresh()) throw new Error('The workspace service is disconnected.'); return { runtime: state.runtime, benchmarks: state.benchmarks, jobs: state.jobs.map(({ id, type, model, status, message }) => ({ id, type, model, status, message })) }; } });
+ await register({ name: 'harness_start_action', title: 'Control a local model', description: 'Start one local MLX-LM action. Deploy downloads a missing model. Benchmark runs three warm measurements. Returns a job ID; inspect inventory to see completion.', inputSchema: { type: 'object', properties: { type: { type: 'string', enum: ['deploy', 'download', 'unload', 'benchmark'] }, model: { type: 'string', description: 'Exact Hugging Face MLX model ID, including its owner.' } }, required: ['type', 'model'], additionalProperties: false }, annotations: { readOnlyHint: false, untrustedContentHint: true }, async execute(input) { if (!input || !['deploy', 'download', 'unload', 'benchmark'].includes(input.type) || typeof input.model !== 'string') throw new Error('Provide an allowed action and model.'); return runAction(input.type, input.model); } });
+ } catch (error) { console.info('WebMCP tools unavailable:', error.message); }
+}
+window.addEventListener('pagehide', () => { lifecycle.abort(); events.close(); });
diff --git a/store/agents/mlx-lm/dist/assets/README.md b/store/agents/mlx-lm/dist/assets/README.md
new file mode 100644
index 000000000..eb813d8c6
--- /dev/null
+++ b/store/agents/mlx-lm/dist/assets/README.md
@@ -0,0 +1,3 @@
+# MLX identity
+
+Official MLX wordmark from https://github.com/ml-explore/mlx/blob/main/docs/logo/mlx_logo.svg (retrieved 2026-09-20). Centered unchanged on a light square tile for the viewer and native app. This is a local third-party harness, not an official MLX application.
diff --git a/store/agents/mlx-lm/dist/assets/mlx-icon.png b/store/agents/mlx-lm/dist/assets/mlx-icon.png
new file mode 100644
index 000000000..384b343dd
Binary files /dev/null and b/store/agents/mlx-lm/dist/assets/mlx-icon.png differ
diff --git a/store/agents/mlx-lm/dist/assets/mlx-icon.svg b/store/agents/mlx-lm/dist/assets/mlx-icon.svg
new file mode 100644
index 000000000..5effa238c
--- /dev/null
+++ b/store/agents/mlx-lm/dist/assets/mlx-icon.svg
@@ -0,0 +1,34 @@
+<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" width="512" height="512" viewBox="0 0 512 512"><rect width="512" height="512" rx="112" fill="#f8f9fc"/><g transform="translate(39 186)"><defs>
+<g>
+<g id="glyph-0-0">
+<path d="M 9.6875 -171.1875 L 188.609375 -171.1875 L 188.609375 -188.8125 L 9.6875 -188.8125 Z M 9.6875 -127.421875 L 188.609375 -127.421875 L 188.609375 -145.046875 L 9.6875 -145.046875 Z M 9.6875 -83.5625 L 188.609375 -83.5625 L 188.609375 -101.1875 L 9.6875 -101.1875 Z M 9.6875 -39.796875 L 188.609375 -39.796875 L 188.609375 -57.421875 L 9.6875 -57.421875 Z M 9.6875 4.0625 L 188.609375 4.0625 L 188.609375 -13.5625 L 9.6875 -13.5625 Z M 9.6875 47.828125 L 188.609375 47.828125 L 188.609375 30.203125 L 9.6875 30.203125 Z M 9.6875 47.828125 "/>
+</g>
+<g id="glyph-0-1">
+<path d="M 13.9375 0 L 42.3125 0 L 42.3125 -91.796875 L 43.671875 -91.796875 L 78.71875 0 L 97.984375 0 L 133.03125 -91.796875 L 134.390625 -91.796875 L 134.390625 0 L 162.765625 0 L 162.765625 -139.71875 L 125.96875 -139.71875 L 88.984375 -42.015625 L 87.8125 -42.015625 L 50.734375 -139.71875 L 13.9375 -139.71875 Z M 13.9375 0 "/>
+</g>
+<g id="glyph-0-2">
+<path d="M 13.9375 0 L 106.21875 0 L 106.21875 -26.046875 L 45.984375 -26.046875 L 45.984375 -139.71875 L 13.9375 -139.71875 Z M 13.9375 0 "/>
+</g>
+<g id="glyph-0-3">
+<path d="M 6.296875 0 L 40.5625 0 L 69.515625 -47.34375 L 70.484375 -47.34375 L 99.625 0 L 135.84375 0 L 91.109375 -69.8125 L 91.109375 -70.296875 L 136.328125 -139.71875 L 100.703125 -139.71875 L 73 -90.71875 L 71.84375 -90.71875 L 43.953125 -139.71875 L 6.484375 -139.71875 L 49.96875 -70.6875 L 49.96875 -70.296875 Z M 6.296875 0 "/>
+</g>
+</g>
+<clipPath id="clip-0">
+<path clip-rule="nonzero" d="M 13 0 L 283 0 L 283 139.71875 L 13 139.71875 Z M 13 0 "/>
+</clipPath>
+<clipPath id="clip-1">
+<path clip-rule="nonzero" d="M 296 0 L 427 0 L 427 139.71875 L 296 139.71875 Z M 296 0 "/>
+</clipPath>
+</defs>
+<g clip-path="url(#clip-0)">
+<g fill="rgb(0%, 0%, 0%)" fill-opacity="1">
+<use xlink:href="#glyph-0-1" x="0" y="139.72"/>
+<use xlink:href="#glyph-0-2" x="176.682092" y="139.72"/>
+</g>
+</g>
+<g clip-path="url(#clip-1)">
+<g fill="rgb(82.998657%, 82.998657%, 82.998657%)" fill-opacity="1">
+<use xlink:href="#glyph-0-3" x="290.57" y="139.72"/>
+</g>
+</g>
+</g></svg>
diff --git a/store/agents/mlx-lm/dist/index.html b/store/agents/mlx-lm/dist/index.html
new file mode 100644
index 000000000..1a4d414d5
--- /dev/null
+++ b/store/agents/mlx-lm/dist/index.html
@@ -0,0 +1,59 @@
+<!doctype html>
+<html lang="en">
+<head>
+ <meta charset="utf-8">
+ <meta name="viewport" content="width=device-width, initial-scale=1">
+ <meta name="theme-color" content="#10151f">
+ <meta name="description" content="Run, inspect, and benchmark your MLX-LM models on this Mac using natural language.">
+ <title>MLX-LM — Your local model space</title>
+ <link rel="icon" type="image/svg+xml" href="/assets/mlx-icon.svg">
+ <link rel="apple-touch-icon" href="/assets/mlx-icon.png">
+ <link rel="stylesheet" href="/styles.css">
+ <script type="module" src="/app.js"></script>
+</head>
+<body>
+ <header class="topbar">
+ <a class="brand" href="/?view=grid" aria-label="MLX-LM model space"><img class="brand-mark" src="/assets/mlx-icon.svg" width="40" height="40" alt=""><span>MLX-LM</span><span class="pilot">LOCAL AI</span></a>
+ <div class="top-status"><span class="local-symbol" aria-hidden="true">◈</span> On your machine <span class="separator"></span><span id="connection-status">Connecting</span></div>
+ </header>
+ <main class="workspace">
+ <section class="control-room" aria-label="MLX-LM control room">
+ <div class="page-heading"><div><p class="eyebrow">MLX-LM / ON THIS MAC</p><h1>Models. Made for this Mac.</h1></div><button id="refresh" class="icon-button" aria-label="Refresh model status" title="Refresh model status">↻</button></div>
+ <div id="runtime-notice" class="runtime-notice" role="status" hidden></div>
+ <div class="stats-row" id="stats-row">
+ <div class="stat"><span>Models on disk</span><strong id="model-count">—</strong></div>
+ <div class="stat"><span>In memory</span><strong id="running-count">—</strong></div>
+ <div class="stat"><span>Last generation</span><strong id="last-speed">— <small>tok/s</small></strong></div>
+ <div class="stat"><span>MLX allocation</span><strong id="model-memory">— <small>GiB</small></strong></div>
+ </div>
+ <section class="map-panel" aria-labelledby="map-heading">
+ <div class="panel-heading"><h2 id="map-heading">MLX on Apple Silicon</h2><span class="mono live-label" id="map-status">DISCOVERING</span></div>
+ <div id="runtime-map" class="runtime-map"><div class="map-empty">Discovering your MLX-LM models…</div></div>
+ <div class="map-footer"><span><i class="legend-dot loaded"></i> In memory <i class="legend-dot"></i> On disk</span><span id="machine-info">Apple Silicon</span></div>
+ </section>
+ <section class="models-panel" aria-labelledby="models-heading">
+ <div class="panel-heading"><h2 id="models-heading">MLX-LM models <span id="models-total" class="count-pill">0</span></h2><span class="secondary">One model in memory at a time</span></div>
+ <div id="models-list" class="models-list"></div>
+ </section>
+ <section class="performance-panel" aria-labelledby="performance-heading">
+ <div class="panel-heading"><h2 id="performance-heading">Performance</h2><a id="export-benchmarks" class="secondary" href="/api/export" download hidden>Export results ↗</a></div>
+ <div id="performance-content" class="performance-content"><div class="empty-small"><span class="empty-icon" aria-hidden="true">▥</span><p>Your first run sets the baseline.</p><span>Benchmark a model to compare speed and response time.</span></div></div>
+ </section>
+ <section class="activity-panel" aria-labelledby="activity-heading"><div class="panel-heading"><h2 id="activity-heading">Activity</h2><span class="secondary">Live from MLX-LM</span></div><div id="activity-content"></div></section>
+ </section>
+ <aside class="assistant-panel" aria-label="Natural language model controls">
+ <div class="assistant-header"><img class="assistant-avatar" src="/assets/mlx-icon.svg" width="36" height="36" alt=""><div><h2>MLX-LM operator</h2><span>Tell it what you want to run.</span></div><span class="private-badge">LOCAL</span></div>
+ <div class="conversation" id="conversation" role="log" aria-live="polite" aria-relevant="additions">
+ <div class="assistant-message"><span class="message-label">MLX-LM</span><p>Your MLX-LM models.<br>At your command.</p><p class="secondary">I can find a model, deploy it with MLX-LM, and measure how it performs here.</p></div>
+ <div class="suggestions" id="suggestions"><button data-prompt="Run a lightweight model">Run a lightweight model <span>↗</span></button><button data-prompt="Find a small model for coding">Find a model for coding <span>↗</span></button><button data-prompt="Show my models">Show my models <span>↗</span></button></div>
+ </div>
+ <div id="job-panel" class="job-panel" hidden></div>
+ <label class="model-picker" for="selected-model">Chat / follow-up model<select id="selected-model"><option value="">Auto · running model</option></select></label>
+ <form id="command-form" class="composer"><label class="sr-only" for="command-input">Ask the MLX-LM operator</label><textarea id="command-input" rows="3" placeholder="Run an MLX model, compare speeds, free up memory…" maxlength="4000"></textarea><div class="composer-footer"><span id="operator-status">MLX-LM harness</span><button id="send-button" type="submit" aria-label="Send request">↑</button></div></form>
+ <div class="assistant-footer">MLX-LM inference stays on this Mac.<span>↵ Send · ⇧↵ New line</span></div>
+ </aside>
+ </main>
+ <dialog id="model-dialog" aria-labelledby="detail-title"><div id="model-detail"></div><button class="dialog-close" aria-label="Close model details">×</button></dialog>
+ <div id="toast" class="toast" role="status" hidden></div>
+</body>
+</html>
diff --git a/store/agents/mlx-lm/dist/styles.css b/store/agents/mlx-lm/dist/styles.css
new file mode 100644
index 000000000..9db286a4b
--- /dev/null
+++ b/store/agents/mlx-lm/dist/styles.css
@@ -0,0 +1,24 @@
+:root{color-scheme:dark;--bg:#10151f;--surface:#171e2a;--surface-2:#202a38;--line:#344154;--muted:#a2aec0;--text:#edf3ee;--accent:#a9caff;--cyan:#8ae8d4;--error:#ffafa4;--mono:ui-monospace,SFMono-Regular,Menlo,monospace;font-family:Inter,-apple-system,BlinkMacSystemFont,"Segoe UI",sans-serif;font-size:16px}
+*{box-sizing:border-box}body{margin:0;background:var(--bg);color:var(--text)}button,textarea,input,select{font:inherit}button{cursor:pointer}button:disabled{cursor:wait;opacity:.55}button,a,textarea,select{outline-offset:4px}button:focus-visible,a:focus-visible{outline:2px solid var(--accent)}button{color:inherit}a{color:inherit}button{border:1px solid var(--line);background:var(--surface);border-radius:8px}button:hover{border-color:#617166}h1,h2,p{margin:0}h2{font-size:1rem;font-weight:600;letter-spacing:-.02em}.mono{font-family:var(--mono)}.secondary{color:var(--muted);font-size:.875rem;line-height:1.6}.topbar{height:76px;border-bottom:1px solid var(--line);display:flex;justify-content:space-between;align-items:center;padding:0 32px}.brand{text-decoration:none;font-size:1rem;font-weight:750;letter-spacing:.15em;display:flex;align-items:center;gap:13px}.brand-mark{background:var(--accent);color:#111b0c;width:30px;height:32px;display:grid;place-items:center;clip-path:polygon(0 0,80% 0,100% 25%,100% 100%,20% 100%,0 75%);font-size:1.125rem;font-weight:900;letter-spacing:-.15em;padding-right:4px}.pilot{font-family:var(--mono);letter-spacing:.05em;font-size:.75rem;font-weight:400;color:var(--muted);padding-left:16px;margin-left:8px;border-left:1px solid var(--line)}.top-status{display:flex;gap:10px;align-items:center;font-size:.8125rem;color:var(--muted)}.local-symbol{color:var(--accent);font-size:1.1rem}.separator{height:16px;width:1px;background:var(--line);margin:0 6px}.workspace{display:grid;grid-template-columns:minmax(0,1fr) 360px;min-height:calc(100vh - 76px);max-width:1900px;margin:auto}.control-room{padding:34px 32px 40px;min-width:0}.page-heading{display:flex;justify-content:space-between;align-items:center;margin-bottom:30px}.eyebrow{font-family:var(--mono);font-size:.75rem;color:var(--accent);letter-spacing:.15em;margin-bottom:10px}h1{font-size:clamp(1.75rem,2.6vw,2.5rem);letter-spacing:-.06em;line-height:1.15;font-weight:550}.icon-button{border-radius:50%;width:38px;height:38px;font-size:1.4rem;background:transparent;color:var(--muted)}.stats-row{display:grid;grid-template-columns:repeat(4,1fr);margin-bottom:30px}.stat{padding:0 22px;border-right:1px solid var(--line);min-width:0}.stat:first-child{padding-left:0}.stat:last-child{border:0}.stat>span{font-size:.8125rem;color:var(--muted);display:block;margin-bottom:9px}.stat strong{font-family:var(--mono);font-size:1.75rem;font-weight:400;letter-spacing:-.07em}.stat small{font-family:inherit;color:var(--muted);font-size:.8rem;letter-spacing:-.02em}.panel-heading{display:flex;justify-content:space-between;align-items:center;gap:12px;margin-bottom:18px}.map-panel{border:1px solid var(--line);background:var(--surface);border-radius:12px;overflow:hidden}.map-panel .panel-heading{padding:20px 20px 0;margin-bottom:0}.live-label{font-size:.6875rem;letter-spacing:.09em;color:var(--muted)}.live-label.online{color:var(--accent)}.runtime-map{min-height:252px;position:relative;padding:25px 22px;background-image:radial-gradient(#3a454238 .8px,transparent .8px);background-size:16px 16px;display:flex;align-items:center;gap:36px}.runtime-node{border:1px solid #596a50;background:#202c21;border-radius:14px;padding:18px;min-width:150px;box-shadow:0 0 40px #a9caff07;z-index:1;position:relative}.runtime-node::after{content:'';width:37px;height:1px;background:#617751;position:absolute;right:-37px;top:50%}.runtime-node.offline{border-color:var(--line);background:var(--surface-2)}.runtime-node .node-icon{width:32px;height:32px;display:grid;place-items:center;background:var(--accent);color:#233014;border-radius:9px;font-weight:700;margin-bottom:13px}.runtime-node strong{font-size:1rem;display:block;margin-bottom:6px}.runtime-node small{font-family:var(--mono);color:var(--muted);font-size:.75rem}.model-nodes{display:flex;gap:10px;flex-wrap:wrap;align-content:center;flex:1;border-left:1px solid #3c4d3c;padding-left:22px;min-height:120px}.model-node{position:relative;text-align:left;padding:13px 15px;border-radius:8px;min-width:160px;max-width:240px;background:#1b2222}.model-node::before{content:'';position:absolute;left:-23px;top:50%;height:1px;width:22px;background:#3c4d3c}.model-node.active{border-color:#7a995a;background:#263324}.model-node strong{display:block;font-size:.875rem;overflow-wrap:anywhere;line-height:1.4}.model-node span{display:block;color:var(--muted);font-size:.75rem;margin-top:6px}.model-node.active span{color:var(--accent)}.map-empty{margin:auto;color:var(--muted);font-size:.875rem;max-width:330px;line-height:1.7;text-align:center}.empty-node{text-align:left;margin:0;padding:12px 0}.empty-node strong{display:block;color:var(--text);font-size:1rem;margin-bottom:6px}.map-footer{border-top:1px solid var(--line);display:flex;align-items:center;justify-content:space-between;gap:12px;padding:12px 20px;font-family:var(--mono);font-size:.6875rem;color:var(--muted)}.map-footer>span:first-child{display:flex;align-items:center;gap:6px}.legend-dot{width:5px;height:5px;display:inline-block;border-radius:50%;background:#6a7a74;margin-left:12px}.legend-dot:first-child{margin-left:0}.legend-dot.loaded{background:var(--accent)}.models-panel,.performance-panel{margin-top:30px}.count-pill{display:inline-grid;place-items:center;font-family:var(--mono);font-size:.75rem;min-width:23px;height:21px;border-radius:5px;border:1px solid var(--line);color:var(--muted);margin-left:6px}.model-row{width:100%;display:grid;grid-template-columns:minmax(0,1fr) 85px 80px 80px;gap:14px;align-items:center;text-align:left;border-width:0 0 1px;border-radius:0;background:transparent;padding:17px 4px;font-size:.875rem}.model-row:hover{background:var(--surface)}.model-title strong{font-weight:500;overflow-wrap:anywhere;display:block}.model-title small{font-size:.75rem;color:var(--muted);display:block;margin-top:6px}.model-status{font-size:.75rem;color:var(--muted)}.model-status.loaded{color:var(--accent)}.model-numeric{font-family:var(--mono);font-size:.75rem;color:var(--muted)}.empty-small{padding:30px 18px;border:1px dashed var(--line);border-radius:10px;color:var(--muted);text-align:center;font-size:.875rem}.empty-small p{color:var(--text);margin:8px 0;font-size:1rem}.empty-small span:not(.empty-icon){font-size:.8125rem;line-height:1.6}.empty-icon{font-size:1.5rem;color:#667973}.assistant-panel{border-left:1px solid var(--line);background:#131a1b;display:flex;flex-direction:column;min-width:0;position:sticky;top:0;height:calc(100vh - 76px);max-height:1200px}.assistant-header{display:flex;gap:12px;align-items:center;padding:26px 22px;border-bottom:1px solid var(--line)}.assistant-avatar{height:35px;width:35px;display:grid;place-items:center;color:var(--accent);background:#293526;font-size:1.5rem;border-radius:10px}.assistant-header h2{font-size:.9375rem}.assistant-header div>span{font-size:.75rem;color:var(--muted);display:block;margin-top:4px}.private-badge{font-size:.625rem;letter-spacing:.08em;color:var(--muted);border:1px solid var(--line);border-radius:4px;padding:4px 5px;margin-left:auto;font-family:var(--mono)}.conversation{padding:26px 22px;overflow:auto;flex:1;min-height:200px;scroll-behavior:smooth}.message-label{font-family:var(--mono);font-size:.625rem;letter-spacing:.1em;color:var(--accent);display:block;margin-bottom:13px}.assistant-message{font-size:.9375rem;line-height:1.65;margin-bottom:25px}.assistant-message p+p{margin-top:12px}.assistant-message .secondary{font-size:.875rem}.suggestions{display:flex;flex-direction:column;gap:8px;margin-top:25px}.suggestions button{display:flex;justify-content:space-between;align-items:center;text-align:left;font-size:.8125rem;padding:13px 12px;background:transparent;color:#c4d0cb}.suggestions button span{color:var(--accent)}.composer{margin:0 18px;border:1px solid #485441;background:#202829;border-radius:11px;overflow:hidden}.composer:focus-within{border-color:var(--accent)}textarea{color:var(--text);display:block;resize:none;background:transparent;width:100%;border:0;padding:15px 15px 5px;line-height:1.6;outline:none;font-size:.875rem}textarea::placeholder{color:#91a29b}.composer-footer{padding:8px 11px 10px 15px;display:flex;justify-content:space-between;align-items:center}.composer-footer>span{font-size:.6875rem;font-family:var(--mono);color:var(--muted)}.composer-footer button{background:var(--accent);color:#243315;border:0;border-radius:7px;font-size:1.4rem;width:31px;height:31px}.assistant-footer{display:flex;flex-wrap:wrap;gap:8px;justify-content:space-between;padding:14px 20px 20px;font-size:.625rem;color:#98a79f;line-height:1.5}.assistant-footer span{font-size:.625rem}.user-message{padding:12px 14px;background:#263127;border:1px solid #3b4e35;border-radius:9px 9px 2px 9px;margin:15px 0 25px;white-space:pre-wrap;overflow-wrap:anywhere;font-size:.875rem;line-height:1.6}.message-text{white-space:pre-wrap;overflow-wrap:anywhere}.job-panel{padding:15px;margin:0 18px 15px;background:var(--surface-2);border:1px solid var(--line);border-radius:10px;font-size:.8125rem}.job-panel strong{font-size:.8125rem;font-weight:500}.job-panel p{color:var(--muted);font-size:.75rem;margin-top:7px}.job-top{display:flex;justify-content:space-between;align-items:center}.job-top button{font-size:.6875rem;padding:4px 7px}.progress-track{height:4px;background:var(--line);border-radius:4px;margin:12px 0 2px;overflow:hidden}.progress-fill{height:100%;width:0;background:var(--accent);transition:width .3s}.toast{position:fixed;bottom:20px;left:50%;transform:translateX(-50%);padding:12px 18px;background:#33402f;color:var(--text);border:1px solid #60784d;border-radius:8px;max-width:90vw;z-index:10;font-size:.875rem}dialog{background:var(--surface);color:var(--text);border:1px solid var(--line);border-radius:16px;padding:30px;max-width:600px;width:calc(100% - 32px)}dialog::backdrop{background:#000a;backdrop-filter:blur(3px)}.dialog-close{position:absolute;right:12px;top:10px;border:0;background:transparent;font-size:1.5rem}.sr-only{position:absolute;width:1px;height:1px;overflow:hidden;clip:rect(0,0,0,0);white-space:nowrap}.detail-grid{display:grid;grid-template-columns:repeat(2,1fr);gap:16px;margin:22px 0}.detail-grid>div{border:1px solid var(--line);padding:14px;border-radius:8px}.detail-grid span{font-size:.75rem;color:var(--muted);display:block;margin-bottom:7px}.detail-grid strong{font-size:1rem;font-weight:500}.actions{display:flex;flex-wrap:wrap;gap:8px;margin-top:18px}.actions button{padding:9px 13px;font-size:.875rem}.actions button.primary{background:var(--accent);color:#233014;border-color:var(--accent)}.chart-bars{display:flex;flex-direction:column;gap:15px;padding:8px 0}.chart-row{display:grid;grid-template-columns:minmax(120px,160px) 1fr 65px;gap:16px;align-items:center;font-size:.8125rem}.chart-row>span:first-child{overflow-wrap:anywhere}.bar-track{height:22px;background:var(--surface-2);border-radius:4px;overflow:hidden}.bar-fill{height:100%;background:var(--accent);border-radius:3px;min-width:2px}.bar-value{font-family:var(--mono);font-size:.8125rem}.benchmark-meta{color:var(--muted);font-size:.75rem;margin-top:15px;line-height:1.8}.plan-card{border:1px solid #536846;border-radius:10px;background:#222e20;padding:15px;margin-top:12px}.plan-card p{font-size:.8125rem;line-height:1.6}.plan-card .actions{margin-top:10px}.error-text{color:var(--error)}.catalog-card{border:1px solid var(--line);border-radius:9px;padding:13px;margin:10px 0;font-size:.8125rem}.catalog-card strong{display:block;margin-bottom:5px}.catalog-card button{margin-top:10px;padding:6px 10px;background:var(--accent);color:#233014;font-size:.75rem;border-color:var(--accent)}.stream-output{max-height:340px;overflow:auto;border-left:2px solid #526c3f;padding-left:12px;font-size:.875rem}.warning-note{font-size:.75rem;color:#d4cca8;line-height:1.6;margin-top:10px}.mini-label{font-family:var(--mono);font-size:.6875rem;color:var(--muted);letter-spacing:.03em}
+@media(min-width:1450px){.workspace{grid-template-columns:minmax(0,1fr) 420px}.control-room{padding:40px 48px}.runtime-map{min-height:295px}.assistant-header,.conversation{padding:28px}.composer{margin:0 24px}.model-node{min-width:180px}}
+@media(max-width:1080px){.workspace{grid-template-columns:minmax(0,1fr) 320px}.control-room{padding:28px 22px}.topbar{padding:0 22px}.stat{padding:0 12px}.stat strong{font-size:1.5rem}.stat>span{font-size:.75rem}.runtime-map{gap:25px;padding:22px 16px}.runtime-node{min-width:118px;padding:14px}.runtime-node::after{width:26px;right:-26px}.model-node{min-width:130px;padding:12px;max-width:100%}.model-nodes{padding-left:14px}.model-node::before{left:-15px;width:14px}.model-row{grid-template-columns:minmax(0,1fr) 68px 65px;gap:7px}.model-row .model-numeric:last-child{display:none}.map-footer{font-size:.625rem;padding:12px}.top-status{font-size:.75rem}.assistant-header{padding:22px 16px}}
+@media(max-width:820px){.workspace{display:flex;flex-direction:column}.assistant-panel{position:relative;height:auto;max-height:none;border-left:0;border-top:1px solid var(--line);min-height:500px}.control-room{padding:26px 20px}.conversation{max-height:480px}.topbar{height:64px;padding:0 20px}.top-status .separator,.top-status #connection-status{display:none}.pilot{font-size:.625rem;padding-left:10px;margin-left:0}.brand{font-size:.875rem;gap:9px}.runtime-map{min-height:230px}.model-nodes{gap:8px}.model-node{max-width:220px}.stats-row{margin-bottom:24px}.stat{padding:0 10px}.stat>span{font-size:.6875rem}.map-footer{font-size:.625rem}.assistant-footer{font-size:.6875rem}.top-status{font-size:.6875rem;gap:5px}.assistant-header h2{font-size:1rem}.assistant-header div>span{font-size:.875rem}.private-badge{font-size:.75rem}.suggestions button{font-size:.875rem}textarea{font-size:1rem}.assistant-message{font-size:1rem}.model-row{grid-template-columns:minmax(0,1fr) 75px 80px}.model-node strong{font-size:.875rem}.model-node span{font-size:.75rem}}
+@media(max-width:430px){.top-status{display:none}.stat strong{font-size:1.35rem}.stat>span{min-height:30px;line-height:1.4}.stat small{font-size:.6875rem}.runtime-map{gap:20px;padding:18px 12px}.runtime-node{min-width:106px;padding:12px}.runtime-node::after{right:-21px;width:21px}.model-node{min-width:120px}.map-footer{flex-wrap:wrap}.panel-heading>.secondary{font-size:.75rem}.chart-row{grid-template-columns:110px 1fr 52px;gap:8px}}
+@media(prefers-reduced-motion:reduce){*{scroll-behavior:auto!important;transition:none!important;animation:none!important}}
+/* Live control and measurement surfaces. */
+[hidden]{display:none!important}.model-picker{display:flex;flex-direction:column;gap:7px;margin:0 18px 12px;color:var(--muted);font-size:.75rem}.model-picker select{width:100%;color:var(--text);background:var(--surface);border:1px solid var(--line);border-radius:7px;padding:9px;font-size:.875rem}.model-node.selected{box-shadow:0 0 0 2px var(--cyan)}.model-row.selected{background:#1a2826}.job-state{color:var(--muted);margin-left:8px;letter-spacing:0;font-family:inherit;text-transform:lowercase}.run-metrics{font-family:var(--mono);font-size:.75rem;color:var(--cyan);margin-top:12px!important}.stream-output{margin-top:14px;line-height:1.7}.chart-heading{display:flex;justify-content:space-between;gap:12px;align-items:center;flex-wrap:wrap;margin-bottom:10px}.chart-heading h3{font-size:.875rem;margin:0;font-weight:500}.chart-heading>span{font-size:.75rem;color:var(--muted)}.second-chart{margin-top:26px}.bar-fill.latency{background:var(--cyan)}.benchmark-details{margin-top:16px;border-top:1px solid var(--line);padding-top:14px;font-size:.875rem}.benchmark-details summary{cursor:pointer;color:var(--muted)}.sample-record{display:flex;flex-direction:column;gap:5px;padding:18px 0;border-bottom:1px solid var(--line);font-size:.875rem;overflow-wrap:anywhere}.sample-record span{color:var(--muted);font-size:.75rem}.chart-row{grid-template-columns:minmax(125px,170px) 1fr 64px}.bar-value{text-align:right}.bar-value small{font-size:.75rem}.detail-grid strong,#detail-title{overflow-wrap:anywhere}#operator-status{overflow:hidden;text-overflow:ellipsis;white-space:nowrap;max-width:calc(100% - 44px)}.job-top{gap:12px}.job-top button{flex-shrink:0}.assistant-message,.stream-output,.user-message{font-size:1rem}.live-label,.map-footer,.private-badge,.message-label,.composer-footer>span,.assistant-footer,.assistant-footer span,.job-top button,.mini-label,.pilot,.stat>span,.top-status,.stat small{font-size:.75rem}.stat>span,.suggestions button,.top-status{font-size:.875rem}.composer textarea{font-size:1rem}.model-node span,.model-title small,.model-status,.model-numeric{font-size:.8125rem}.map-footer{line-height:1.6;flex-wrap:wrap}.topbar{gap:14px}.assistant-panel{min-height:650px}.job-panel p{overflow-wrap:anywhere}.catalog-card p{line-height:1.6}.model-nodes{min-width:0}.model-node{min-width:0;flex:1 1 150px}button,a,select{-webkit-tap-highlight-color:transparent}
+@media(max-width:1080px){.stat strong{font-size:1.45rem}.stat>span{min-height:32px}.model-row{grid-template-columns:minmax(0,1fr) 76px 76px}.chart-row{grid-template-columns:130px 1fr 60px;gap:10px}}
+@media(max-width:820px){.assistant-panel{min-height:650px}.top-status{font-size:.75rem}.stat>span{font-size:.75rem}.stat small{font-size:.75rem}.model-picker select{font-size:1rem}.assistant-panel{height:auto}.model-nodes{flex-basis:140px}.model-node{min-width:0}.map-footer{font-size:.75rem}}
+@media(max-width:430px){.chart-row{grid-template-columns:100px 1fr 54px;gap:8px}.model-row{grid-template-columns:minmax(0,1fr) 72px 62px}.model-numeric,.model-status{font-size:.75rem}.stats-row{gap:0}.stat{padding:0 7px}.stat strong{font-size:1.3rem}.stat>span{min-height:36px}.runtime-node{min-width:100px}.control-room{padding:24px 16px}.runtime-map{gap:17px}.runtime-node::after{width:18px;right:-18px}.map-footer>span{flex-wrap:wrap}.assistant-header{padding:20px 16px}.conversation{padding:22px 18px}}
+@media(pointer:coarse){button{min-height:44px}.model-picker select{min-height:44px}.composer-footer button{min-width:44px}.icon-button{width:44px;height:44px}}
+.activity-panel{margin-top:32px}.activity-row{display:flex;align-items:flex-start;gap:16px;padding:17px 0;border-top:1px solid var(--line);font-size:.875rem}.activity-state{min-width:78px;font-family:var(--mono);font-size:.75rem;color:var(--muted);padding-top:3px}.activity-row>div{flex:1;min-width:0}.activity-row strong{font-weight:500;overflow-wrap:anywhere}.activity-row p{font-size:.8125rem;line-height:1.6;margin-top:5px;color:var(--muted);overflow-wrap:anywhere}.activity-time{font-size:.75rem;white-space:nowrap;color:var(--muted)}.activity-row button{padding:5px 9px;font-size:.75rem}.embedded-viewer .workspace{display:block;max-width:1440px}.embedded-viewer .assistant-panel{display:none}.embedded-viewer .control-room{padding:34px clamp(20px,4vw,60px)}.embedded-viewer .runtime-map{min-height:240px}.embedded-viewer .topbar{height:64px}.embedded-viewer .workspace{min-height:calc(100vh - 64px)}.embedded-viewer .stats-row{max-width:1050px}.embedded-viewer .model-nodes{flex:1}.embedded-viewer .model-node{flex:0 1 230px}.embedded-viewer .model-row{grid-template-columns:minmax(0,1fr) 100px 110px 100px}.embedded-viewer .runtime-node{min-width:160px}.embedded-viewer .model-row .model-numeric:last-child{display:block}
+@media(max-width:600px){.embedded-viewer .model-row{grid-template-columns:minmax(0,1fr) 76px 76px}.embedded-viewer .model-row .model-numeric:last-child{display:none}.embedded-viewer .runtime-node{min-width:100px}.activity-row{gap:10px}.activity-state{min-width:64px}.activity-time{display:none}.embedded-viewer .model-node{flex-basis:160px}.embedded-viewer .control-room{padding:24px 16px}}
+.embedded-viewer [data-chat]{display:none}
+
+/* MLX-LM identity: use the official mark at every logo size. */
+.brand{font-size:1.45rem;letter-spacing:-.04em;text-transform:none;gap:12px}.brand .brand-mark{display:block;width:40px;height:40px;padding:0;background:none;clip-path:none;border-radius:10px;object-fit:contain}.brand .pilot{font-size:.65rem;letter-spacing:.13em;margin-left:10px;padding-left:20px}.runtime-node .node-icon{display:block;width:42px;height:42px;padding:0;margin:0 auto 9px;background:none;border-radius:10px;object-fit:contain}.assistant-header .assistant-avatar{padding:0;object-fit:contain;background:none;border-radius:9px}
+
+.runtime-notice{margin:0 0 20px;padding:16px 18px;border:1px solid #614840;border-radius:12px;background:#241e1a}.runtime-notice[hidden]{display:none}.runtime-notice strong{color:var(--error);font-size:.9rem}.runtime-notice p{margin:8px 0;color:var(--text);font-size:.85rem;line-height:1.5}.runtime-notice summary{cursor:pointer;color:var(--muted);font-size:.8rem}.runtime-notice pre{white-space:pre-wrap;overflow-wrap:anywhere;font: .75rem/1.5 var(--mono);color:var(--error)}
+
+/* Hugging Face model names are longer than Ollama tags. */
+.model-node strong,.model-title strong,.chart-row>span:first-child,.activity-row strong{overflow-wrap:anywhere}.model-node{min-width:0}.chart-row>span:first-child{font-size:.72rem}.activity-row>div{min-width:0}.model-title strong{white-space:normal}.sample-record strong{overflow-wrap:anywhere}
diff --git a/store/agents/mlx-lm/harness.json b/store/agents/mlx-lm/harness.json
new file mode 100644
index 000000000..acdc2b43d
--- /dev/null
+++ b/store/agents/mlx-lm/harness.json
@@ -0,0 +1,29 @@
+{
+ "spec": 1,
+ "id": "autonomous/mlx-lm",
+ "name": "MLX-LM",
+ "category": "Local AI",
+ "author": "OpenHarness contributors",
+ "description": "Run language models directly on Apple Silicon with MLX-LM. Talk to Codex to download, load and compare models, with live progress and native performance measurements.",
+ "engine": "codex",
+ "workspace": {
+ "template": "template",
+ "marker": "local-ai.json"
+ },
+ "agent": {
+ "instructions": "agent/AGENTS.md",
+ "env": {
+ "LOCAL_AI": "${dsh}/toolchain/local-ai",
+ "MLX_LM": "${dsh}/toolchain/mlx-lm"
+ }
+ },
+ "toolchain": {
+ "setup": "toolchain/setup.sh",
+ "doctor": "toolchain/doctor.sh"
+ },
+ "viewer": {
+ "command": "./viewer.sh",
+ "url": "http://127.0.0.1:${port}/?view=grid"
+ },
+ "verdict": ".harness/verdict.json"
+}
diff --git a/store/agents/mlx-lm/licenses/mlx-lm.txt b/store/agents/mlx-lm/licenses/mlx-lm.txt
new file mode 100644
index 000000000..98ff47b9e
--- /dev/null
+++ b/store/agents/mlx-lm/licenses/mlx-lm.txt
@@ -0,0 +1,21 @@
+MIT License
+
+Copyright © 2023 Apple Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
diff --git a/store/agents/mlx-lm/licenses/mlx.txt b/store/agents/mlx-lm/licenses/mlx.txt
new file mode 100644
index 000000000..98ff47b9e
--- /dev/null
+++ b/store/agents/mlx-lm/licenses/mlx.txt
@@ -0,0 +1,21 @@
+MIT License
+
+Copyright © 2023 Apple Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
diff --git a/store/agents/mlx-lm/main.mjs b/store/agents/mlx-lm/main.mjs
new file mode 100644
index 000000000..344acbadf
--- /dev/null
+++ b/store/agents/mlx-lm/main.mjs
@@ -0,0 +1,8 @@
+import { dirname, join, resolve } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { start } from './src/server.mjs';
+import { MlxAdapter, MLX_PROFILE } from './src/mlx.mjs';
+const packageRoot = dirname(fileURLToPath(import.meta.url));
+const workspace = resolve(process.env.HARNESS_WORKSPACE || packageRoot);
+const adapter = new MlxAdapter({ packageRoot, dataDir: join(workspace, '.harness') });
+await start(Number(process.env.HARNESS_VIEWER_PORT || process.env.HARNESS_PORT || 4311), { workspaceRoot: workspace, assetRoot: join(packageRoot, 'dist'), profile: MLX_PROFILE, adapter });
diff --git a/store/agents/mlx-lm/requirements.lock b/store/agents/mlx-lm/requirements.lock
new file mode 100644
index 000000000..11dae7725
--- /dev/null
+++ b/store/agents/mlx-lm/requirements.lock
@@ -0,0 +1,34 @@
+annotated-doc==0.0.5
+anyio==4.15.1
+certifi==2026.7.22
+click==8.5.0
+filelock==4.0.1
+fsspec==2026.9.0
+h11==0.16.0
+hf-xet==1.6.0
+httpcore==1.0.9
+httpx==0.28.1
+huggingface-hub==1.32.0
+idna==3.20
+jinja2==3.1.6
+markdown-it-py==4.2.0
+markupsafe==3.0.3
+mdurl==0.1.2
+mlx==0.32.2
+mlx-lm==0.31.3
+mlx-metal==0.32.2
+numpy==2.5.3
+packaging==26.3
+protobuf==7.36.2
+pygments==2.21.0
+pyyaml==6.0.3
+regex==2026.9.10
+rich==15.0.0
+safetensors==0.8.0
+sentencepiece==0.2.2
+shellingham==1.5.4
+tokenizers==0.23.2
+tqdm==4.70.1
+transformers==5.17.0
+typer==0.27.2
+typing-extensions==4.16.0
diff --git a/store/agents/mlx-lm/requirements.txt b/store/agents/mlx-lm/requirements.txt
new file mode 100644
index 000000000..2b5f23b98
--- /dev/null
+++ b/store/agents/mlx-lm/requirements.txt
@@ -0,0 +1,2 @@
+mlx-lm==0.31.3
+mlx==0.32.2
diff --git a/store/agents/mlx-lm/src/catalog.mjs b/store/agents/mlx-lm/src/catalog.mjs
new file mode 100644
index 000000000..616380e9d
--- /dev/null
+++ b/store/agents/mlx-lm/src/catalog.mjs
@@ -0,0 +1,6 @@
+export const CATALOG = [
+ { id: 'qwen3:0.6b', name: 'Qwen 3 · 0.6B', purpose: 'A lightweight first model', description: 'Quick to download. Good for trying the harness, short answers, and simple tasks.', downloadGB: 0.523, use: 'general', small: true, source: 'https://ollama.com/library/qwen3:0.6b' },
+ { id: 'qwen2.5-coder:1.5b', name: 'Qwen 2.5 Coder · 1.5B', purpose: 'Small coding assistant', description: 'A compact model for code snippets, explanations, and simple programming tasks.', downloadGB: 0.986, use: 'coding', small: true, source: 'https://ollama.com/library/qwen2.5-coder:1.5b' },
+ { id: 'gemma3:1b', name: 'Gemma 3 · 1B', purpose: 'Everyday text tasks', description: 'A small text-only model for summarizing, rewriting, and everyday questions.', downloadGB: 0.815, use: 'writing', small: true, source: 'https://ollama.com/library/gemma3:1b' },
+];
+export const DEFAULT_MODEL = CATALOG[0].id;
diff --git a/store/agents/mlx-lm/src/controller.mjs b/store/agents/mlx-lm/src/controller.mjs
new file mode 100644
index 000000000..79613900b
--- /dev/null
+++ b/store/agents/mlx-lm/src/controller.mjs
@@ -0,0 +1,191 @@
+import { EventEmitter } from 'node:events';
+import { randomUUID, createHash } from 'node:crypto';
+import os from 'node:os';
+import { setTimeout as delay } from 'node:timers/promises';
+import * as ollama from './ollama.mjs';
+import { resolveIntent, validateAction } from './intents.mjs';
+import { CATALOG } from './catalog.mjs';
+
+export const BENCHMARK_PROMPT = 'Explain how a bicycle works to a curious beginner. Discuss the pedals, chain, gears, wheels, brakes, and balance. Write a clear explanation of at least 250 words without a heading.';
+export const BENCHMARK_SETTINGS = { context: 2048, maxTokens: 128, seed: 42, temperature: 0, thinking: false };
+export const median = values => { const v = values.filter(Number.isFinite).sort((a, b) => a - b); return v.length ? (v[Math.floor((v.length - 1) / 2)] + v[Math.ceil((v.length - 1) / 2)]) / 2 : null; };
+const terminal = job => !['queued', 'running'].includes(job.status);
+export const safeError = error => /TLS handshake timeout|fetch failed/.test(error.message) ? 'The model download or runtime connection timed out. Check your connection and retry; partial downloads can resume.' : String(error.message || error).replace(/https?:\/\/[^\s"<>]+/g, url => url.split('?')[0]).slice(0, 700);
+
+export class Controller extends EventEmitter {
+ constructor(store, adapter = ollama, profile = {}) {
+ super();
+ this.profile = { id: 'ollama', name: 'Ollama', catalog: CATALOG, ...profile };
+ this.store = store; this.adapter = adapter; this.queue = []; this.aborts = new Map(); this.actions = new Map(); this.processing = false; this.closing = false;
+ this.system = { platform: os.platform(), arch: os.arch(), chip: os.cpus()[0]?.model || 'Local machine', totalMemory: os.totalmem(), cores: os.cpus().length };
+ this.runtime = { id: this.profile.id, name: this.profile.name, online: false, models: [] };
+ this.updatedAt = null;
+ }
+ snapshot() { return { runtime: this.runtime, system: { ...this.system, freeMemory: os.freemem() }, benchmarks: this.store.data.benchmarks, loadTests: this.store.data.loadTests, runs: this.store.data.runs, jobs: this.store.data.jobs, catalog: this.profile.catalog, updatedAt: this.updatedAt }; }
+ async refresh() {
+ if (this.refreshing) return this.refreshing;
+ this.refreshing = this.adapter.discover().then(runtime => { this.runtime = runtime; this.updatedAt = new Date().toISOString(); this.emit('state', this.snapshot()); }).finally(() => { this.refreshing = null; });
+ return this.refreshing;
+ }
+ async command(text, { selectedModel, history } = {}) {
+ await this.refresh();
+ const intent = (this.profile.resolveIntent || resolveIntent)(text, { models: this.runtime.models, selectedModel, profile: this.profile });
+ if (intent.kind === 'action') {
+ if (intent.action.type === 'chat') intent.action.history = history;
+ return { kind: 'job', job: await this.submit(intent.action) };
+ }
+ if (intent.kind === 'inventory') {
+ const models = this.runtime.models;
+ return { kind: 'reply', message: !this.runtime.online ? `${this.profile.name} is not running. Say “start ${this.profile.name}” or “run a lightweight model”.` : models.length ? `${models.length} model${models.length === 1 ? '' : 's'} on disk, ${models.filter(m => m.running).length} in memory.\n${models.map(m => `${m.name} — ${m.running ? 'in memory' : 'on disk'}`).join('\n')}` : `${this.profile.name} is ready. Say “run a lightweight model” to download your first model.` };
+ }
+ return intent;
+ }
+ async submit(input) {
+ if (this.closing) throw new Error('Harness is shutting down. Try again after it restarts.');
+ if (this.queue.length >= 8) throw new Error('There are already eight jobs waiting. Let them finish or cancel one.');
+ const action = this.profile.validateAction ? this.profile.validateAction(input) : validateAction(input, this.profile.validateModel);
+ const job = { id: randomUUID(), type: action.type, model: action.model || null, models: action.models || [], status: 'queued', progress: 0, message: 'Waiting for the model operator', createdAt: new Date().toISOString(), output: '', result: null };
+ this.store.data.jobs.unshift(job);
+ this.store.data.jobs = this.store.data.jobs.filter((j, i) => i < 40 || !terminal(j));
+ this.actions.set(job.id, action); this.queue.push(job.id);
+ await this.store.save(); this.emit('job', job);
+ void this.drain();
+ return job;
+ }
+ async cancel(id) {
+ const job = this.store.data.jobs.find(j => j.id === id);
+ if (!job) throw new Error('Job not found.');
+ if (terminal(job)) return job;
+ this.aborts.get(id)?.abort(new Error('Cancelled by you.'));
+ if (job.status === 'queued') { job.status = 'cancelled'; job.message = 'Cancelled before starting.'; job.completedAt = new Date().toISOString(); this.queue = this.queue.filter(queued => queued !== id); this.actions.delete(id); await this.store.save(); }
+ else job.message = 'Cancelling…';
+ this.emit('job', job); return job;
+ }
+ async drain() {
+ if (this.processing) return;
+ this.processing = true;
+ try {
+ while (this.queue.length && !this.closing) {
+ const id = this.queue.shift(); const job = this.store.data.jobs.find(j => j.id === id); const action = this.actions.get(id);
+ if (!job || job.status !== 'queued') continue;
+ const abort = new AbortController(); this.aborts.set(id, abort);
+ job.status = 'running'; job.startedAt = new Date().toISOString();
+ const progress = (message, value, extra) => { job.message = message; if (Number.isFinite(value)) job.progress = Math.max(0, Math.min(100, value)); if (extra) job.download = extra; this.emit('job', job); };
+ try {
+ await this.store.save();
+ progress(`Connecting to ${this.profile.name}`, 1);
+ await this.adapter.ensureRunning(this.store.directory, { signal: abort.signal, onProgress: progress });
+ await this.refresh();
+ job.result = await this.execute(action, { signal: abort.signal, progress, job });
+ abort.signal.throwIfAborted();
+ job.status = 'succeeded'; job.progress = 100; job.message = job.result.message;
+ } catch (error) {
+ job.status = abort.signal.aborted ? 'cancelled' : 'failed';
+ job.message = abort.signal.aborted ? 'Cancelled. Any downloaded files stay cached; retrying can resume the download.' : safeError(error);
+ } finally {
+ job.completedAt = new Date().toISOString(); this.aborts.delete(id); this.actions.delete(id);
+ try { await this.refresh(); await this.store.save(); } catch (error) { job.persistenceError = error.message; }
+ this.emit('job', job);
+ }
+ }
+ } finally { this.processing = false; }
+ }
+ async deploy(model, { signal, progress }, load = true, context) {
+ await this.refresh();
+ if (!this.runtime.models.some(m => m.id === model)) {
+ progress(`Inspecting ${model} before downloading`, 2);
+ await this.adapter.pullModel(model, { signal, onProgress: (message, value, extra) => progress(message, 3 + (value || 0) * 0.75, extra) });
+ }
+ await this.adapter.localModelDetails(model, { signal });
+ if (load) {
+ progress(`Loading ${model} into memory`, 82);
+ await this.adapter.loadModel(model, { signal, context: context ?? (this.profile.preserveServingConfig ? undefined : 4096) });
+ for (let i = 0; i < 15; i++) { await this.refresh(); if (this.runtime.models.some(m => m.id === model && m.running)) return; await delay(200, undefined, { signal }); }
+ throw new Error(`${this.profile.name} accepted the load request, but the model did not appear in memory. Check the runtime log and try again.`);
+ }
+ await this.refresh();
+ if (!this.runtime.models.some(m => m.id === model)) throw new Error(`${this.profile.name} did not report the model as installed after the download.`);
+ }
+ async execute(action, ctx) {
+ const { signal, progress, job } = ctx;
+ if (action.model) job.activeModel = action.model;
+ if (action.type === 'start') return { message: `${this.profile.name} ${this.runtime.version} is ready. ${this.runtime.models.length} models found on disk.` };
+ if (action.type === 'serve') {
+ await this.deploy(action.model, ctx, false);
+ progress(`Starting ${action.model} with the requested serving settings`, 80);
+ await this.adapter.loadModel(action.model, { ...action, signal });
+ await this.refresh();
+ return { model: action.model, serving: this.runtime.serving, message: `${action.model} is serving · ${this.runtime.serving.context.toLocaleString()} context · ${this.runtime.serving.maxSequences} parallel sequences · prefix cache ${this.runtime.serving.prefixCaching ? 'on' : 'off'}.` };
+ }
+ if (action.type === 'load_test') {
+ await this.deploy(action.model, ctx);
+ progress('Warming the server before the concurrent test', 5);
+ await this.adapter.generate(action.model, [{ role: 'user', content: BENCHMARK_PROMPT }], { ...BENCHMARK_SETTINGS, maxTokens: 32, signal });
+ const serving = { ...this.runtime.serving }; const samples = []; let next = 0; let complete = 0;
+ const began = performance.now();
+ const worker = async () => {
+ while (next < action.requests) {
+ signal.throwIfAborted(); const index = next++;
+ try {
+ const { metrics } = await this.adapter.generate(action.model, [{ role: 'user', content: `${BENCHMARK_PROMPT} Reference number: ${index + 1}.` }], { ...BENCHMARK_SETTINGS, signal });
+ samples.push({ index, status: 'succeeded', ...metrics });
+ } catch (error) { signal.throwIfAborted(); samples.push({ index, status: 'failed', error: safeError(error) }); }
+ complete++; progress(`${complete} of ${action.requests} requests completed · concurrency ${action.concurrency}`, 10 + complete / action.requests * 85);
+ }
+ };
+ const outcomes = await Promise.allSettled(Array.from({ length: action.concurrency }, worker)); signal.throwIfAborted();
+ const rejected = outcomes.find(outcome => outcome.status === 'rejected'); if (rejected) throw rejected.reason;
+ const totalMs = performance.now() - began;
+ const passed = samples.filter(s => s.status === 'succeeded');
+ const latency = passed.map(s => s.totalMs).sort((a, b) => a - b);
+ const result = { id: randomUUID(), at: new Date().toISOString(), model: action.model, digest: this.runtime.models.find(m => m.id === action.model)?.digest, runtime: { name: this.profile.name, version: this.runtime.version, backend: this.runtime.backend, backendVersion: this.runtime.backendVersion }, hardware: this.system, concurrency: action.concurrency, requests: action.requests, succeeded: passed.length, failed: samples.length - passed.length, totalMs, aggregateTokensPerSecond: passed.reduce((sum, s) => sum + s.tokens, 0) / (totalMs / 1000), medianLatencyMs: median(latency), p95LatencyMs: latency.length ? latency[Math.ceil(latency.length * 0.95) - 1] : null, medianFirstTokenMs: median(passed.map(s => s.firstTokenMs)), settings: { ...BENCHMARK_SETTINGS, context: serving.context }, serving, prompt: BENCHMARK_PROMPT, promptVariation: 'Reference number: {1-based request index}.', mode: 'one discarded warmup; repeated prompt prefixes may be cached; client end-to-end measurements', samples: samples.sort((a, b) => a.index - b.index) };
+ this.store.data.loadTests.unshift(result); this.store.data.loadTests = this.store.data.loadTests.slice(0, 200); await this.store.save();
+ if (!passed.length) throw new Error(`All ${action.requests} requests failed. The load-test record contains each error.`);
+ return { model: action.model, loadTestId: result.id, message: `${passed.length}/${action.requests} requests succeeded at concurrency ${action.concurrency} · ${result.aggregateTokensPerSecond.toFixed(1)} aggregate tokens/s · ${Math.round(result.p95LatencyMs)} ms p95 request latency · ${result.failed} errors.` };
+ }
+ if (['deploy', 'download'].includes(action.type)) {
+ await this.deploy(action.model, ctx, action.type === 'deploy');
+ return { model: action.model, message: action.type === 'deploy' ? `${action.model} is ready on this Mac. Ask a question, or say “benchmark it”. ${this.profile.readyMessage || 'Idle models unload after 15 minutes; their files stay on disk.'}` : `${action.model} is downloaded. Say “run ${action.model}” to load it into memory.` };
+ }
+ if (['unload', 'unload_all'].includes(action.type)) {
+ const targets = action.type === 'unload_all' ? this.runtime.models.filter(m => m.running).map(m => m.id) : [action.model];
+ for (const model of targets) { progress(`Releasing ${model} from memory`, 30); await this.adapter.unloadModel(model, { signal }); }
+ await this.refresh();
+ if (targets.some(model => this.runtime.models.some(m => m.id === model && m.running))) throw new Error('The runtime still reports a requested model in memory. It may be in use by another client.');
+ return { message: targets.length ? `${targets.length === 1 ? targets[0] : `${targets.length} models`} unloaded. Downloaded files are still on disk.` : 'No models were in memory.' };
+ }
+ if (action.type === 'chat') {
+ await this.deploy(action.model, ctx);
+ progress(`Generating with ${action.model}`, 90);
+ let lastEmit = 0;
+ const generated = await this.adapter.generate(action.model, [...action.history, { role: 'user', content: action.prompt }], { signal, onToken: text => { job.output += text; if (Date.now() - lastEmit > 80) { this.emit('job', job); lastEmit = Date.now(); } } });
+ const run = { id: randomUUID(), at: new Date().toISOString(), ...generated.metrics };
+ this.store.data.runs.unshift(run); this.store.data.runs = this.store.data.runs.slice(0, 200);
+ return { model: action.model, metrics: generated.metrics, message: `Response from ${action.model}.` };
+ }
+ if (action.type === 'benchmark') {
+ const results = [];
+ for (const [modelIndex, model] of action.models.entries()) {
+ job.activeModel = model;
+ await this.deploy(model, ctx, true, BENCHMARK_SETTINGS.context);
+ const metadata = this.runtime.models.find(m => m.id === model);
+ progress(`Warming up ${model} (not included in results)`, 5 + modelIndex / action.models.length * 90);
+ await this.adapter.generate(model, [{ role: 'user', content: BENCHMARK_PROMPT }], { ...BENCHMARK_SETTINGS, maxTokens: 32, signal });
+ const samples = [];
+ for (let i = 0; i < 3; i++) {
+ progress(`${model} · measured run ${i + 1} of 3`, 10 + (modelIndex + i / 3) / action.models.length * 85);
+ const { metrics } = await this.adapter.generate(model, [{ role: 'user', content: BENCHMARK_PROMPT }], { ...BENCHMARK_SETTINGS, signal });
+ if (!Number.isFinite(metrics.tokensPerSecond) || !Number.isFinite(metrics.firstTokenMs)) throw new Error(`${this.profile.name} did not return usable timing data; no benchmark was recorded.`);
+ samples.push(metrics);
+ }
+ await this.refresh();
+ const result = { id: randomUUID(), model, digest: metadata?.digest || null, at: new Date().toISOString(), runtime: { name: this.profile.name, version: this.runtime.version, mlxVersion: this.runtime.mlxVersion, backend: this.runtime.backend, backendVersion: this.runtime.backendVersion }, hardware: this.system, settings: BENCHMARK_SETTINGS, prompt: BENCHMARK_PROMPT, promptHash: createHash('sha256').update(BENCHMARK_PROMPT).digest('hex'), mode: this.profile.benchmarkMode || 'warm; repeated prompt may use prefix cache', samples, medianTokensPerSecond: median(samples.map(s => s.tokensPerSecond)), medianFirstTokenMs: median(samples.map(s => s.firstTokenMs)), medianFirstOutputMs: median(samples.map(s => s.firstOutputMs)), residentBytes: this.runtime.models.find(m => m.id === model)?.resident?.size || null };
+ this.store.data.benchmarks.unshift(result); this.store.data.benchmarks = this.store.data.benchmarks.slice(0, 200); results.push(result);
+ await this.store.save();
+ }
+ return { model: action.models.at(-1), benchmarkIds: results.map(r => r.id), message: results.map(r => `${r.model}: ${r.medianTokensPerSecond.toFixed(1)} tokens/s · ${Math.round(r.medianFirstTokenMs)} ms to first token (median of 3 warm runs).`).join('\n') };
+ }
+ throw new Error('Unknown action.');
+ }
+ async close() { this.closing = true; for (const abort of this.aborts.values()) abort.abort(new Error('Harness shutting down.')); await this.store.save(); await this.adapter.close?.(); }
+}
diff --git a/store/agents/mlx-lm/src/intents.mjs b/store/agents/mlx-lm/src/intents.mjs
new file mode 100644
index 000000000..7aa35e8df
--- /dev/null
+++ b/store/agents/mlx-lm/src/intents.mjs
@@ -0,0 +1,91 @@
+import { CATALOG, DEFAULT_MODEL } from './catalog.mjs';
+import { validateModel } from './ollama.mjs';
+
+export const HELP = 'Try “run a lightweight model”, “find a model for coding”, “benchmark it”, “ask it: explain unified memory”, or “free up memory”. You can also name any local Ollama model, such as qwen3:0.6b.';
+
+// Only explicit operational requests enter the executor. Model replies are text,
+// never instructions. There is no shell-command generation or eval here.
+export function resolveIntent(input, { models = [], selectedModel, profile = {} } = {}) {
+ const catalog = profile.catalog || CATALOG;
+ const defaultModel = profile.defaultModel || DEFAULT_MODEL;
+ const normalize = profile.validateModel || validateModel;
+ const help = profile.help || HELP;
+ if (typeof input !== 'string' || !input.trim() || input.length > 4000) throw new Error('Enter a request between 1 and 4,000 characters.');
+ const text = input.trim();
+ const lower = text.toLowerCase();
+ const selected = selectedModel ? normalize(selectedModel) : null;
+ const preferred = selected || models.find(m => m.running)?.id || models[0]?.id;
+ const action = (type, props = {}) => ({ kind: 'action', action: { type, ...props } });
+ const reply = message => ({ kind: 'reply', message });
+ const choose = () => reply('Which model? Select one in the model map, or include its name in your request.');
+ // Prompt contents must never be parsed as commands or model identifiers.
+ const ask = text.match(/^(?:(?:please |can you )?)(?:ask|tell|chat with)\s+(.+?)\s*:\s*([\s\S]+)$/i);
+ if (ask) {
+ const target = /^(it|me|us|the model|my model)$/i.test(ask[1]) ? preferred : normalize(ask[1].trim());
+ return target ? action('chat', { model: target, prompt: ask[2] }) : choose();
+ }
+ const names = [...new Set([...models.map(m => m.id), ...catalog.map(m => m.id)])];
+ const mentioned = names.filter(name => lower.includes(name.toLowerCase()));
+ for (const match of text.matchAll(profile.modelPattern || /\b(?:[a-z0-9][a-z0-9._-]*\/)?[a-z0-9][a-z0-9._-]*:[a-z0-9][a-z0-9._-]*\b/g)) {
+ if (!mentioned.includes(match[0]) && !/^\d+:\d+$/.test(match[0])) mentioned.push(normalize(match[0]));
+ }
+ const modelForTask = /cod(?:e|ing)|programming/.test(lower) ? catalog[1].id : /writing|summari[sz]|rewrite/.test(lower) ? catalog[2].id : defaultModel;
+ const lead = lower.replace(/^(?:please\s+|(?:can|could|would) you\s+|i(?:'d| would) like (?:you )?to\s+|i want (?:you )?to\s+)+/, '');
+ if (/\b(don['’]?t|do not|never|not yet|cancel|abort)\b/.test(lower) && /\b(run|load|download|deploy|start|stop|unload|benchmark|compare|cancel|abort)\b/.test(lower)) return reply('I haven’t started an action. Use the Cancel button on an active job to stop it.');
+ if (/^(how (?:do|would|can|should)|what (?:if|happens)|why|explain|help)\b/.test(lead)) return reply(help);
+ if (/^(show|list|what|which|check|status|refresh)\b/.test(lead) && /models?|running|loaded|status|memory|installed/.test(lead) || /^(status|models|refresh)[.!?]?$/.test(lead)) return { kind: 'inventory' };
+ if (/^(find|recommend|suggest|pick|choose|what['’]?s|what is)\b/.test(lead) && /model|coding|writing/.test(lead)) return { kind: 'catalog', message: 'Start small, then choose a larger model once you have a baseline. These downloads run locally on this Mac.', models: /coding|code|programming/.test(lower) ? [catalog[1]] : /writing|summari[sz]/.test(lower) ? [catalog[2]] : catalog };
+ if (/^(delete|remove|erase)\b/.test(lead)) return reply('This pilot keeps downloaded models on disk. To release their memory, say “unload it” or “free up memory”.');
+ if (/^(free (?:up )?(?:my |the )?memory|unload all(?: models)?|stop all(?: models)?|release (?:all )?memory)[.!?]?$/.test(lead)) return action('unload_all');
+ if (/^(start|launch|open) (?:the )?(?:ollama|mlx(?:-lm)?|runtime|server)[.!?]?$/.test(lead)) return action('start');
+
+ const operation = lead.match(/^(run|deploy|load|start|download|pull|install|unload|stop|benchmark|test|measure|compare)\b/);
+ if (operation) {
+ if (/\b(?:and then|then|after that)\b/.test(lead) || /\band\s+(run|deploy|download|stop|unload|benchmark|ask)\b/.test(lead)) return reply('Let’s do one operation at a time. Start with the first request, then ask for the next step when it finishes.');
+ const verb = operation[1];
+ const rest = lead.slice(verb.length).trim().replace(/[.!?]+$/, '');
+ let model = mentioned[0];
+ if (!model && /^[a-z0-9][a-z0-9._-]*(?:\/[a-z0-9][a-z0-9._-]*)?$/.test(rest) && !/^(it|this|that|model|ollama|speed|performance|all)$/.test(rest)) {
+ model = names.find(name => name.split(':')[0] === rest) || normalize(rest);
+ }
+ if (!model && /\b(it|this model|that model|my model|the model|the running model)\b/.test(rest)) model = preferred;
+ if (['benchmark', 'test', 'measure', 'compare'].includes(verb)) {
+ if (verb === 'compare' && /\ball\b/.test(rest)) return models.length > 4 ? reply('This pilot compares up to four models at a time. Name the models to compare.') : models.length ? action('benchmark', { models: models.map(m => m.id) }) : choose();
+ const targets = mentioned.length ? mentioned : model ? [model] : preferred ? [preferred] : [];
+ if (verb === 'compare' && targets.length < 2) return reply('Name two models from this framework to compare. Each gets the same prompt and settings.');
+ return targets.length ? action('benchmark', { models: targets }) : choose();
+ }
+ if (['unload', 'stop'].includes(verb)) return model ? action('unload', { model }) : choose();
+ if (!model && /\b(lightweight|small|tiny|fast|coding|writing|starter|first)\b/.test(rest)) model = modelForTask;
+ if (!model) return reply('Name the model you want, or say “run a lightweight model” and I’ll use Qwen 3 0.6B.');
+ return action(['download', 'pull', 'install'].includes(verb) ? 'download' : 'deploy', { model });
+ }
+ if (/\b(benchmark|speed|fast|tokens per second|performance)\b/.test(lower) && /^(how|test|measure)\b/.test(lead)) return preferred ? action('benchmark', { models: [preferred] }) : choose();
+ if (/^(chat|talk)(?: with| to)?(?: it| the model)?[.!?]?$/.test(lead)) return preferred ? reply(`${preferred} is selected. Ask a question below, or say “ask it: …”.`) : choose();
+ if (/^(run|deploy|download|load|start|stop|unload|benchmark|compare|install|delete|remove)\b/.test(lead) || /\b(shell|terminal|sudo|rm -rf)\b/.test(lower)) return reply(help);
+ return preferred ? action('chat', { model: preferred, prompt: text }) : reply(help);
+}
+
+export function validateAction(input, normalize = validateModel) {
+ if (!input || typeof input !== 'object' || Array.isArray(input)) throw new Error('Provide a model action.');
+ const { type } = input;
+ if (!['start', 'download', 'deploy', 'unload', 'unload_all', 'benchmark', 'chat'].includes(type)) throw new Error('That action is not supported by this pilot.');
+ const action = { type };
+ if (['download', 'deploy', 'unload', 'chat'].includes(type)) action.model = normalize(input.model);
+ if (type === 'benchmark') {
+ if (!Array.isArray(input.models) || !input.models.length || input.models.length > 4) throw new Error('Benchmark between one and four models at a time.');
+ action.models = [...new Set(input.models.map(normalize))];
+ }
+ if (type === 'chat') {
+ if (typeof input.prompt !== 'string' || !input.prompt.trim() || input.prompt.length > 4000) throw new Error('Enter a prompt between 1 and 4,000 characters.');
+ action.prompt = input.prompt.trim();
+ action.history = [];
+ if (Array.isArray(input.history)) {
+ action.history = input.history.slice(-6).map(m => {
+ if (!['user', 'assistant'].includes(m.role) || typeof m.content !== 'string' || m.content.length > 6000) throw new Error('Invalid chat history.');
+ return { role: m.role, content: m.content };
+ });
+ }
+ }
+ return action;
+}
diff --git a/store/agents/mlx-lm/src/mlx.mjs b/store/agents/mlx-lm/src/mlx.mjs
new file mode 100644
index 000000000..a347b2ea1
--- /dev/null
+++ b/store/agents/mlx-lm/src/mlx.mjs
@@ -0,0 +1,156 @@
+import { spawn } from 'node:child_process';
+import { access, mkdir, open } from 'node:fs/promises';
+import { constants } from 'node:fs';
+import { createInterface } from 'node:readline';
+import { join } from 'node:path';
+import { randomUUID } from 'node:crypto';
+import { totalmem } from 'node:os';
+import { performance } from 'node:perf_hooks';
+
+export function validateMlxModel(model) {
+ if (typeof model !== 'string' || model.length > 180 || !/^[A-Za-z0-9][A-Za-z0-9_.-]*\/[A-Za-z0-9][A-Za-z0-9_.-]*$/.test(model) || model.includes('..')) throw new Error('Use a Hugging Face model ID, such as mlx-community/Qwen3-0.6B-4bit.');
+ return model;
+}
+
+export const MLX_CATALOG = [
+ { id: 'mlx-community/Qwen3-0.6B-4bit', name: 'Qwen 3 · 0.6B', purpose: 'A lightweight first model', description: 'A small MLX model for trying local chat and measuring this Mac.', downloadGB: 0.4, use: 'general', small: true },
+ { id: 'mlx-community/Qwen2.5-Coder-1.5B-Instruct-4bit', name: 'Qwen 2.5 Coder · 1.5B', purpose: 'Small coding assistant', description: 'A compact MLX coding model for explanations and short snippets.', downloadGB: 1.0, use: 'coding', small: true },
+ { id: 'mlx-community/Qwen2.5-0.5B-Instruct-4bit', name: 'Qwen 2.5 · 0.5B', purpose: 'Small baseline', description: 'A compact instruction model for short answers and a second performance baseline.', downloadGB: 0.3, use: 'writing', small: true },
+].map(model => ({ ...model, source: `https://huggingface.co/${model.id}` }));
+
+export const MLX_PROFILE = {
+ id: 'mlx-lm', name: 'MLX-LM', catalog: MLX_CATALOG, defaultModel: MLX_CATALOG[0].id,
+ validateModel: validateMlxModel, modelPattern: /\b[A-Za-z0-9][A-Za-z0-9_.-]*\/[A-Za-z0-9][A-Za-z0-9_.-]*\b/g,
+ help: 'Try “run a lightweight model”, “benchmark it”, “ask it: explain unified memory”, or “free up memory”. Use Hugging Face MLX model IDs, such as mlx-community/Qwen3-0.6B-4bit.',
+ benchmarkMode: 'warm model; fresh prompt cache for each sample',
+ readyMessage: 'MLX-LM keeps one model loaded per workspace. Switching models releases the previous model; idle models unload after 15 minutes.',
+};
+
+export class MlxAdapter {
+ constructor({ packageRoot, dataDir, python, interpreterArgs = ['-u'] } = {}) {
+ this.python = python || join(packageRoot, '.venv/bin/python');
+ this.interpreterArgs = interpreterArgs;
+ this.workerPath = join(packageRoot, 'worker.py');
+ this.dataDir = dataDir;
+ this.child = null; this.starting = null; this.pending = new Map(); this.generation = 0;
+ this.current = { models: [] }; this.version = null; this.mlxVersion = null; this.expiresAt = null;
+ this.installed = false; this.closed = false;
+ }
+ async ensureRunning(_directory, { signal } = {}) {
+ signal?.throwIfAborted();
+ if (this.closed) throw new Error('The MLX-LM workspace is closing.');
+ if (this.stopping) await this.stopping;
+ signal?.throwIfAborted();
+ if (this.closed) throw new Error('The MLX-LM workspace is closing.');
+ if (this.child && this.ready) return;
+ if (this.starting) return this.starting;
+ this.starting = this.startWorker().finally(() => { this.starting = null; });
+ return this.starting;
+ }
+ async startWorker() {
+ try { await access(this.python, constants.X_OK); this.installed = true; }
+ catch { throw new Error('MLX-LM needs setup. Run this harness’s setup command to install its isolated Python environment.'); }
+ await mkdir(this.dataDir, { recursive: true });
+ const log = await open(join(this.dataDir, 'mlx-lm.log'), 'a', 0o600);
+ const child = spawn(this.python, [...this.interpreterArgs, this.workerPath], { stdio: ['pipe', 'pipe', log.fd], env: { ...process.env, HF_HUB_DISABLE_TELEMETRY: '1', TOKENIZERS_PARALLELISM: 'false', HARNESS_MLX_REGISTRY: join(this.dataDir, 'mlx-models.json') } });
+ await log.close();
+ this.child = child; this.ready = false;
+ child.stdin.on('error', () => {}); // The write callback rejects the corresponding request.
+ const generation = ++this.generation;
+ await new Promise((resolve, reject) => {
+ const timer = setTimeout(() => { child.kill('SIGKILL'); reject(new Error('MLX-LM did not start within 45 seconds. Check .harness/mlx-lm.log.')); }, 45000);
+ const startupFailure = error => { clearTimeout(timer); reject(error); };
+ child.once('error', startupFailure);
+ const lines = createInterface({ input: child.stdout });
+ lines.on('line', line => {
+ try {
+ if (line.length > 8 * 1024 * 1024) throw new Error('MLX-LM returned an oversized event.');
+ const event = JSON.parse(line);
+ if (this.generation !== generation) return;
+ if (event.event === 'ready') {
+ clearTimeout(timer); this.ready = true; this.current = event;
+ this.version = event.version; this.mlxVersion = event.mlxVersion;
+ resolve(); return;
+ }
+ if (event.state) this.current = { ...this.current, ...event.state };
+ const pending = this.pending.get(event.id); if (!pending) return;
+ if (event.event === 'error') pending.reject(new Error(event.message));
+ else if (event.event === 'done') pending.resolve(event.result);
+ else pending.onEvent(event);
+ } catch (error) { child.kill('SIGKILL'); startupFailure(error); }
+ });
+ child.once('exit', (code, signal) => {
+ clearTimeout(timer); lines.close();
+ const error = new Error(`MLX-LM worker stopped (${signal || code}). Its loaded model was released. Check .harness/mlx-lm.log if unexpected.`);
+ if (this.generation === generation) { this.child = null; this.ready = false; this.expiresAt = null; this.current = { ...this.current, models: this.current.models.map(m => ({ ...m, running: false, resident: null })) }; }
+ for (const request of this.pending.values()) if (request.child === child) request.reject(error);
+ startupFailure(error);
+ });
+ });
+ }
+ stopWorker(child, signal = 'SIGTERM') {
+ if (child !== this.child) return;
+ this.ready = false;
+ if (this.stopping) return;
+ this.stopping = new Promise(resolve => {
+ const timer = setTimeout(() => child.kill('SIGKILL'), 1500);
+ child.once('close', () => { clearTimeout(timer); resolve(); });
+ child.kill(signal);
+ }).finally(() => { this.stopping = null; });
+ }
+ async request(action, body = {}, { signal, onEvent = () => {}, timeout = 180000 } = {}) {
+ await this.ensureRunning(undefined, { signal }); signal?.throwIfAborted();
+ const child = this.child; const id = randomUUID();
+ return new Promise((resolve, reject) => {
+ let settled = false;
+ const finish = (fn, value) => { if (settled) return; settled = true; clearTimeout(timer); signal?.removeEventListener('abort', abort); this.pending.delete(id); fn(value); };
+ const abort = () => { this.stopWorker(child); finish(reject, signal.reason || new Error('Cancelled.')); };
+ const timer = setTimeout(() => { this.stopWorker(child, 'SIGKILL'); finish(reject, new Error(`MLX-LM ${action} timed out. Its worker was stopped to release memory.`)); }, timeout);
+ this.pending.set(id, { child, resolve: value => finish(resolve, value), reject: error => finish(reject, error), onEvent });
+ signal?.addEventListener('abort', abort, { once: true });
+ child.stdin.write(JSON.stringify({ id, action, ...body }) + '\n', error => { if (error) finish(reject, error); });
+ });
+ }
+ async discover() {
+ let error;
+ try {
+ await this.ensureRunning();
+ if (!this.pending.size) {
+ const loaded = this.current.models.find(m => m.running);
+ if (loaded && this.expiresAt && Date.now() > this.expiresAt) await this.unloadModel(loaded.id);
+ await this.request('inventory');
+ }
+ } catch (e) { error = e.message; }
+ return { id: 'mlx-lm', name: 'MLX-LM', online: Boolean(this.child && this.ready), installed: this.installed, version: this.version, mlxVersion: this.mlxVersion, error,
+ endpoint: 'Private local worker', cacheRoot: this.current.cacheRoot, memoryScope: 'MLX active allocations; one model per workspace',
+ models: this.current.models.map(m => ({ ...m, resident: m.resident ? { ...m.resident, expires_at: this.expiresAt ? new Date(this.expiresAt).toISOString() : null } : null })) };
+ }
+ async pullModel(model, { signal, onProgress = () => {} } = {}) {
+ return this.request('download', { model: validateMlxModel(model), memoryBudget: Math.floor(totalmem() * 0.8) }, { signal, timeout: 30 * 60 * 1000, onEvent: event => { if (event.event === 'progress') onProgress(event.message, event.percent, { completed: event.completed, total: event.total }); } });
+ }
+ localModelDetails(model, options) { return this.request('details', { model: validateMlxModel(model) }, options); }
+ async loadModel(model, { signal, context = 4096 } = {}) {
+ const result = await this.request('load', { model: validateMlxModel(model), context }, { signal });
+ this.expiresAt = Date.now() + 15 * 60 * 1000; return result;
+ }
+ async unloadModel(model, { signal } = {}) {
+ const result = await this.request('unload', { model: validateMlxModel(model) }, { signal });
+ this.expiresAt = null; return result;
+ }
+ async generate(model, messages, { signal, onToken = () => {}, ...options } = {}) {
+ const started = performance.now(); let firstTokenMs = null; let firstOutputMs = null;
+ const result = await this.request('generate', { model: validateMlxModel(model), messages, options }, { signal, onEvent: event => {
+ if (event.event === 'first_token' && firstTokenMs === null) firstTokenMs = performance.now() - started;
+ if (event.event === 'token') { if (firstOutputMs === null && /\S/.test(event.text)) firstOutputMs = performance.now() - started; onToken(event.text); }
+ } });
+ this.expiresAt = Date.now() + 15 * 60 * 1000;
+ result.metrics.firstTokenMs = firstTokenMs; result.metrics.firstOutputMs = firstOutputMs;
+ return result;
+ }
+ async close() {
+ this.closed = true;
+ if (this.starting) await this.starting.catch(() => {});
+ if (this.child) this.stopWorker(this.child);
+ await this.stopping;
+ }
+}
diff --git a/store/agents/mlx-lm/src/ollama.mjs b/store/agents/mlx-lm/src/ollama.mjs
new file mode 100644
index 000000000..17063efec
--- /dev/null
+++ b/store/agents/mlx-lm/src/ollama.mjs
@@ -0,0 +1,187 @@
+import { access, mkdir, open, statfs } from 'node:fs/promises';
+import { constants } from 'node:fs';
+import { homedir, totalmem } from 'node:os';
+import { join } from 'node:path';
+import { spawn } from 'node:child_process';
+import { setTimeout as delay } from 'node:timers/promises';
+import { performance } from 'node:perf_hooks';
+
+export const OLLAMA_URL = 'http://127.0.0.1:11434';
+export const DEFAULT_CONTEXT = 4096;
+let starting;
+
+export function validateModel(model) {
+ if (typeof model !== 'string' || model.length > 180 || !/^[a-zA-Z0-9][a-zA-Z0-9._-]*(?:\/[a-zA-Z0-9][a-zA-Z0-9._-]*)?(?::[a-zA-Z0-9][a-zA-Z0-9._-]*)?$/.test(model) || model.includes('..')) throw new Error('Use an Ollama model name, such as qwen3:0.6b.');
+ if (/(?:^|[-:])cloud(?:$|[-:])/i.test(model)) throw new Error('This harness runs local models. Choose a model without a cloud tag.');
+ return model.includes(':') ? model : `${model}:latest`;
+}
+
+export async function* readNDJSON(body) {
+ const decoder = new TextDecoder();
+ let buffer = '';
+ for await (const chunk of body) {
+ buffer += typeof chunk === 'string' ? chunk : decoder.decode(chunk, { stream: true });
+ if (buffer.length > 8 * 1024 * 1024) throw new Error('The runtime returned an oversized event.');
+ let newline;
+ while ((newline = buffer.indexOf('\n')) !== -1) {
+ const line = buffer.slice(0, newline).trim(); buffer = buffer.slice(newline + 1);
+ if (line) { const event = JSON.parse(line); if (event.error) throw new Error(event.error); yield event; }
+ }
+ }
+ buffer += decoder.decode();
+ if (buffer.trim()) { const event = JSON.parse(buffer); if (event.error) throw new Error(event.error); yield event; }
+}
+
+export async function streamRequest(path, body, { signal, timeout = 30 * 60 * 1000, onEvent = () => {} } = {}) {
+ const response = await fetch(`${OLLAMA_URL}${path}`, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ ...body, stream: true }), signal: signal ? AbortSignal.any([signal, AbortSignal.timeout(timeout)]) : AbortSignal.timeout(timeout) });
+ if (!response.ok) { const message = await response.text(); throw new Error(message.slice(0, 600) || `Ollama returned ${response.status}`); }
+ let last;
+ for await (const event of readNDJSON(response.body)) { last = event; await onEvent(event); }
+ return last;
+}
+
+export async function ensureRunning(dataDir, { signal, onProgress = () => {} } = {}) {
+ try { return await ollamaJSON('/api/version', { timeout: 1500, signal }); } catch {}
+ signal?.throwIfAborted();
+ if (starting) return starting;
+ starting = (async () => {
+ const binary = await findOllama();
+ if (!binary) throw new Error('Ollama is not installed. Install the official Ollama app from ollama.com, then try again.');
+ onProgress('Starting Ollama on this Mac');
+ await mkdir(dataDir, { recursive: true });
+ const log = await open(join(dataDir, 'ollama.log'), 'a', 0o600);
+ let failure;
+ const child = spawn(binary, ['serve'], { env: { ...process.env, OLLAMA_HOST: '127.0.0.1:11434', OLLAMA_NO_CLOUD: '1' }, detached: true, stdio: ['ignore', log.fd, log.fd] });
+ child.on('error', error => { failure = error; });
+ child.unref();
+ await log.close();
+ const deadline = Date.now() + 30000;
+ while (Date.now() < deadline) {
+ if (failure) throw failure;
+ signal?.throwIfAborted();
+ try { return await ollamaJSON('/api/version', { timeout: 1000, signal }); } catch {}
+ if (child.exitCode !== null) throw new Error(`Ollama exited during startup. Details are in ${join(dataDir, 'ollama.log')}.`);
+ await delay(300, undefined, { signal });
+ }
+ throw new Error('Ollama did not become ready within 30 seconds. Check the runtime log and try again.');
+ })().finally(() => { starting = null; });
+ return starting;
+}
+
+export async function modelManifest(model, { signal } = {}) {
+ const normalized = validateModel(model);
+ const [path, tag] = normalized.split(':');
+ const repository = path.includes('/') ? path : `library/${path}`;
+ const url = `https://registry.ollama.ai/v2/${repository}/manifests/${tag}`;
+ const response = await fetch(url, { signal: signal ? AbortSignal.any([signal, AbortSignal.timeout(12000)]) : AbortSignal.timeout(12000), headers: { Accept: 'application/vnd.docker.distribution.manifest.v2+json' } });
+ if (!response.ok) throw new Error(response.status === 404 ? `The model ${model} was not found in Ollama's registry. Check its name and tag.` : `Could not inspect the download size (${response.status}). Try again when the registry is reachable.`);
+ const manifest = await response.json();
+ const layers = [...(manifest.layers || []), ...(manifest.config ? [manifest.config] : [])];
+ if (!layers.length || layers.some(layer => !Number.isSafeInteger(layer.size) || layer.size < 0)) throw new Error('The model registry returned an invalid manifest.');
+ return { bytes: layers.reduce((total, layer) => total + layer.size, 0), repository, tag };
+}
+
+export async function pullModel(model, options = {}) {
+ model = validateModel(model);
+ const manifest = await modelManifest(model, options);
+ if (manifest.bytes > totalmem() * 0.8) throw new Error(`This model download is ${(manifest.bytes / 1024 ** 3).toFixed(1)} GiB, too large for the pilot's memory budget on this Mac. Choose a smaller quantization or model.`);
+ try {
+ const disk = await statfs(process.env.OLLAMA_MODELS || join(homedir(), '.ollama'));
+ if (disk.bavail * disk.bsize < manifest.bytes + 2 * 1024 ** 3) throw new Error('There is not enough free disk space for this download plus 2 GiB of headroom.');
+ } catch (error) { if (!['ENOENT', 'EPERM', 'EACCES'].includes(error.code)) throw error; }
+ const layers = new Map();
+ const last = await streamRequest('/api/pull', { model }, { ...options, onEvent: event => {
+ if (event.digest && event.total) layers.set(event.digest, { total: event.total, completed: event.completed || 0 });
+ const known = [...layers.values()];
+ const completed = known.reduce((sum, layer) => sum + layer.completed, 0);
+ options.onProgress?.(event.status || 'Downloading model', Math.min(99, Math.floor(completed / manifest.bytes * 100)), { completed, total: manifest.bytes });
+ } });
+ if (last?.status !== 'success') throw new Error('The model download ended before Ollama confirmed success. You can retry to resume it.');
+ return manifest;
+}
+
+export async function localModelDetails(model, options = {}) {
+ const details = await ollamaJSON('/api/show', { method: 'POST', body: { model: validateModel(model) }, timeout: 15000, ...options });
+ if (details.remote_host || details.remote_model) throw new Error('This model forwards inference to a remote service. Choose a local model for this harness.');
+ if (Array.isArray(details.capabilities) && !details.capabilities.includes('completion')) throw new Error('This pilot supports language generation models. The selected model does not expose completion capability.');
+ return details;
+}
+
+export async function loadModel(model, { signal, context = DEFAULT_CONTEXT, keepAlive = '15m' } = {}) {
+ await localModelDetails(model, { signal });
+ return ollamaJSON('/api/generate', { method: 'POST', body: { model, prompt: '', stream: false, keep_alive: keepAlive, options: { num_ctx: context } }, timeout: 180000, signal });
+}
+
+export async function unloadModel(model, { signal } = {}) {
+ return ollamaJSON('/api/generate', { method: 'POST', body: { model: validateModel(model), stream: false, keep_alive: 0 }, timeout: 30000, signal });
+}
+
+export async function generate(model, messages, { signal, maxTokens = 512, context = DEFAULT_CONTEXT, keepAlive = '15m', seed = 42, temperature = 0.3, thinking = true, onToken = () => {} } = {}) {
+ const details = await localModelDetails(model, { signal });
+ const start = performance.now();
+ let firstTokenMs = null;
+ let firstOutputMs = null;
+ let text = '';
+ const body = { model, messages, options: { num_predict: maxTokens, num_ctx: context, temperature, seed }, keep_alive: keepAlive };
+ if (details.capabilities?.includes('thinking')) body.think = /^gpt-oss(?::|$)/.test(model) ? 'low' : Boolean(thinking);
+ const last = await streamRequest('/api/chat', body, { signal, timeout: 180000, onEvent: event => {
+ const piece = event.message?.content || '';
+ if (firstTokenMs === null && (piece || event.message?.thinking)) firstTokenMs = performance.now() - start;
+ if (piece) { if (firstOutputMs === null) firstOutputMs = performance.now() - start; text += piece; onToken(piece); }
+ } });
+ if (!last?.done) throw new Error('The model response ended unexpectedly. Partial output was not counted as a successful run.');
+ const elapsedMs = performance.now() - start;
+ return { text, metrics: {
+ model, ...responseMetrics(last), firstTokenMs, firstOutputMs, elapsedMs,
+ context, maxTokens, temperature, seed, thinking: body.think ?? null,
+ } };
+}
+
+export function responseMetrics(last) {
+ return {
+ tokens: last.eval_count ?? null,
+ promptTokens: last.prompt_eval_count ?? null,
+ tokensPerSecond: last.eval_duration > 0 && Number.isFinite(last.eval_count) ? last.eval_count / (last.eval_duration / 1e9) : null,
+ promptTokensPerSecond: last.prompt_eval_duration > 0 && Number.isFinite(last.prompt_eval_count) ? last.prompt_eval_count / (last.prompt_eval_duration / 1e9) : null,
+ loadMs: Number.isFinite(last.load_duration) ? last.load_duration / 1e6 : null,
+ evalMs: Number.isFinite(last.eval_duration) ? last.eval_duration / 1e6 : null,
+ doneReason: last.done_reason || null,
+ };
+}
+
+export async function findOllama() {
+ const candidates = [process.env.OLLAMA_BIN, '/usr/local/bin/ollama', '/opt/homebrew/bin/ollama', '/Applications/Ollama.app/Contents/Resources/ollama', ...String(process.env.PATH || '').split(':').map(p => join(p, 'ollama'))].filter(Boolean);
+ for (const candidate of candidates) {
+ try { await access(candidate, constants.X_OK); return candidate; } catch {}
+ }
+ return null;
+}
+
+export async function ollamaJSON(path, { method = 'GET', body, timeout = 5000, signal } = {}) {
+ const response = await fetch(`${OLLAMA_URL}${path}`, {
+ method, headers: { 'Content-Type': 'application/json' },
+ body: body === undefined ? undefined : JSON.stringify(body),
+ signal: signal ? AbortSignal.any([signal, AbortSignal.timeout(timeout)]) : AbortSignal.timeout(timeout),
+ });
+ const result = await response.json();
+ if (!response.ok || result.error) throw new Error(result.error || `Ollama returned ${response.status}`);
+ return result;
+}
+
+export async function discover() {
+ const binary = await findOllama();
+ try {
+ const [version, tags, running] = await Promise.all([ollamaJSON('/api/version'), ollamaJSON('/api/tags'), ollamaJSON('/api/ps')]);
+ return {
+ id: 'ollama', name: 'Ollama', online: true, installed: Boolean(binary), version: version.version, endpoint: OLLAMA_URL,
+ models: (tags.models || []).map(m => ({
+ id: m.name, name: m.name, size: m.size, digest: m.digest, modifiedAt: m.modified_at,
+ family: m.details?.family, parameters: m.details?.parameter_size, quantization: m.details?.quantization_level,
+ running: Boolean((running.models || []).find(r => r.name === m.name || r.model === m.name)),
+ resident: (running.models || []).find(r => r.name === m.name || r.model === m.name) || null,
+ })).filter(m => !/(?:^|[-:])cloud(?:$|[-:])/i.test(m.name)),
+ };
+ } catch (error) {
+ return { id: 'ollama', name: 'Ollama', online: false, installed: Boolean(binary), version: null, endpoint: OLLAMA_URL, models: [], error: error.message };
+ }
+}
diff --git a/store/agents/mlx-lm/src/server.mjs b/store/agents/mlx-lm/src/server.mjs
new file mode 100644
index 000000000..06984c602
--- /dev/null
+++ b/store/agents/mlx-lm/src/server.mjs
@@ -0,0 +1,143 @@
+import http from 'node:http';
+import { readFile, writeFile, rename } from 'node:fs/promises';
+import { fileURLToPath, pathToFileURL } from 'node:url';
+import { join, dirname, resolve } from 'node:path';
+import { randomBytes, timingSafeEqual } from 'node:crypto';
+import { Store } from './store.mjs';
+import { Controller } from './controller.mjs';
+import { createViewerRelay } from './viewer-relay.mjs';
+
+export const ROOT = dirname(dirname(fileURLToPath(import.meta.url)));
+const assets = { '/': ['index.html', 'text/html'], '/app.js': ['app.js', 'text/javascript'], '/styles.css': ['styles.css', 'text/css'], '/assets/ollama-icon.svg': ['assets/ollama-icon.svg', 'image/svg+xml'], '/assets/ollama-icon.png': ['assets/ollama-icon.png', 'image/png'], '/favicon.ico': ['assets/ollama-icon.png', 'image/png'], '/assets/mlx-icon.svg': ['assets/mlx-icon.svg', 'image/svg+xml'], '/assets/mlx-icon.png': ['assets/mlx-icon.png', 'image/png'] };
+assets['/assets/vllm-icon.png'] = ['assets/vllm-icon.png', 'image/png'];
+const json = (res, status, value) => { res.writeHead(status, { 'Content-Type': 'application/json; charset=utf-8' }); res.end(JSON.stringify(value)); };
+
+async function readBody(req) {
+ if (!/^application\/json(?:\s*;|$)/i.test(req.headers['content-type'] || '')) throw Object.assign(new Error('Use application/json for control requests.'), { status: 415 });
+ if (Number(req.headers['content-length']) > 65536) throw Object.assign(new Error('Request too large.'), { status: 413 });
+ const chunks = []; let size = 0;
+ for await (const chunk of req) { size += chunk.length; if (size > 65536) throw Object.assign(new Error('Request too large.'), { status: 413 }); chunks.push(chunk); }
+ try { return JSON.parse(Buffer.concat(chunks).toString('utf8')); } catch { throw Object.assign(new Error('Invalid JSON request.'), { status: 400 }); }
+}
+
+export function createServer(controller, { assetRoot = join(ROOT, 'dist') } = {}) {
+ const token = randomBytes(32).toString('hex');
+ const events = new Set();
+ const sendEvent = (res, type, data) => { if (res.writableLength > 1024 * 1024) { res.destroy(); events.delete(res); } else if (!res.destroyed) res.write(`event: ${type}\ndata: ${JSON.stringify(data)}\n\n`); };
+ const onState = state => { for (const res of events) sendEvent(res, 'state', state); };
+ const onJob = job => { for (const res of events) sendEvent(res, 'job', job); };
+ controller.on('state', onState); controller.on('job', onJob);
+ const server = http.createServer(async (req, res) => {
+ try {
+ const port = server.address()?.port;
+ const hosts = [`127.0.0.1:${port}`, `localhost:${port}`];
+ if (!hosts.includes(req.headers.host)) return json(res, 403, { error: 'Local requests only.' });
+ const origin = `http://${req.headers.host}`;
+ if (req.headers.origin && req.headers.origin !== origin || req.headers['sec-fetch-site'] === 'cross-site') return json(res, 403, { error: 'Cross-origin control is disabled.' });
+ const url = new URL(req.url, origin);
+ res.setHeader('X-Content-Type-Options', 'nosniff');
+ res.setHeader('Referrer-Policy', 'no-referrer');
+ res.setHeader('Cache-Control', 'no-store');
+ res.setHeader('Content-Security-Policy', "default-src 'self'; script-src 'self'; style-src 'self' 'unsafe-inline'; connect-src 'self'; img-src 'self' data:; frame-ancestors 'self'; base-uri 'none'; form-action 'self'");
+ if (req.method === 'GET' && url.pathname === '/api/health') return json(res, 200, { application: 'local-ai-harnesses', protocol: 1, workspace: controller.workspace, runtime: controller.profile?.id || 'ollama', pid: process.pid });
+ if (req.method === 'POST') {
+ const submitted = Buffer.from(String(req.headers['x-harness-token'] || ''));
+ const expected = Buffer.from(token);
+ if (submitted.length !== expected.length || !timingSafeEqual(submitted, expected)) return json(res, 403, { error: 'Refresh the viewer before sending a control request.' });
+ const body = await readBody(req);
+ if (!body || typeof body !== 'object' || Array.isArray(body)) return json(res, 400, { error: 'Expected a JSON object.' });
+ if (url.pathname === '/api/command') return json(res, 200, await controller.command(body.text, { selectedModel: body.selectedModel, history: body.history }));
+ if (url.pathname === '/api/action') return json(res, 202, { kind: 'job', job: await controller.submit(body) });
+ if (url.pathname === '/api/cancel') return json(res, 200, { job: await controller.cancel(body.id) });
+ }
+ if (req.method === 'GET' && url.pathname === '/api/state') { await controller.refresh(); return json(res, 200, { ...controller.snapshot(), token }); }
+ if (req.method === 'GET' && url.pathname === '/api/jobs') return json(res, 200, { jobs: controller.snapshot().jobs });
+ if (req.method === 'GET' && url.pathname === '/api/events') {
+ if (events.size >= 32) return json(res, 503, { error: 'Too many viewer connections.' });
+ res.writeHead(200, { 'Content-Type': 'text/event-stream', Connection: 'keep-alive', 'X-Accel-Buffering': 'no' });
+ res.write('retry: 2000\n\n'); events.add(res); sendEvent(res, 'state', controller.snapshot());
+ req.on('close', () => events.delete(res)); return;
+ }
+ if (req.method === 'GET' && url.pathname === '/api/export') {
+ res.setHeader('Content-Disposition', `attachment; filename="${controller.profile?.id || 'ollama'}-benchmarks.json"`);
+ return json(res, 200, { exportedAt: new Date().toISOString(), system: controller.system, benchmarks: controller.snapshot().benchmarks, loadTests: controller.snapshot().loadTests });
+ }
+ if (req.method === 'GET' && Object.hasOwn(assets, url.pathname)) {
+ let [file, type] = assets[url.pathname];
+ if (url.pathname === '/favicon.ico' && controller.profile?.id === 'mlx-lm') file = 'assets/mlx-icon.png';
+ if (url.pathname === '/favicon.ico' && controller.profile?.id === 'vllm') file = 'assets/vllm-icon.png';
+ const data = await readFile(join(assetRoot, file));
+ res.writeHead(200, { 'Content-Type': `${type}; charset=utf-8` }); return res.end(data);
+ }
+ json(res, 404, { error: 'Not found.' });
+ } catch (error) { if (!res.headersSent) json(res, error.status || 400, { error: error.message }); else res.end(); }
+ });
+ const heartbeat = setInterval(() => { for (const res of events) res.write(': heartbeat\n\n'); }, 15000); heartbeat.unref();
+ server.on('close', () => { clearInterval(heartbeat); controller.off('state', onState); controller.off('job', onJob); });
+ server.closeEvents = () => { for (const res of events) res.end(); events.clear(); };
+ server.requestTimeout = 30000;
+ return server;
+}
+
+export async function start(port = Number(process.env.HARNESS_VIEWER_PORT || process.env.HARNESS_PORT || 4310), options = {}) {
+ if (!Number.isInteger(port) || port < 0 || port > 65535) throw new Error('The viewer port must be between 0 and 65535.');
+ const workspace = resolve(process.env.HARNESS_WORKSPACE || options.workspaceRoot || ROOT);
+ const runtime = options.profile?.id || 'ollama';
+ let marker;
+ try { marker = JSON.parse(await readFile(join(workspace, 'local-ai.json'), 'utf8')); }
+ catch (error) { if (error.code !== 'ENOENT') throw error; }
+ if (marker?.runtime && marker.runtime !== runtime) throw new Error(`This workspace belongs to ${marker.runtime}. Open a separate ${runtime} workspace.`);
+ let existing;
+ let conflictingRuntime;
+ try {
+ const endpoint = JSON.parse(await readFile(join(workspace, '.harness/endpoint.json'), 'utf8'));
+ if (endpoint.workspace === workspace && Number.isInteger(endpoint.port) && endpoint.port > 0 && endpoint.port <= 65535) {
+ const health = await (await fetch(`http://127.0.0.1:${endpoint.port}/api/health`, { signal: AbortSignal.timeout(1200) })).json();
+ if (health.application === 'local-ai-harnesses' && health.workspace === workspace && health.pid === endpoint.pid) {
+ if ((health.runtime || 'ollama') === runtime) existing = endpoint;
+ else conflictingRuntime = health.runtime || 'ollama';
+ }
+ }
+ } catch {}
+ if (conflictingRuntime) throw new Error(`This workspace is already serving ${conflictingRuntime}. Open a separate ${runtime} workspace.`);
+ if (existing) {
+ if (existing.port === port) throw new Error(`This workspace is already running at http://127.0.0.1:${port}. Open that viewer instead of starting a second copy.`);
+ const relay = createViewerRelay(existing.port);
+ await new Promise((resolve, reject) => { relay.once('error', reject); relay.listen(port, '127.0.0.1', resolve); });
+ console.log(`Local: http://127.0.0.1:${relay.address().port} (shared workspace service)`);
+ const stopRelay = () => { relay.closeAllConnections(); relay.close(); };
+ process.once('SIGINT', stopRelay); process.once('SIGTERM', stopRelay);
+ return { server: relay, controller: null };
+ }
+ const store = await new Store(join(workspace, '.harness')).init();
+ const controller = new Controller(store, options.adapter, options.profile);
+ controller.workspace = workspace;
+ const server = createServer(controller, options);
+ await new Promise((resolve, reject) => { server.once('error', reject); server.listen(port, '127.0.0.1', resolve); });
+ const boundPort = server.address().port;
+ await writeFile(join(store.directory, 'endpoint.json'), JSON.stringify({ port: boundPort, pid: process.pid, workspace, runtime, startedAt: new Date().toISOString() }), { mode: 0o600 });
+ console.log(`Local: http://127.0.0.1:${boundPort}`);
+ let verdictWriting = Promise.resolve();
+ let lastVerdict;
+ const verdict = async snapshot => {
+ const last = snapshot.jobs[0];
+ const failed = last?.status === 'failed';
+ const affected = last?.activeModel || last?.model || last?.models?.join(', ') || controller.profile.name;
+ const failure = failed ? `${affected} · ${last.type} failed: ${last.message}` : undefined;
+ const data = { spec: 1, ready: snapshot.runtime.online && !failed, summary: failed ? failure : snapshot.runtime.online ? `${snapshot.runtime.models.length} local models · ${controller.profile.name} connected` : `${controller.profile.name} is ready to start`, findings: failed ? [{ severity: 'error', kind: 'runtime', message: failure }] : [], updatedAt: new Date().toISOString() };
+ const key = JSON.stringify({ ...data, updatedAt: undefined });
+ if (key === lastVerdict) return;
+ lastVerdict = key;
+ verdictWriting = verdictWriting.catch(() => {}).then(async () => { const temp = join(store.directory, 'verdict.json.tmp'); await writeFile(temp, JSON.stringify(data, null, 2), { mode: 0o600 }); await rename(temp, join(store.directory, 'verdict.json')); });
+ await verdictWriting;
+ };
+ controller.on('state', snapshot => { void verdict(snapshot).catch(() => {}); });
+ controller.on('job', () => { void verdict(controller.snapshot()).catch(() => {}); });
+ await controller.refresh();
+ const poll = setInterval(() => { void controller.refresh().catch(error => console.error('Discovery:', error.message)); }, 5000); poll.unref();
+ let stopping = false;
+ const stop = async () => { if (stopping) return; stopping = true; clearInterval(poll); server.closeEvents(); server.close(); await controller.close(); };
+ process.once('SIGINT', stop); process.once('SIGTERM', stop);
+ return { server, controller };
+}
+if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) await start();
diff --git a/store/agents/mlx-lm/src/store.mjs b/store/agents/mlx-lm/src/store.mjs
new file mode 100644
index 000000000..59957fa1b
--- /dev/null
+++ b/store/agents/mlx-lm/src/store.mjs
@@ -0,0 +1,21 @@
+import { mkdir, readFile, writeFile, rename } from 'node:fs/promises';
+import { join } from 'node:path';
+
+export class Store {
+ constructor(directory) { this.directory = directory; this.file = join(directory, 'state.json'); this.data = { benchmarks: [], loadTests: [], runs: [], jobs: [] }; this.writing = Promise.resolve(); }
+ async init() {
+ await mkdir(this.directory, { recursive: true });
+ try {
+ const parsed = JSON.parse(await readFile(this.file, 'utf8'));
+ for (const key of ['benchmarks', 'loadTests', 'runs', 'jobs']) if (Array.isArray(parsed[key])) this.data[key] = parsed[key];
+ this.data.jobs = this.data.jobs.map(job => ['queued', 'running'].includes(job.status) ? { ...job, status: 'interrupted', message: 'Harness restarted before this job completed. Retry the action to continue.', completedAt: new Date().toISOString() } : { ...job, message: String(job.message || '').replace(/https?:\/\/[^\s"<>]+/g, url => url.split('?')[0]).slice(0, 700) });
+ } catch (error) { if (error.code !== 'ENOENT') throw new Error(`Could not read saved Harness data: ${error.message}`); }
+ await this.save();
+ return this;
+ }
+ save() {
+ const content = JSON.stringify(this.data, null, 2);
+ this.writing = this.writing.catch(() => {}).then(async () => { const temp = `${this.file}.tmp`; await writeFile(temp, content, { mode: 0o600 }); await rename(temp, this.file); });
+ return this.writing;
+ }
+}
diff --git a/store/agents/mlx-lm/src/viewer-relay.mjs b/store/agents/mlx-lm/src/viewer-relay.mjs
new file mode 100644
index 000000000..7aa643ddd
--- /dev/null
+++ b/store/agents/mlx-lm/src/viewer-relay.mjs
@@ -0,0 +1,26 @@
+import http from 'node:http';
+
+// Multiple views of one workspace share one controller and job queue. The native
+// host assigns a fresh viewer port; this loopback relay serves that port without
+// opening a second writer against the workspace's history.
+export function createViewerRelay(targetPort) {
+ if (!Number.isInteger(targetPort) || targetPort < 1 || targetPort > 65535) throw new Error('Invalid local service port.');
+ const server = http.createServer((req, res) => {
+ const port = server.address().port;
+ const host = req.headers.host;
+ const reject = (status, message) => { res.writeHead(status, { 'Content-Type': 'application/json' }); res.end(JSON.stringify({ error: message })); };
+ if (![`localhost:${port}`, `127.0.0.1:${port}`].includes(host)) return reject(403, 'Local requests only.');
+ if (req.headers.origin && req.headers.origin !== `http://${host}` || req.headers['sec-fetch-site'] === 'cross-site') return reject(403, 'Cross-origin control is disabled.');
+ if (!req.url.startsWith('/') || req.url.startsWith('//')) return reject(400, 'Invalid local path.');
+ const headers = { ...req.headers, host: `127.0.0.1:${targetPort}` };
+ if (headers.origin) headers.origin = `http://127.0.0.1:${targetPort}`;
+ const upstream = http.request({ hostname: '127.0.0.1', port: targetPort, path: req.url, method: req.method, headers }, response => {
+ res.writeHead(response.statusCode || 502, response.headers); response.pipe(res);
+ });
+ upstream.on('error', () => { if (!res.headersSent) reject(502, 'The workspace service stopped. Reopen this viewer to reconnect.'); else res.end(); });
+ req.on('aborted', () => upstream.destroy()); res.on('close', () => upstream.destroy());
+ req.pipe(upstream);
+ });
+ server.requestTimeout = 30000;
+ return server;
+}
diff --git a/store/agents/mlx-lm/store.json b/store/agents/mlx-lm/store.json
new file mode 100644
index 000000000..782deffba
--- /dev/null
+++ b/store/agents/mlx-lm/store.json
@@ -0,0 +1,7 @@
+{
+ "homepage": "https://github.com/ml-explore/mlx-lm",
+ "upstream": "https://github.com/ml-explore/mlx-lm",
+ "license": "MIT",
+ "tagline": "Generating text with large language models on Apple silicon with MLX",
+ "screenshots": []
+}
diff --git a/store/agents/mlx-lm/template/local-ai.json b/store/agents/mlx-lm/template/local-ai.json
new file mode 100644
index 000000000..18983069d
--- /dev/null
+++ b/store/agents/mlx-lm/template/local-ai.json
@@ -0,0 +1 @@
+{ "spec": 1, "runtime": "mlx-lm", "scope": "local-language-models", "notes": "MLX-LM runs one loaded model per workspace. Weights stay in the Hugging Face cache; measurements belong to this workspace." }
diff --git a/store/agents/mlx-lm/toolchain/doctor.sh b/store/agents/mlx-lm/toolchain/doctor.sh
new file mode 100755
index 000000000..ea7453bdd
--- /dev/null
+++ b/store/agents/mlx-lm/toolchain/doctor.sh
@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+set -euo pipefail
+cd "$(dirname "$0")/.."
+. toolchain/node.sh
+"$LOCAL_AI_NODE" -e 'console.log(`ok Node ${process.versions.node}`)'
+if [ ! -x .venv/bin/python ]; then echo 'miss MLX-LM environment — run toolchain/setup.sh' >&2; exit 1; fi
+.venv/bin/python - <<'PY'
+import importlib.metadata, platform
+import mlx.core as mx
+assert platform.system() == 'Darwin' and platform.machine() == 'arm64', 'Apple Silicon macOS is required'
+assert mx.metal.is_available(), 'Metal is unavailable'
+print('ok MLX-LM', importlib.metadata.version('mlx-lm'))
+print('ok MLX', importlib.metadata.version('mlx'), '· Metal available')
+print('ok Local model worker · Hugging Face cache · no inference API key')
+PY
diff --git a/store/agents/mlx-lm/toolchain/local-ai b/store/agents/mlx-lm/toolchain/local-ai
new file mode 100755
index 000000000..4940fa91a
--- /dev/null
+++ b/store/agents/mlx-lm/toolchain/local-ai
@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+package_dir="$(cd "$(dirname "$0")/.." && pwd)"
+export HARNESS_WORKSPACE="${HARNESS_WORKSPACE:-$PWD}"
+export LOCAL_AI_RUNTIME=mlx-lm
+export LOCAL_AI_NAME=MLX-LM
+. "$package_dir/toolchain/node.sh"
+exec "$LOCAL_AI_NODE" "$package_dir/bin/harness.mjs" "$@"
diff --git a/store/agents/mlx-lm/toolchain/mlx-lm b/store/agents/mlx-lm/toolchain/mlx-lm
new file mode 100755
index 000000000..85909ff8f
--- /dev/null
+++ b/store/agents/mlx-lm/toolchain/mlx-lm
@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+set -euo pipefail
+package_dir="$(cd "$(dirname "$0")/.." && pwd)"
+exec "$package_dir/.venv/bin/python" -m mlx_lm "$@"
diff --git a/store/agents/mlx-lm/toolchain/node.sh b/store/agents/mlx-lm/toolchain/node.sh
new file mode 100644
index 000000000..df3e4ee05
--- /dev/null
+++ b/store/agents/mlx-lm/toolchain/node.sh
@@ -0,0 +1,15 @@
+# Use this Mac's Node or the interpreter managed by Harness, including GUI launches.
+local_ai_node_candidate="$(command -v node 2>/dev/null || true)"
+if [ -n "$local_ai_node_candidate" ] && "$local_ai_node_candidate" -e 'process.exit(Number(process.versions.node.split(".")[0]) >= 22 ? 0 : 1)' >/dev/null 2>&1; then
+ LOCAL_AI_NODE="$local_ai_node_candidate"
+else
+ local_ai_node_record="${ADAPTER_RUNTIME_DIR:-${HOME}/.harness/runtime}/current-node"
+ local_ai_node_candidate="$(cat "$local_ai_node_record" 2>/dev/null || true)"
+ if [ -n "$local_ai_node_candidate" ] && [ -x "${local_ai_node_candidate%/*}/node" ] && "${local_ai_node_candidate%/*}/node" -e 'process.exit(Number(process.versions.node.split(".")[0]) >= 22 ? 0 : 1)' >/dev/null 2>&1; then
+ LOCAL_AI_NODE="${local_ai_node_candidate%/*}/node"
+ else
+ echo 'miss Node.js 22 or newer — update Harness or install Node.js 22' >&2
+ return 1
+ fi
+fi
+export LOCAL_AI_NODE
diff --git a/store/agents/mlx-lm/toolchain/setup.sh b/store/agents/mlx-lm/toolchain/setup.sh
new file mode 100755
index 000000000..38479b21f
--- /dev/null
+++ b/store/agents/mlx-lm/toolchain/setup.sh
@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+set -euo pipefail
+cd "$(dirname "$0")/.."
+if [ "$(uname -s)" != Darwin ] || [ "$(uname -m)" != arm64 ]; then
+ echo 'miss This harness targets Apple Silicon macOS.' >&2
+ exit 1
+fi
+. toolchain/node.sh
+if [ ! -x .venv/bin/python ]; then
+ mlx_python=''
+ for candidate in /opt/homebrew/opt/[email protected]/bin/python3.12 /opt/homebrew/opt/[email protected]/bin/python3.13 /opt/homebrew/bin/python3 /usr/local/bin/python3; do
+ if [ -x "$candidate" ] && "$candidate" -c 'import sys; sys.exit(0 if sys.version_info >= (3, 10) else 1)' >/dev/null 2>&1; then mlx_python="$candidate"; break; fi
+ done
+ if [ -z "$mlx_python" ]; then echo 'miss Python 3.10+ for Apple Silicon is required for setup.' >&2; exit 1; fi
+ "$mlx_python" -m venv .venv
+fi
+if [ -x /opt/homebrew/bin/uv ]; then
+ /opt/homebrew/bin/uv pip sync --python .venv/bin/python requirements.lock
+else
+ .venv/bin/python -m pip install -r requirements.lock
+fi
+exec ./toolchain/doctor.sh
diff --git a/store/agents/mlx-lm/viewer.sh b/store/agents/mlx-lm/viewer.sh
new file mode 100755
index 000000000..ff6ca35df
--- /dev/null
+++ b/store/agents/mlx-lm/viewer.sh
@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+set -euo pipefail
+cd "$(dirname "$0")"
+. toolchain/node.sh
+exec "$LOCAL_AI_NODE" main.mjs
diff --git a/store/agents/mlx-lm/worker.py b/store/agents/mlx-lm/worker.py
new file mode 100644
index 000000000..6f0f1adf3
--- /dev/null
+++ b/store/agents/mlx-lm/worker.py
@@ -0,0 +1,260 @@
+#!/usr/bin/env python3
+"""Private JSON-lines bridge to MLX-LM. One loaded model per workspace worker."""
+import gc
+import importlib.metadata
+import json
+import os
+from pathlib import Path
+import re
+import shutil
+import sys
+import time
+import uuid
+
+# Vendor output belongs in the diagnostic log, never in the JSON protocol.
+protocol = sys.stdout
+sys.stdout = sys.stderr
+os.environ.setdefault("HF_HUB_DISABLE_TELEMETRY", "1")
+os.environ.setdefault("TOKENIZERS_PARALLELISM", "false")
+from huggingface_hub import HfApi, hf_hub_download
+from huggingface_hub.constants import HF_HUB_CACHE
+import mlx.core as mx
+from mlx_lm import load, stream_generate
+from mlx_lm.sample_utils import make_sampler
+
+model = tokenizer = None
+loaded_id = None
+loaded_context = 4096
+loaded_config = {}
+registry_path = Path(os.environ.get("HARNESS_MLX_REGISTRY", Path(__file__).parent / ".harness/mlx-models.json"))
+try:
+ revisions = json.loads(registry_path.read_text())
+ if not isinstance(revisions, dict):
+ revisions = {}
+except (OSError, ValueError):
+ revisions = {}
+extra_repos = set(revisions)
+cache_root = Path(HF_HUB_CACHE)
+
+
+def emit(value):
+ protocol.write(json.dumps(value, ensure_ascii=False, allow_nan=False) + "\n")
+ protocol.flush()
+
+
+def validate_id(value):
+ if not isinstance(value, str) or len(value) > 180 or not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]*/[A-Za-z0-9][A-Za-z0-9_.-]*", value) or ".." in value:
+ raise ValueError("Use a Hugging Face model ID, such as mlx-community/Qwen3-0.6B-4bit.")
+ return value
+
+
+def snapshot(repo):
+ base = cache_root / ("models--" + repo.replace("/", "--"))
+ revision = revisions.get(repo, "")
+ if isinstance(revision, str) and re.fullmatch(r"[a-f0-9]{40,64}", revision):
+ candidate = base / "snapshots" / revision
+ if candidate.is_dir():
+ return candidate
+ try:
+ revision = (base / "refs/main").read_text().strip()
+ if not re.fullmatch(r"[a-f0-9]{40,64}", revision):
+ raise ValueError("Invalid cached revision.")
+ candidate = base / "snapshots" / revision
+ if candidate.is_dir():
+ return candidate
+ except (OSError, ValueError):
+ pass
+ candidates = sorted((base / "snapshots").glob("*"), key=lambda p: p.stat().st_mtime, reverse=True)
+ return candidates[0] if candidates else None
+
+
+def describe(repo):
+ path = snapshot(repo)
+ if path is None:
+ raise ValueError(f"{repo} is not downloaded.")
+ config = json.loads((path / "config.json").read_text())
+ weights = list(path.glob("*.safetensors"))
+ index = path / "model.safetensors.index.json"
+ if index.exists():
+ expected = set(json.loads(index.read_text())["weight_map"].values())
+ if any(Path(name).name != name or not (path / name).is_file() for name in expected):
+ raise ValueError(f"{repo} has incomplete model weights; download it again to resume.")
+ if not weights or any(not p.is_file() for p in weights) or not (path / "tokenizer_config.json").is_file():
+ raise ValueError(f"{repo} has an incomplete snapshot; download it to resume.")
+ if "vision_config" in config and config.get("model_type") not in {"gemma3_text"}:
+ raise ValueError("This harness supports text models; multimodal models need MLX-VLM.")
+ quant = config.get("quantization") or config.get("quantization_config") or {}
+ bits = quant.get("bits")
+ size = sum(p.stat().st_size for p in path.iterdir() if p.is_file())
+ running = repo == loaded_id
+ return {
+ "id": repo, "name": repo.split("/")[-1], "size": size, "digest": path.name,
+ "family": config.get("model_type"), "parameters": None,
+ "quantization": f"{bits}-bit" if bits else str(config.get("torch_dtype", "Unquantized")),
+ "running": running, "resident": {"size": mx.get_active_memory(), "context_length": loaded_context} if running else None,
+ }
+
+
+def inventory():
+ repos = set(extra_repos)
+ for base in cache_root.glob("models--*"):
+ parts = base.name.split("--", 2)
+ if len(parts) == 3 and (parts[1] == "mlx-community" or "mlx" in parts[2].lower()):
+ repos.add(parts[1] + "/" + parts[2])
+ rows = []
+ for repo in sorted(repos):
+ try:
+ rows.append(describe(repo))
+ except (OSError, ValueError, KeyError):
+ continue # An incomplete download is not an installed model.
+ return {"models": rows, "activeMemoryBytes": mx.get_active_memory(), "cacheMemoryBytes": mx.get_cache_memory()}
+
+
+def unload():
+ global model, tokenizer, loaded_id, loaded_config
+ model = tokenizer = None
+ loaded_id = None
+ loaded_config = {}
+ gc.collect()
+ mx.clear_cache()
+
+
+def download(repo, request_id, memory_budget):
+ info = HfApi().model_info(repo, files_metadata=True)
+ # Safetensors and tokenizer/config data only. Never fetch or execute model Python.
+ suffixes = {".json", ".safetensors", ".model", ".tiktoken", ".txt", ".jinja"}
+ files = [f for f in info.siblings if Path(f.rfilename).suffix in suffixes]
+ if not any(f.rfilename.endswith(".safetensors") for f in files):
+ raise ValueError("This repository has no safetensors weights for MLX-LM.")
+ if any(f.size is None or f.size < 0 or Path(f.rfilename).is_absolute() or ".." in Path(f.rfilename).parts for f in files):
+ raise ValueError("The model repository returned invalid file metadata.")
+ total = sum(f.size for f in files)
+ if total > memory_budget:
+ raise ValueError("This download exceeds the pilot's 80% of RAM model-size budget. Choose a smaller model.")
+ cache_root.mkdir(parents=True, exist_ok=True)
+ if shutil.disk_usage(cache_root).free < total + 2 * 1024**3:
+ raise ValueError("Not enough free storage for the model plus 2 GiB of headroom.")
+ complete = 0
+ for index, file in enumerate(files):
+ emit({"id": request_id, "event": "progress", "message": f"Downloading {file.rfilename} · {index + 1}/{len(files)} files", "percent": int(complete / max(total, 1) * 100), "completed": complete, "total": total})
+ hf_hub_download(repo, file.rfilename, revision=info.sha, cache_dir=str(cache_root))
+ complete += file.size
+ # Pin this workspace's revision without changing another application's HF refs.
+ revisions[repo] = info.sha
+ registry_path.parent.mkdir(parents=True, exist_ok=True)
+ temp = registry_path.with_suffix(f".{uuid.uuid4().hex}.tmp")
+ temp.write_text(json.dumps(revisions))
+ temp.replace(registry_path)
+ extra_repos.add(repo)
+ return describe(repo)
+
+
+def load_model(repo, context):
+ global model, tokenizer, loaded_id, loaded_context, loaded_config
+ describe(repo)
+ if repo != loaded_id:
+ unload()
+ try:
+ model, tokenizer, loaded_config = load(str(snapshot(repo)), tokenizer_config={"trust_remote_code": False, "local_files_only": True}, return_config=True)
+ except Exception:
+ unload()
+ raise
+ loaded_id = repo
+ extra_repos.add(repo)
+ loaded_context = context
+ return describe(repo)
+
+
+def generate(repo, message, request_id):
+ if repo != loaded_id:
+ raise ValueError("Load the model before generating.")
+ options = message.get("options", {})
+ context = int(options.get("context", 4096))
+ maximum = int(options.get("maxTokens", 512))
+ messages = message["messages"]
+ template = tokenizer.chat_template or ""
+ thinking = bool(options.get("thinking", True)) and "enable_thinking" in template
+ if template:
+ prompt_text = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True, enable_thinking=thinking)
+ prompt = tokenizer.encode(prompt_text, add_special_tokens=False)
+ else:
+ prompt_text = "\n".join(m["role"] + ": " + m["content"] for m in messages) + "\nassistant:"
+ prompt = tokenizer.encode(prompt_text)
+ if len(prompt) + maximum > context:
+ raise ValueError(f"Prompt plus output budget exceeds this harness's {context:,}-token context. Shorten the conversation.")
+ mx.random.seed(int(options.get("seed", 42)))
+ mx.reset_peak_memory()
+ started = time.perf_counter()
+ raw = visible = ""
+ first_token = first_output = None
+ starts_thinking = thinking and bool(re.search(r"<think>\s*$", prompt_text))
+ last = None
+ for response in stream_generate(model, tokenizer, prompt, max_tokens=maximum, sampler=make_sampler(temp=float(options.get("temperature", 0.3))), max_kv_size=context, prefill_step_size=512):
+ last = response
+ elapsed = (time.perf_counter() - started) * 1000
+ if first_token is None:
+ first_token = elapsed
+ emit({"id": request_id, "event": "first_token"})
+ raw += response.text
+ current = raw
+ if thinking:
+ if starts_thinking:
+ current = raw.split("</think>", 1)[1] if "</think>" in raw else ""
+ current = re.sub(r"<think>.*?(?:</think>|$)", "", current, flags=re.S)
+ # Hold incomplete delimiters and whitespace until actual answer text exists.
+ # Otherwise the opening newline is incorrectly counted as visible output.
+ for tag in ("<think>", "</think>"):
+ for length in range(1, len(tag)):
+ if current.endswith(tag[:length]):
+ current = current[:-length]
+ break
+ current = current.lstrip()
+ if len(current) > len(visible):
+ if first_output is None:
+ first_output = elapsed
+ emit({"id": request_id, "event": "token", "text": current[len(visible):]})
+ visible = current
+ if last is None:
+ raise ValueError("MLX-LM returned no generation result.")
+ mx.synchronize()
+ metrics = {
+ "model": repo, "tokens": last.generation_tokens, "promptTokens": last.prompt_tokens,
+ "tokensPerSecond": last.generation_tps, "promptTokensPerSecond": last.prompt_tps,
+ "nativeFirstTokenMs": first_token, "nativeFirstOutputMs": first_output,
+ "peakMemoryBytes": round(last.peak_memory * 1e9), "thinking": thinking,
+ "doneReason": last.finish_reason, "wallMs": (time.perf_counter() - started) * 1000,
+ "measurement": "MLX-LM native generation_tps; TTFT includes local IPC; fresh prompt cache",
+ }
+ return {"text": visible, "metrics": metrics}
+
+
+emit({"event": "ready", "version": importlib.metadata.version("mlx-lm"), "mlxVersion": importlib.metadata.version("mlx"), "cacheRoot": str(cache_root), **inventory()})
+for line in sys.stdin:
+ request_id = None
+ try:
+ if len(line) > 100_000:
+ raise ValueError("Worker request too large.")
+ message = json.loads(line)
+ request_id = message["id"]
+ action = message["action"]
+ repo = validate_id(message["model"]) if "model" in message else None
+ if action == "inventory":
+ result = inventory()
+ elif action == "details":
+ result = describe(repo)
+ elif action == "download":
+ result = download(repo, request_id, int(message["memoryBudget"]))
+ elif action == "load":
+ result = load_model(repo, int(message.get("context", 4096)))
+ elif action == "unload":
+ if loaded_id == repo:
+ unload()
+ result = {"unloaded": repo}
+ elif action == "generate":
+ result = generate(repo, message, request_id)
+ else:
+ raise ValueError("Unknown worker action.")
+ emit({"id": request_id, "event": "done", "result": result, "state": inventory()})
+ except Exception as error:
+ emit({"id": request_id, "event": "error", "message": str(error)[:1500], "state": inventory()})
+unload()
diff --git a/store/agents/ollama/.gitignore b/store/agents/ollama/.gitignore
new file mode 100644
index 000000000..bfeee06c9
--- /dev/null
+++ b/store/agents/ollama/.gitignore
@@ -0,0 +1,10 @@
+.harness/
+.venv/
+.runtime/
+__pycache__/
+*.pyc
+*.log
+node_modules/
+.DS_Store
+!/dist/
+!/dist/**
diff --git a/store/agents/ollama/LICENSE b/store/agents/ollama/LICENSE
new file mode 100644
index 000000000..a499cd17e
--- /dev/null
+++ b/store/agents/ollama/LICENSE
@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Autonomous, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
diff --git a/store/agents/ollama/README.md b/store/agents/ollama/README.md
new file mode 100644
index 000000000..7d2924078
--- /dev/null
+++ b/store/agents/ollama/README.md
@@ -0,0 +1,154 @@
+# Local AI harnesses
+
+Talk to Codex to deploy and use local language models. The viewer beside the conversation
+shows which models are on disk, which are in memory, deployment progress, and measured speed.
+Three working harnesses are available: **Ollama** (this root package), **[MLX-LM](../mlx-lm/README.md)** and **[vLLM](../vllm/README.md)** using Metal on this Mac.
+They share the queue, agent runner, inventory layout and performance charts. Each has its
+own framework identity, native adapter and workspace; this is not an aggregate multi-runtime dashboard.
+
+The [revised product direction](../../../docs/research/local-ai-harnesses/architecture.md) prioritizes direct control of MLX-LM
+and llama.cpp, plus advanced vLLM/SGLang serving capabilities. vLLM Metal now has a working initial integration. Harness plus the agent is
+the management interface; further wrappers are added when they bring a distinct capability.
+New language-model harnesses should declare `"category": "Local AI"` in their manifest.
+
+## Try Ollama in Harness
+
+Install with `harness dsh install autonomous/ollama`. In OpenHarness,
+choose **Local AI → Ollama**, then create a workspace on your Mac.
+The package opens its dashboard in the viewer on the left; Codex is the operator.
+No Ollama CLI knowledge is required.
+
+Try saying:
+
+- “Run a lightweight model.”
+- “Find a small model for coding.”
+- “Run gemma3:1b.”
+- “Ask Qwen to explain this code.” (Codex interprets the request and calls the model.)
+- “Compare Qwen 3 0.6B and Gemma 3 1B on this Mac.”
+- “Unload Qwen but keep its files.”
+
+A standalone viewer is also served at `http://127.0.0.1:4310/?view=grid` while `./viewer.sh`
+is running. Omit the query string for the optional browser command panel. That panel
+supports the explicit phrases above via a bounded intent parser; general free-form
+model management is handled by Codex in Harness, not a hidden second language model.
+If Harness launches its own viewer, it assigns a port and records it in
+`.harness/endpoint.json`. Opening the same workspace in Harness reuses an already running workspace service
+through a loopback relay, so its queue and recorded history stay shared. If the original
+service stops, reopen the native viewer to establish a new service.
+
+## What it does
+
+- Starts the installed Ollama runtime using its native CLI, on loopback with cloud disabled.
+- Uses Ollama's local API for discovery, streamed downloads, loading/unloading, and chat.
+- Checks download size and available storage before pulling. Downloads larger than 80%
+ of physical RAM are rejected by this conservative pilot heuristic; this is not a full
+ model-fit estimator. It reserves 2 GiB of disk headroom when the cache volume is readable.
+- Verifies residency after load/unload. Files remain in Ollama's existing model cache.
+- Streams generated text; records token rates, first token and first visible output time.
+- Runs repeatable benchmarks with raw samples, model digest, runtime version and hardware.
+- Serializes jobs in one workspace service, supports cancellation and resumable download retries,
+ and preserves interrupted/failed jobs as such across restarts.
+- Exports benchmark data as JSON. No cloud hosting, paid API key or npm packages are needed.
+
+Chat enables reasoning for models that support it. GPT-OSS uses its supported `low`
+reasoning level. Chat has a 4,096-token context, an output budget of 512 tokens including
+reasoning, and a 15-minute idle keep-alive. Long reasoning may exhaust that budget before
+visible output; the viewer reports the output limit. The small catalog models are examples,
+not claims about current best model quality.
+
+## Native controls and agent runner
+
+The adapter uses `ollama serve` to start the service and native `/api/tags`, `/api/ps`,
+`/api/show`, `/api/pull`, `/api/generate`, and `/api/chat` endpoints for structured operations.
+The agent-facing runner exposes these shared actions so its operations appear in the viewer:
+
+```sh
+./toolchain/local-ai status
+./toolchain/local-ai action deploy qwen3:0.6b
+./toolchain/local-ai action chat qwen3:0.6b 'Explain how a bicycle works.'
+./toolchain/local-ai action benchmark qwen3:0.6b gemma3:1b
+./toolchain/local-ai action unload qwen3:0.6b
+./toolchain/local-ai 'run a lightweight model'
+```
+
+This CLI is for agents and development. The user interface is the conversation and viewer.
+The adapter never turns model-generated text into a shell command. Direct Ollama commands
+remain available for advanced diagnosis; discovery observes their effects on the next poll.
+
+## Measurements
+
+A benchmark uses one discarded 32-token warm-up, then three runs of the same prompt at
+2,048-token context, at most 128 generated tokens, seed 42 and temperature 0. Thinking is
+disabled when supported; the actual setting is recorded per sample. Reported speed is
+`eval_count / eval_duration`, with Ollama's nanoseconds converted to seconds. Time to first
+token is observed at the harness, including the first reasoning token when reasoning is on.
+Time to first visible output is stored separately. Repeated prompts may use prefix caching.
+
+These are **warm inference measurements**, not cold-start tests, quality scores, or isolated
+lab comparisons. Other applications and resident models may affect results. The two demo
+models were loaded together during the comparison. Ollama allocation is not total process
+RSS; Apple Silicon's CPU and GPU share unified memory.
+
+## Local data and lifecycle
+
+- `.harness/state.json`: up to 200 benchmarks, 200 generation metric records and 40 recent
+ jobs. Job records include generated output; chat prompts are not persisted by this harness.
+- Browser conversation context is kept in page memory (last six turns, bounded); reloading
+ resets it. The selected model uses session storage.
+- `.harness/ollama.log`: the native runtime log if this harness started Ollama.
+- `.harness/endpoint.json` and `verdict.json`: local viewer discovery and Harness readiness.
+- Existing model files stay in `~/.ollama/models`, or the runtime's configured model directory.
+
+Stopping the viewer stops its requests but leaves Ollama running. The runtime is shared with
+other local clients; this pilot does not kill unrelated services or delete model files. Views of the same workspace share its queue; separate workspaces and external
+Ollama clients still have independent requests. No remote-machine controls are
+implemented. Local model names and `/api/show` are checked to reject cloud models and aliases.
+The harness binds only to loopback, validates Host/Origin and requires a per-process control
+token on JSON mutations. Downloads require network access; generation uses the local service.
+Codex itself still uses its configured provider; the harness does not make Codex's reasoning local.
+
+## Development and checks
+
+Prerequisites: Node.js 22+ and Ollama installed. GUI launch can use Harness's managed Node.
+The setup/doctor checks dependencies without downloading a model.
+
+```sh
+./viewer.sh
+harness dsh check .
+harness dsh doctor autonomous/ollama
+# Shared-source development, from store/tools/local-ai:
+npm run build
+npm run check
+npm test
+npm run test:live # opt-in; unloads/reloads qwen3:0.6b and performs inference
+```
+
+To register another checkout: `harness dsh install /absolute/path/to/checkout --link`.
+The conformance tool's optional skills warning is expected: the pilot ships a single agent
+instruction file, rather than duplicating its operations guide as a separate skill.
+
+`test:live` records fresh evidence in the shared development folder’s ignored `evidence/` directory. Fixture tests are isolated
+from real model files; HTTP tests bind a temporary loopback port. The viewer exposes two
+feature-detected WebMCP tools, inventory and start action. No supported WebMCP execution
+context was available in this session, so that optional integration is not verified.
+
+## Research and next decision
+
+- [Framework research and original shortlist](../../../docs/research/local-ai-harnesses/recommendation.md)
+- [Machine-readable candidate list and current decision](../../../docs/research/local-ai-harnesses/shortlist.json)
+- [vLLM Metal findings and concurrency measurements](../../../docs/research/local-ai-harnesses/vllm/findings.md)
+- [What the Ollama pilot taught us](../../../docs/research/local-ai-harnesses/pilot-findings.md)
+- [Raw measured benchmark samples](../../../docs/research/local-ai-harnesses/pilot-benchmarks.json)
+- [Real deployment and chat check](../../../docs/research/local-ai-harnesses/pilot-verification.json)
+
+The package uses OpenHarness's existing `harness.json`/viewer contract, also used by the
+Grid package. The Ollama model adapter is framework-specific; the control plane and viewer logic are shared with MLX-LM. Grid's fleet collector
+was not reused because its engine and telemetry contract differs from Ollama's local API.
+
+## Credit and stewardship
+
+The native framework and official mark belong to [Ollama](https://github.com/ollama/ollama).
+OpenHarness contributors maintain this independent integration, its agent instructions
+and viewer. The harness is MIT-licensed; see [LICENSE](LICENSE). Upstream notices
+and license texts are recorded in [THIRD_PARTY.md](THIRD_PARTY.md) and `licenses/`.
+Model weights are downloaded separately under their respective model licenses.
diff --git a/store/agents/ollama/THIRD_PARTY.md b/store/agents/ollama/THIRD_PARTY.md
new file mode 100644
index 000000000..ea15a6ddc
--- /dev/null
+++ b/store/agents/ollama/THIRD_PARTY.md
@@ -0,0 +1,10 @@
+# Attribution
+
+This independently maintained OpenHarness integration is MIT-licensed; see LICENSE.
+It is not an official application of the upstream framework. Upstream names and
+marks identify the tools being operated and remain their owners’ marks.
+
+Official logo source and retrieval date are recorded in dist/assets/README.md.
+Upstream framework license texts are included in licenses/. The setup scripts use
+the upstream distributions; Python and runtime dependencies retain their own licenses.
+Model weights are not included and retain their individual model licenses.
diff --git a/store/agents/ollama/agent/AGENTS.md b/store/agents/ollama/agent/AGENTS.md
new file mode 100644
index 000000000..9067bb801
--- /dev/null
+++ b/store/agents/ollama/agent/AGENTS.md
@@ -0,0 +1,53 @@
+# Ollama model operator
+
+These instructions apply to an Ollama harness workspace containing `local-ai.json`.
+The conversation with you is the natural-language interface. The viewer beside it
+shows live model inventory, memory and real measurements. Do not start a second
+agent or add a second conversation pane to the embedded viewer.
+
+Use the workspace-aware `$LOCAL_AI` runner for normal actions so progress and results
+appear in the viewer. It wraps the real installed Ollama CLI and its loopback API.
+No vendor CLI knowledge is required from the user.
+
+```
+"$LOCAL_AI" status
+"$LOCAL_AI" action start
+"$LOCAL_AI" action deploy qwen3:0.6b
+"$LOCAL_AI" action download gemma3:1b
+"$LOCAL_AI" action chat qwen3:0.6b "Explain unified memory in three sentences."
+"$LOCAL_AI" action benchmark qwen3:0.6b gemma3:1b
+"$LOCAL_AI" action unload qwen3:0.6b
+"$LOCAL_AI" action unload_all
+"$LOCAL_AI" cancel JOB_ID
+```
+
+The runner also accepts a natural-language request as a single argument. Use typed
+actions for complex user requests you have already interpreted. The runner waits
+for completion and returns a nonzero status on failure. Status reports a model
+as ready only when Ollama reports it in memory. Deployment alone is not an inference
+test: follow the first deployment with a short real chat request before reporting
+that the model works. Inspect and resolve failures; never edit saved measurements
+or snapshots to make a failed action appear successful.
+
+Start with `status`. If the user requests an unspecified first model, `qwen3:0.6b`
+is the small pilot default (~523 MB download). Explain its limited capability;
+recommend larger models based on the user's workload and available memory after
+consulting current model sources. Loading uses a 4,096-token context and keeps a
+model ready for 15 minutes of inactivity. Unload releases memory and keeps files.
+Deleting model files, remote execution, model training, and cloud models are outside
+this pilot's actions. Do not reinterpret “stop” as deletion.
+
+Benchmarks use one discarded warm-up followed by three measured runs of the same
+prompt, 2,048-token context, at most 128 generated tokens, temperature 0 and seed 42.
+Report medians, model tag/digest and Ollama version; disclose warm prefix caching.
+This measures speed, not answer quality. Other running workloads can affect results.
+On Apple Silicon, runtime allocation is unified-memory allocation, not a separate
+physical GPU memory pool. Missing telemetry is unknown, never zero.
+
+Data lives in `.harness/` in this workspace; model weights remain in Ollama's cache.
+The viewer starts the workspace API and records its loopback port in
+`.harness/endpoint.json`. If the runner cannot connect, check that the viewer is open
+and inspect its reported error. Do not start a conflicting viewer or restart an
+unrelated Ollama service. Native `ollama` commands and the documented local API may
+be used for diagnosis, with argument arrays and without generated shell scripts.
+Record lasting decisions and comparisons in workspace Markdown notes.
diff --git a/store/agents/ollama/bin/harness.mjs b/store/agents/ollama/bin/harness.mjs
new file mode 100755
index 000000000..16b63570d
--- /dev/null
+++ b/store/agents/ollama/bin/harness.mjs
@@ -0,0 +1,54 @@
+#!/usr/bin/env node
+import { readFile } from 'node:fs/promises';
+import { resolve, join, dirname } from 'node:path';
+import { fileURLToPath } from 'node:url';
+const args = process.argv.slice(2);
+const runtimeName = process.env.LOCAL_AI_NAME || 'Ollama';
+const runtime = process.env.LOCAL_AI_RUNTIME || 'ollama';
+if (!args.length) { console.log(`Talk to the ${runtimeName} agent, or use:\n local-ai "run a lightweight model"\n local-ai status\n local-ai action deploy MODEL\n local-ai action benchmark MODEL [MODEL…]\n local-ai action chat MODEL "PROMPT"\n local-ai action unload MODEL\n local-ai action-json '{"type":"serve","model":"OWNER/REPO","maxSequences":4}' (vLLM)\n local-ai cancel JOB_ID`); process.exit(0); }
+const packageRoot = dirname(dirname(fileURLToPath(import.meta.url)));
+const workspace = resolve(process.env.HARNESS_WORKSPACE || packageRoot);
+let port = Number(process.env.HARNESS_PORT || ({ 'mlx-lm': 4311, vllm: 4312 }[runtime] || 4310));
+try { const endpoint = JSON.parse(await readFile(join(workspace, '.harness', 'endpoint.json'), 'utf8')); if (Number.isInteger(endpoint.port) && endpoint.port > 0 && endpoint.port <= 65535 && endpoint.workspace === workspace) port = endpoint.port; } catch {}
+const base = `http://127.0.0.1:${port}`;
+const get = async path => { const response = await fetch(`${base}${path}`, { signal: AbortSignal.timeout(15000) }); const value = await response.json(); if (!response.ok) throw new Error(value.error || `HTTP ${response.status}`); return value; };
+let currentJob;
+try {
+ const health = await get('/api/health');
+ if (health.application !== 'local-ai-harnesses' || health.workspace !== workspace || (health.runtime || 'ollama') !== runtime) throw new Error(`This endpoint belongs to another workspace or framework. Open the ${runtimeName} viewer for this workspace.`);
+ const state = await get('/api/state');
+ const post = async (path, body) => {
+ const response = await fetch(`${base}${path}`, { method: 'POST', headers: { 'Content-Type': 'application/json', 'X-Harness-Token': state.token }, body: JSON.stringify(body), signal: AbortSignal.timeout(15000) });
+ const value = await response.json(); if (!response.ok) throw new Error(value.error || `HTTP ${response.status}`); return value;
+ };
+ if (args[0] === 'status' && args.length === 1) {
+ const { token, ...publicState } = state; console.log(JSON.stringify(publicState, null, 2)); process.exit(0);
+ }
+ if (args[0] === 'cancel') { const { job } = await post('/api/cancel', { id: args[1] }); console.log(JSON.stringify({ id: job.id, status: job.status, message: job.message }, null, 2)); process.exit(0); }
+ let result;
+ if (args[0] === 'action-json') {
+ if (args.length !== 2) throw new Error('Pass one JSON action object.');
+ result = await post('/api/action', JSON.parse(args[1]));
+ } else if (args[0] === 'action') {
+ const type = args[1];
+ const body = type === 'benchmark' ? { type, models: args.slice(2) } : type === 'chat' ? { type, model: args[2], prompt: args.slice(3).join(' ') } : { type, model: args[2] };
+ result = await post('/api/action', body);
+ } else result = await post('/api/command', { text: args.join(' ') });
+ if (!result.job) { console.log(result.message || JSON.stringify(result, null, 2)); if (result.models) console.log(result.models.map(m => `${m.id}: ${m.description}`).join('\n')); process.exit(0); }
+ currentJob = result.job.id;
+ console.log(`Job ${currentJob}`);
+ process.once('SIGINT', async () => { try { await post('/api/cancel', { id: currentJob }); console.log('Cancellation requested.'); } finally { process.exit(130); } });
+ let last;
+ while (true) {
+ const { jobs } = await get('/api/jobs');
+ const job = jobs.find(j => j.id === currentJob);
+ if (!job) throw new Error('The job could not be found. Check the viewer.');
+ if (job.message !== last) { console.log(job.message); last = job.message; }
+ if (!['queued', 'running'].includes(job.status)) {
+ if (job.output) console.log(job.output);
+ if (job.result?.metrics) console.log(JSON.stringify(job.result.metrics, null, 2));
+ process.exit(job.status === 'succeeded' ? 0 : 1);
+ }
+ await new Promise(resolve => setTimeout(resolve, 1000));
+ }
+} catch (error) { console.error(error.cause?.code === 'ECONNREFUSED' ? `The viewer is not running. Open the ${runtimeName} harness viewer.` : `${error.message}${currentJob ? ` (job ${currentJob}; check its status before retrying)` : ''}`); process.exitCode = 1; }
diff --git a/store/agents/ollama/dist/app.js b/store/agents/ollama/dist/app.js
new file mode 100644
index 000000000..d1b2e6e45
--- /dev/null
+++ b/store/agents/ollama/dist/app.js
@@ -0,0 +1,209 @@
+const $ = selector => document.querySelector(selector);
+const escapeHTML = value => String(value ?? '').replace(/[&<>"']/g, c => ({ '&': '&amp;', '<': '&lt;', '>': '&gt;', '"': '&quot;', "'": '&#39;' }[c]));
+const gib = bytes => ((bytes || 0) / 1024 ** 3).toFixed(1);
+const number = (value, digits = 1) => Number.isFinite(value) ? value.toFixed(digits) : '—';
+const active = job => ['queued', 'running'].includes(job.status);
+let state;
+let selectedModel = sessionStorage.getItem('harness-model') || '';
+let pending = false;
+let modelSignature = '';
+let benchmarkSignature = '';
+let initialized = false;
+let lastReceived = Date.now();
+const jobElements = new Map();
+const jobSignatures = new Map();
+const chatHistory = new Map();
+const chatRequests = new Map();
+const completed = new Set();
+const embedded = new URLSearchParams(location.search).get('view') === 'grid';
+document.body.classList.toggle('embedded-viewer', embedded);
+
+function toast(text) { const el = $('#toast'); el.textContent = text; el.hidden = false; clearTimeout(toast.timer); toast.timer = setTimeout(() => { el.hidden = true; }, 4500); }
+function scrollConversation() { const el = $('#conversation'); el.scrollTop = el.scrollHeight; }
+function appendMessage(text, user = false) {
+ const el = document.createElement('div'); el.className = user ? 'user-message' : 'assistant-message';
+ if (user) el.textContent = text;
+ else el.innerHTML = `<span class="message-label">OLLAMA</span><p class="message-text">${escapeHTML(text)}</p>`;
+ $('#conversation').append(el); scrollConversation(); return el;
+}
+async function api(path, body) {
+ if (!state?.token) await refresh();
+ const response = await fetch(path, { method: 'POST', headers: { 'Content-Type': 'application/json', 'X-Harness-Token': state?.token || '' }, body: JSON.stringify(body) });
+ const result = await response.json();
+ if (!response.ok) { if (response.status === 403) void refresh(); throw new Error(result.error || 'The request failed.'); }
+ return result;
+}
+async function refresh() {
+ try {
+ const response = await fetch('/api/state'); if (!response.ok) throw new Error('Could not reach the Ollama viewer');
+ applyState(await response.json());
+ } catch (error) { $('#connection-status').textContent = 'Ollama viewer disconnected'; $('#operator-status').textContent = 'Reconnect to send a request'; return false; }
+ return true;
+}
+function applyState(next) {
+ lastReceived = Date.now(); state = { ...state, ...next };
+ if (selectedModel && !state.runtime.models.some(m => m.id === selectedModel)) selectedModel = '';
+ if (!selectedModel) selectedModel = state.runtime.models.find(m => m.running)?.id || '';
+ render();
+ if (!initialized) {
+ [...state.jobs].slice(0, 4).reverse().forEach(job => showJob(job, true)); initialized = true;
+ }
+ for (const job of state.jobs) if (jobElements.has(job.id) || active(job)) showJob(job);
+ renderActiveJob();
+ renderActivity();
+}
+function selectModel(model) {
+ selectedModel = model; sessionStorage.setItem('harness-model', model);
+ $('#selected-model').value = model;
+ $('#operator-status').textContent = model || 'Ollama harness';
+ document.querySelectorAll('[data-model]').forEach(el => el.classList.toggle('selected', el.dataset.model === model));
+}
+function render() {
+ const { runtime, system, benchmarks, runs } = state;
+ const models = runtime.models || [];
+ const loaded = models.filter(m => m.running);
+ const lastJob = state.jobs[0];
+ const notice = $('#runtime-notice');
+ notice.hidden = lastJob?.status !== 'failed';
+ if (!notice.hidden) {
+ const affected = lastJob.activeModel || lastJob.model || lastJob.models?.join(', ') || 'Ollama';
+ const explanation = /tensor.*size overflow/i.test(lastJob.message)
+ ? "Ollama could not read this model’s weights. Other models can still run."
+ : 'The last operation failed. Check the details before retrying.';
+ notice.innerHTML = `<strong>${escapeHTML(affected)} · ${escapeHTML(lastJob.type)} failed</strong><p>${escapeHTML(explanation)}</p><details><summary>Error details</summary><pre>${escapeHTML(lastJob.message)}</pre></details>`;
+ }
+ $('#connection-status').textContent = runtime.online ? 'Ollama connected' : 'Ollama offline';
+ $('#model-count').textContent = runtime.online ? models.length : '—';
+ $('#models-total').textContent = models.length;
+ $('#running-count').textContent = runtime.online ? loaded.length : '—';
+ $('#model-memory').innerHTML = `${runtime.online ? gib(loaded.reduce((sum, m) => sum + (m.resident?.size || 0), 0)) : '—'} <small>GiB</small>`;
+ const recent = runs[0];
+ $('#last-speed').innerHTML = `${number(recent?.tokensPerSecond)} <small>tok/s</small>`;
+ $('#last-speed').title = recent ? `${recent.model} · ${new Date(recent.at).toLocaleString()}` : 'A chat response will appear here';
+ $('#map-status').textContent = runtime.online ? 'LIVE CONNECTION' : 'READY TO START';
+ $('#map-status').classList.toggle('online', runtime.online);
+ $('#machine-info').textContent = `${system.chip} · ${Math.round(system.totalMemory / 1024 ** 3)} GiB`;
+ const signature = JSON.stringify({ online: runtime.online, version: runtime.version, models, selectedModel });
+ if (signature !== modelSignature) {
+ modelSignature = signature;
+ $('#runtime-map').innerHTML = `<div class="runtime-node ${runtime.online ? '' : 'offline'}"><img class="node-icon" src="/assets/ollama-icon.svg" width="42" height="42" alt=""><strong>Ollama</strong><small>${runtime.online ? `v${escapeHTML(runtime.version)}` : 'Not running'}</small></div><div class="model-nodes">${models.length ? models.map(m => `<button class="model-node ${m.running ? 'active' : ''} ${selectedModel === m.id ? 'selected' : ''}" data-model="${escapeHTML(m.id)}"><strong>${escapeHTML(m.name)}</strong><span>${m.running ? 'In memory' : 'On disk'} · ${gib(m.size)} GiB</span></button>`).join('') : `<div class="map-empty empty-node"><strong>Your first Ollama model starts here.</strong>Ask the operator to run your first model.<br>Its connection will appear here.</div>`}</div>`;
+ $('#models-list').innerHTML = models.length ? models.map(m => `<button class="model-row ${selectedModel === m.id ? 'selected' : ''}" data-model="${escapeHTML(m.id)}"><span class="model-title"><strong>${escapeHTML(m.name)}</strong><small>${escapeHTML([m.family, m.parameters, m.quantization].filter(Boolean).join(' · '))}</small></span><span class="model-status ${m.running ? 'loaded' : ''}">${m.running ? 'In memory' : 'On disk'}</span><span class="model-numeric">${gib(m.size)} GiB</span><span class="model-numeric">Ollama</span></button>`).join('') : `<div class="empty-small"><p>${runtime.online ? 'No models downloaded yet.' : 'Start Ollama to discover your models.'}</p><span>Say “run a lightweight model” to get started.</span></div>`;
+ $('#selected-model').innerHTML = `<option value="">Auto · running model</option>${models.map(m => `<option value="${escapeHTML(m.id)}">${escapeHTML(m.name)}</option>`).join('')}`;
+ $('#selected-model').value = selectedModel;
+ $('#operator-status').textContent = selectedModel || 'Ollama harness';
+ }
+ const benchSignature = JSON.stringify(benchmarks.map(b => b.id));
+ if (benchSignature !== benchmarkSignature) { benchmarkSignature = benchSignature; renderBenchmarks(); }
+ $('#export-benchmarks').hidden = benchmarks.length === 0;
+}
+function renderBenchmarks() {
+ const latest = [...new Map([...state.benchmarks].reverse().map(b => [b.model, b])).values()].sort((a, b) => b.medianTokensPerSecond - a.medianTokensPerSecond);
+ if (!latest.length) return;
+ const maxSpeed = Math.max(...latest.map(b => b.medianTokensPerSecond), 1);
+ const maxLatency = Math.max(...latest.map(b => b.medianFirstTokenMs), 1);
+ const bars = (key, max, unit) => latest.map(b => `<div class="chart-row"><span>${escapeHTML(b.model)}</span><div class="bar-track"><div class="bar-fill ${key === 'medianFirstTokenMs' ? 'latency' : ''}" style="width:${Math.max(0, b[key] / max * 100)}%"></div></div><span class="bar-value">${number(b[key], key === 'medianFirstTokenMs' ? 0 : 1)} <small>${unit}</small></span></div>`).join('');
+ $('#performance-content').innerHTML = `<div class="chart-heading"><h3>Generation speed</h3><span>tokens/s · higher is faster</span></div><div class="chart-bars" role="group" aria-label="Measured median generation speed">${bars('medianTokensPerSecond', maxSpeed, '')}</div><div class="chart-heading second-chart"><h3>Time to first token</h3><span>milliseconds · lower is faster</span></div><div class="chart-bars" role="group" aria-label="Measured median time to first token">${bars('medianFirstTokenMs', maxLatency, '')}</div><p class="benchmark-meta">Latest benchmark per model · median of 3 warm runs<br>Same prompt · 2,048-token context · up to 128 generated tokens · seed 42<br>Thinking off when supported; actual mode is recorded per run.<br>Repeated prompts can use the prefix cache. Speed measures inference, not answer quality.</p><details class="benchmark-details"><summary>Samples and measurement details</summary>${latest.map(b => `<div class="sample-record"><strong>${escapeHTML(b.model)}</strong><span>${new Date(b.at).toLocaleString()} · Ollama ${escapeHTML(b.runtime.version)}</span><span>Speed: ${b.samples.map(s => number(s.tokensPerSecond)).join(' / ')} tokens/s</span><span>First token: ${b.samples.map(s => number(s.firstTokenMs, 0)).join(' / ')} ms</span><span>Tokens generated: ${b.samples.map(s => s.tokens).join(' / ')} · thinking: ${escapeHTML(String(b.samples[0].thinking ?? 'not reported'))} · allocation: ${gib(b.residentBytes)} GiB</span></div>`).join('')}</details>`;
+}
+function showJob(job, historical = false) {
+ let el = jobElements.get(job.id);
+ if (!el) {
+ el = document.createElement('div'); el.className = 'assistant-message job-message';
+ jobElements.set(job.id, el); $('#conversation').append(el);
+ }
+ const signature = JSON.stringify(job);
+ if (jobSignatures.get(job.id) === signature) return;
+ jobSignatures.set(job.id, signature);
+ const failed = ['failed', 'cancelled', 'interrupted'].includes(job.status);
+ const metrics = job.result?.metrics;
+ const label = job.type === 'chat' ? job.model : 'OLLAMA';
+ el.innerHTML = `<span class="message-label">${escapeHTML(label)} <span class="job-state ${failed ? 'error-text' : ''}">${escapeHTML(job.status)}</span></span><p class="message-text ${failed ? 'error-text' : ''}">${escapeHTML(job.message)}</p>${job.output ? `<div class="stream-output message-text">${escapeHTML(job.output)}</div>` : ''}${metrics ? `<p class="run-metrics">${number(metrics.tokensPerSecond)} tok/s · ${number(metrics.firstTokenMs, 0)} ms first token · ${metrics.tokens ?? '—'} tokens${metrics.thinking ? ` · ${number(metrics.firstOutputMs, 0)} ms to visible output · reasoning on` : ''}${metrics.peakMemoryBytes ? ` · ${gib(metrics.peakMemoryBytes)} GiB peak allocation` : ''}${metrics.doneReason === 'length' ? ' · output limit reached' : ''}</p>` : ''}${job.persistenceError ? '<p class="error-text">Could not save this result to disk.</p>' : ''}${job.status === 'succeeded' && job.type === 'deploy' ? `<div class="actions"><button data-prompt="Benchmark ${escapeHTML(job.model)}">Benchmark it</button><button data-chat="${escapeHTML(job.model)}">Ask a question</button></div>` : ''}${job.status === 'failed' ? '<p class="secondary">You can retry the request after addressing the error.</p>' : ''}`;
+ if (job.status === 'succeeded' && !completed.has(job.id)) {
+ completed.add(job.id);
+ if (!historical && ['deploy', 'chat', 'benchmark'].includes(job.type) && job.result?.model) selectModel(job.result.model);
+ const request = chatRequests.get(job.id);
+ if (request) {
+ const history = chatHistory.get(job.model) || [];
+ history.push({ role: 'user', content: request }, { role: 'assistant', content: job.output });
+ chatHistory.set(job.model, history.slice(-6).map(m => ({ ...m, content: m.content.slice(0, 6000) }))); chatRequests.delete(job.id);
+ }
+ }
+ if (!historical) scrollConversation();
+}
+function renderActiveJob() {
+ const running = state.jobs.find(j => j.status === 'running') || state.jobs.find(j => j.status === 'queued');
+ const panel = $('#job-panel'); panel.hidden = !running;
+ if (!running) return;
+ const queued = state.jobs.filter(j => j.status === 'queued').length;
+ panel.innerHTML = `<div class="job-top"><strong>${escapeHTML(running.type === 'chat' ? 'Generating' : running.type.replaceAll('_', ' '))}</strong><button data-cancel="${escapeHTML(running.id)}">Cancel</button></div><div class="progress-track" role="progressbar" aria-label="Job progress" aria-valuemin="0" aria-valuemax="100" aria-valuenow="${Math.round(running.progress)}"><div class="progress-fill" style="width:${running.progress}%"></div></div><p>${escapeHTML(running.message)}${queued ? ` · ${queued} queued` : ''}</p>`;
+}
+function receiveJob(job) {
+ if (!state) return;
+ state.jobs = [job, ...state.jobs.filter(j => j.id !== job.id)];
+ showJob(job); renderActiveJob(); renderActivity();
+}
+function renderActivity() {
+ const jobs = state.jobs.slice(0, 6);
+ $('#activity-content').innerHTML = jobs.length ? jobs.map(job => `<div class="activity-row"><span class="activity-state ${job.status === 'failed' ? 'error-text' : ''}">${escapeHTML(job.status)}</span><div><strong>${escapeHTML(job.model || job.models.join(', ') || 'Ollama')}</strong><p>${escapeHTML(job.message)}</p></div>${active(job) ? `<button data-cancel="${escapeHTML(job.id)}">Cancel</button>` : '<span class="activity-time">' + new Date(job.completedAt || job.createdAt).toLocaleTimeString([], { hour: '2-digit', minute: '2-digit' }) + '</span>'}</div>`).join('') : '<p class="secondary">Ask the agent to run a model. Its progress will appear here.</p>';
+}
+async function sendCommand(text) {
+ if (pending || !text.trim()) return;
+ pending = true; $('#send-button').disabled = true; $('#suggestions').hidden = true;
+ appendMessage(text, true); $('#command-input').value = '';
+ try {
+ const result = await api('/api/command', { text, selectedModel: selectedModel || undefined, history: chatHistory.get(selectedModel) || [] });
+ if (result.job) {
+ if (result.job.type === 'chat') { const prompt = text.match(/^(?:ask|tell|chat with)\s+.+?\s*:\s*([\s\S]+)$/i)?.[1] || text; chatRequests.set(result.job.id, prompt); }
+ receiveJob(result.job);
+ } else {
+ const el = appendMessage(result.message || 'Choose a model to get started.');
+ if (result.models) el.insertAdjacentHTML('beforeend', result.models.map(m => `<div class="catalog-card"><strong>${escapeHTML(m.name)}</strong><p>${escapeHTML(m.description)}</p><p class="secondary">About ${number(m.downloadGB, 2)} GB to download</p><button data-action="deploy" data-target="${escapeHTML(m.id)}">Run this model</button></div>`).join(''));
+ }
+ } catch (error) { appendMessage(error.message).classList.add('error-text'); }
+ finally { pending = false; $('#send-button').disabled = false; scrollConversation(); }
+}
+async function runAction(type, model) {
+ $('#model-dialog').close();
+ const action = type === 'benchmark' ? { type, models: [model] } : { type, model };
+ appendMessage(`${type === 'deploy' ? 'Run' : type[0].toUpperCase() + type.slice(1)} ${model}`, true);
+ try { const result = await api('/api/action', action); receiveJob(result.job); return { jobId: result.job.id, status: result.job.status }; }
+ catch (error) { appendMessage(error.message).classList.add('error-text'); throw error; }
+}
+function inspectModel(id) {
+ const model = state.runtime.models.find(m => m.id === id); if (!model) return;
+ selectModel(id);
+ const resident = model.resident;
+ $('#model-detail').innerHTML = `<p class="eyebrow">OLLAMA / MODEL DETAILS</p><h2 id="detail-title">${escapeHTML(model.name)}</h2><p class="secondary">${model.running ? 'Loaded and ready for inference' : 'Downloaded and available to load'}</p><div class="detail-grid"><div><span>Model parameters</span><strong>${escapeHTML(model.parameters || 'Unknown')}</strong></div><div><span>Quantization</span><strong>${escapeHTML(model.quantization || 'Unknown')}</strong></div><div><span>Size on disk</span><strong>${gib(model.size)} GiB</strong></div><div><span>Runtime allocation</span><strong>${resident ? `${gib(resident.size)} GiB` : 'Not loaded'}</strong></div><div><span>Context window</span><strong>${resident?.context_length?.toLocaleString() || 'Set when loaded'}</strong></div><div><span>Idle expiry</span><strong>${resident?.expires_at ? new Date(resident.expires_at).toLocaleTimeString() : '—'}</strong></div></div><p class="secondary">Ollama reports runtime allocation. On Apple Silicon, CPU and GPU share unified memory; this is not a separate physical VRAM pool.</p><div class="actions"><button class="primary" data-action="deploy" data-target="${escapeHTML(id)}">${model.running ? 'Keep ready for 15m' : 'Load model'}</button><button data-action="benchmark" data-target="${escapeHTML(id)}">Benchmark</button><button data-chat="${escapeHTML(id)}">Chat</button>${model.running ? `<button data-action="unload" data-target="${escapeHTML(id)}">Unload</button>` : ''}</div>`;
+ $('#model-dialog').showModal();
+}
+
+$('#refresh').addEventListener('click', async () => { if (await refresh()) toast('Model status refreshed'); });
+$('#selected-model').addEventListener('change', event => selectModel(event.target.value));
+$('#command-form').addEventListener('submit', event => { event.preventDefault(); void sendCommand($('#command-input').value); });
+$('#command-input').addEventListener('keydown', event => { if (event.key === 'Enter' && !event.shiftKey && !event.isComposing) { event.preventDefault(); $('#command-form').requestSubmit(); } });
+$('.dialog-close').addEventListener('click', () => $('#model-dialog').close());
+document.addEventListener('click', event => {
+ const button = event.target.closest('button'); if (!button) return;
+ if (button.dataset.prompt) void sendCommand(button.dataset.prompt);
+ if (button.dataset.model) inspectModel(button.dataset.model);
+ if (button.dataset.action) void runAction(button.dataset.action, button.dataset.target).catch(() => {});
+ if (button.dataset.chat) { selectModel(button.dataset.chat); $('#model-dialog').close(); $('#command-input').value = 'Ask it: '; $('#command-input').focus(); }
+ if (button.dataset.cancel) void api('/api/cancel', { id: button.dataset.cancel }).then(result => receiveJob(result.job)).catch(error => toast(error.message));
+});
+
+await refresh();
+const events = new EventSource('/api/events');
+events.addEventListener('state', event => { try { applyState(JSON.parse(event.data)); } catch { void refresh(); } });
+events.addEventListener('job', event => { lastReceived = Date.now(); receiveJob(JSON.parse(event.data)); });
+events.addEventListener('error', () => { $('#connection-status').textContent = 'Reconnecting…'; });
+setInterval(() => { if (Date.now() - lastReceived > 12000) void refresh(); }, 10000);
+
+// The same visible UI actions are available to supported agent browsers.
+const lifecycle = new AbortController();
+if (document.modelContext?.registerTool) {
+ const register = tool => document.modelContext.registerTool(tool, { signal: lifecycle.signal });
+ try {
+ await register({ name: 'harness_inventory', title: 'Inspect local models', description: 'Read Ollama models, residency, recent benchmarks and job status from this viewer.', inputSchema: { type: 'object', properties: {}, additionalProperties: false }, annotations: { readOnlyHint: true, untrustedContentHint: true }, async execute() { if (!await refresh()) throw new Error('The workspace service is disconnected.'); return { runtime: state.runtime, benchmarks: state.benchmarks, jobs: state.jobs.map(({ id, type, model, status, message }) => ({ id, type, model, status, message })) }; } });
+ await register({ name: 'harness_start_action', title: 'Control a local model', description: 'Start one local Ollama action. Deploy downloads a missing model. Benchmark runs three warm measurements. Returns a job ID; inspect inventory to see completion.', inputSchema: { type: 'object', properties: { type: { type: 'string', enum: ['deploy', 'download', 'unload', 'benchmark'] }, model: { type: 'string', description: 'Exact local Ollama model name and tag.' } }, required: ['type', 'model'], additionalProperties: false }, annotations: { readOnlyHint: false, untrustedContentHint: true }, async execute(input) { if (!input || !['deploy', 'download', 'unload', 'benchmark'].includes(input.type) || typeof input.model !== 'string') throw new Error('Provide an allowed action and model.'); return runAction(input.type, input.model); } });
+ } catch (error) { console.info('WebMCP tools unavailable:', error.message); }
+}
+window.addEventListener('pagehide', () => { lifecycle.abort(); events.close(); });
diff --git a/store/agents/ollama/dist/assets/README.md b/store/agents/ollama/dist/assets/README.md
new file mode 100644
index 000000000..5ca42176a
--- /dev/null
+++ b/store/agents/ollama/dist/assets/README.md
@@ -0,0 +1,3 @@
+Ollama logo source: https://github.com/ollama/ollama/blob/main/docs/ollama-logo.svg
+Retrieved 2026-09-19. The official path is preserved in ollama.svg.
+ollama-icon.svg packages that mark on a light rounded tile for dark and light app chrome.
diff --git a/store/agents/ollama/dist/assets/ollama-icon.png b/store/agents/ollama/dist/assets/ollama-icon.png
new file mode 100644
index 000000000..130cb9ba4
Binary files /dev/null and b/store/agents/ollama/dist/assets/ollama-icon.png differ
diff --git a/store/agents/ollama/dist/assets/ollama-icon.svg b/store/agents/ollama/dist/assets/ollama-icon.svg
new file mode 100644
index 000000000..24a21f56a
--- /dev/null
+++ b/store/agents/ollama/dist/assets/ollama-icon.svg
@@ -0,0 +1,3 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 36 36"><rect width="36" height="36" rx="9" fill="#f4f6ef"/><g transform="translate(9.5 5.5)">
+<path fill-rule="evenodd" clip-rule="evenodd" d="M4.40517 0.102088C4.62117 0.198678 4.81617 0.357766 4.99317 0.56799C5.28817 0.915712 5.53718 1.41342 5.72718 2.00318C5.91818 2.59635 6.04218 3.25316 6.08918 3.91224C6.71878 3.5075 7.41754 3.26103 8.13818 3.18953L8.18918 3.18498C9.05919 3.10544 9.91919 3.28384 10.6692 3.72361C10.7702 3.78384 10.8692 3.84861 10.9662 3.91679C11.0162 3.27021 11.1382 2.62817 11.3262 2.04863C11.5162 1.45773 11.7652 0.961166 12.0592 0.612308C12.2235 0.410338 12.4245 0.251368 12.6482 0.146406C12.9052 0.032771 13.1782 0.0123167 13.4442 0.098679C13.8452 0.228223 14.1892 0.516855 14.4602 0.936167C14.7082 1.3191 14.8942 1.81 15.0212 2.39863C15.2512 3.45998 15.2912 4.85655 15.1362 6.54061L15.1892 6.58607L15.2152 6.60766C15.9722 7.26219 16.4992 8.19513 16.7782 9.27807C17.2133 10.9678 16.9943 12.8632 16.2442 13.9235L16.2262 13.9473L16.2282 13.9507C16.6453 14.8166 16.8983 15.7314 16.9523 16.678L16.9543 16.7121C17.0183 17.9223 16.7542 19.1404 16.1402 20.337L16.1332 20.3484L16.1432 20.3756C16.6152 21.6904 16.7632 23.0142 16.5812 24.3369L16.5752 24.3813C16.547 24.5744 16.4525 24.7472 16.3125 24.8612C16.1725 24.9753 15.9983 25.0219 15.8282 24.9903C15.744 24.9753 15.6632 24.9417 15.5904 24.8912C15.5177 24.8408 15.4544 24.7744 15.4042 24.696C15.3541 24.6178 15.318 24.529 15.2981 24.4347C15.2782 24.3406 15.2748 24.2428 15.2882 24.1472C15.4552 22.9733 15.2982 21.7961 14.8082 20.5984C14.7625 20.4871 14.7422 20.3645 14.7492 20.242C14.7562 20.1194 14.7902 20.0009 14.8482 19.8972L14.8522 19.8904C15.4562 18.8404 15.7062 17.8109 15.6522 16.7996C15.6062 15.9143 15.3272 15.045 14.8522 14.2166C14.7598 14.0556 14.7269 13.8597 14.7606 13.6713C14.7943 13.4829 14.8918 13.3171 15.0322 13.2098L15.0412 13.203C15.2842 13.0223 15.5082 12.561 15.6212 11.9303C15.7459 11.1846 15.7133 10.4159 15.5262 9.68716C15.3212 8.89171 14.9462 8.22809 14.4212 7.77468C13.8262 7.25878 13.0382 7.00992 12.0412 7.08151C11.9108 7.09115 11.7809 7.05613 11.6682 6.98097C11.5556 6.90581 11.4653 6.79399 11.4092 6.65993C11.0952 5.90426 10.6372 5.36336 10.0662 5.02814C9.51799 4.71723 8.90425 4.58657 8.29418 4.65087C7.04918 4.76337 5.95118 5.56108 5.62418 6.56675C5.57792 6.70829 5.4947 6.8304 5.38568 6.91672C5.27666 7.00301 5.14703 7.04942 5.01417 7.0497C3.94717 7.05197 3.12117 7.33606 2.51717 7.84855C1.99517 8.29172 1.63916 8.91103 1.45116 9.65307C1.28104 10.3515 1.25774 11.0857 1.38316 11.7962C1.49516 12.4303 1.71416 12.9553 1.96517 13.2382L1.97317 13.2462C2.18517 13.4814 2.23017 13.8485 2.08217 14.1382C1.72216 14.845 1.45316 15.8984 1.40916 16.9109C1.35916 18.0677 1.59516 19.0722 2.12817 19.7927L2.14417 19.8143C2.22461 19.9208 2.27633 20.0514 2.29319 20.1905C2.31003 20.3295 2.29127 20.4711 2.23917 20.5984C1.66316 22.0029 1.48616 23.1574 1.67716 24.0665C1.71148 24.2556 1.67954 24.4524 1.58812 24.6149C1.4967 24.7776 1.35302 24.8933 1.18766 24.9374C1.0223 24.9817 0.848322 24.9506 0.702741 24.8512C0.557141 24.7517 0.451463 24.5917 0.408163 24.4051C0.165162 23.2483 0.330162 21.9233 0.881162 20.4302L0.895162 20.3904L0.887162 20.3768C0.616341 19.9222 0.414243 19.4195 0.289162 18.8893L0.284162 18.8677C0.132362 18.2062 0.0726416 17.5218 0.107162 16.8393C0.151162 15.8052 0.385163 14.7462 0.729162 13.8962L0.741162 13.8666L0.739162 13.8644C0.446163 13.3894 0.229162 12.7814 0.109162 12.1087L0.104162 12.0814C-0.0611788 11.1431 -0.0293187 10.1737 0.197162 9.25194C0.459163 8.21218 0.974162 7.31901 1.73316 6.67356C1.79316 6.62243 1.85616 6.57129 1.91916 6.52357C1.76016 4.827 1.80016 3.42134 2.03117 2.35317C2.15817 1.76455 2.34517 1.27365 2.59317 0.890713C2.86317 0.472537 3.20717 0.183905 3.60817 0.0532252C3.87417 -0.0331371 4.14817 -0.0126829 4.40517 0.102088ZM8.52118 10.4315C9.45719 10.4315 10.3212 10.7871 10.9672 11.403C11.5972 12.0019 11.9722 12.8064 11.9722 13.6076C11.9722 14.6166 11.5662 15.403 10.8392 15.9052C10.2192 16.3314 9.38819 16.5382 8.43618 16.5382C7.42718 16.5382 6.56518 16.2439 5.94318 15.7041C5.32618 15.17 4.98017 14.42 4.98017 13.6076C4.98017 12.8042 5.37818 11.9973 6.03618 11.3962C6.70418 10.786 7.58618 10.4315 8.52118 10.4315ZM8.52118 11.4496C7.82742 11.4428 7.15204 11.7031 6.60518 12.1883C6.14418 12.6087 5.88318 13.1371 5.88318 13.6087C5.88318 14.095 6.09318 14.5507 6.49318 14.8973C6.94818 15.2916 7.61718 15.52 8.43618 15.52C9.23519 15.52 9.90919 15.353 10.3682 15.0359C10.8312 14.7178 11.0682 14.2564 11.0682 13.6076C11.0682 13.1269 10.8222 12.5962 10.3852 12.1803C9.90119 11.7201 9.24519 11.4496 8.52118 11.4496ZM9.18319 12.8246L9.18719 12.8292C9.30719 13.0007 9.28219 13.2496 9.13119 13.386L8.83919 13.6473V14.1541C8.83865 14.267 8.79877 14.375 8.72829 14.4544C8.6578 14.5339 8.56246 14.5783 8.46318 14.578C8.3639 14.5783 8.26856 14.5339 8.19808 14.4544C8.12758 14.375 8.0877 14.267 8.08718 14.1541V13.6314L7.81618 13.3837C7.78042 13.3511 7.7507 13.3109 7.72872 13.2652C7.70674 13.2195 7.69294 13.1694 7.6881 13.1176C7.68326 13.0658 7.6875 13.0135 7.70056 12.9636C7.71362 12.9137 7.73524 12.8672 7.76418 12.8269C7.8232 12.7452 7.9082 12.6934 8.0007 12.6825C8.09318 12.6717 8.18572 12.7027 8.25818 12.7689L8.47318 12.9644L8.69318 12.7667C8.76538 12.7018 8.85702 12.6716 8.94854 12.6825C9.04009 12.6933 9.12427 12.7443 9.18319 12.8246ZM4.14317 10.644C4.62117 10.644 5.01017 11.0871 5.01017 11.6337C5.01043 11.8957 4.91917 12.1471 4.75641 12.3327C4.59365 12.5183 4.37273 12.6229 4.14217 12.6235C3.91195 12.6226 3.69143 12.518 3.52893 12.3327C3.36641 12.1474 3.27517 11.8965 3.27517 11.6349C3.27463 11.3729 3.36565 11.1213 3.52821 10.9355C3.69079 10.7497 3.91261 10.6449 4.14317 10.644ZM12.8492 10.644C13.3292 10.644 13.7172 11.0871 13.7172 11.6337C13.7175 11.8957 13.6262 12.1471 13.4634 12.3327C13.3007 12.5183 13.0798 12.6229 12.8492 12.6235C12.619 12.6226 12.3985 12.518 12.236 12.3327C12.0734 12.1474 11.9822 11.8965 11.9822 11.6349C11.9817 11.3729 12.0727 11.1213 12.2352 10.9355C12.3978 10.7497 12.6186 10.6449 12.8492 10.644ZM3.94017 1.47705L3.93717 1.47932C3.82131 1.53657 3.72239 1.63046 3.65217 1.74977L3.64717 1.75659C3.50917 1.97136 3.38917 2.28727 3.29917 2.70203C3.12917 3.48839 3.08317 4.55541 3.17517 5.86335C3.60517 5.7179 4.07417 5.62699 4.57917 5.59404L4.58917 5.5929L4.60817 5.55426C4.65417 5.46108 4.70317 5.37131 4.75617 5.28268C4.87917 4.40655 4.77817 3.35998 4.50317 2.50545C4.36917 2.09182 4.20617 1.76682 4.05017 1.5816C4.01797 1.5431 3.98207 1.5088 3.94317 1.47932L3.94017 1.47705ZM13.1142 1.52251L13.1122 1.52364C13.0733 1.55312 13.0374 1.58741 13.0052 1.62591C12.8492 1.81114 12.6852 2.13727 12.5522 2.5509C12.2622 3.45316 12.1652 4.56905 12.3222 5.47358L12.3802 5.58381L12.3882 5.59972H12.4182C12.9145 5.59988 13.4082 5.68101 13.8842 5.84062C13.9702 4.56337 13.9222 3.51907 13.7562 2.74749C13.6662 2.33272 13.5462 2.01682 13.4072 1.80205L13.4032 1.79523C13.3331 1.67548 13.2342 1.58121 13.1182 1.52364L13.1142 1.52251Z" fill="black"/>
+</g></svg>
diff --git a/store/agents/ollama/dist/assets/ollama.svg b/store/agents/ollama/dist/assets/ollama.svg
new file mode 100644
index 000000000..b215c89b9
--- /dev/null
+++ b/store/agents/ollama/dist/assets/ollama.svg
@@ -0,0 +1,3 @@
+<svg width="17" height="25" viewBox="0 0 17 25" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path fill-rule="evenodd" clip-rule="evenodd" d="M4.40517 0.102088C4.62117 0.198678 4.81617 0.357766 4.99317 0.56799C5.28817 0.915712 5.53718 1.41342 5.72718 2.00318C5.91818 2.59635 6.04218 3.25316 6.08918 3.91224C6.71878 3.5075 7.41754 3.26103 8.13818 3.18953L8.18918 3.18498C9.05919 3.10544 9.91919 3.28384 10.6692 3.72361C10.7702 3.78384 10.8692 3.84861 10.9662 3.91679C11.0162 3.27021 11.1382 2.62817 11.3262 2.04863C11.5162 1.45773 11.7652 0.961166 12.0592 0.612308C12.2235 0.410338 12.4245 0.251368 12.6482 0.146406C12.9052 0.032771 13.1782 0.0123167 13.4442 0.098679C13.8452 0.228223 14.1892 0.516855 14.4602 0.936167C14.7082 1.3191 14.8942 1.81 15.0212 2.39863C15.2512 3.45998 15.2912 4.85655 15.1362 6.54061L15.1892 6.58607L15.2152 6.60766C15.9722 7.26219 16.4992 8.19513 16.7782 9.27807C17.2133 10.9678 16.9943 12.8632 16.2442 13.9235L16.2262 13.9473L16.2282 13.9507C16.6453 14.8166 16.8983 15.7314 16.9523 16.678L16.9543 16.7121C17.0183 17.9223 16.7542 19.1404 16.1402 20.337L16.1332 20.3484L16.1432 20.3756C16.6152 21.6904 16.7632 23.0142 16.5812 24.3369L16.5752 24.3813C16.547 24.5744 16.4525 24.7472 16.3125 24.8612C16.1725 24.9753 15.9983 25.0219 15.8282 24.9903C15.744 24.9753 15.6632 24.9417 15.5904 24.8912C15.5177 24.8408 15.4544 24.7744 15.4042 24.696C15.3541 24.6178 15.318 24.529 15.2981 24.4347C15.2782 24.3406 15.2748 24.2428 15.2882 24.1472C15.4552 22.9733 15.2982 21.7961 14.8082 20.5984C14.7625 20.4871 14.7422 20.3645 14.7492 20.242C14.7562 20.1194 14.7902 20.0009 14.8482 19.8972L14.8522 19.8904C15.4562 18.8404 15.7062 17.8109 15.6522 16.7996C15.6062 15.9143 15.3272 15.045 14.8522 14.2166C14.7598 14.0556 14.7269 13.8597 14.7606 13.6713C14.7943 13.4829 14.8918 13.3171 15.0322 13.2098L15.0412 13.203C15.2842 13.0223 15.5082 12.561 15.6212 11.9303C15.7459 11.1846 15.7133 10.4159 15.5262 9.68716C15.3212 8.89171 14.9462 8.22809 14.4212 7.77468C13.8262 7.25878 13.0382 7.00992 12.0412 7.08151C11.9108 7.09115 11.7809 7.05613 11.6682 6.98097C11.5556 6.90581 11.4653 6.79399 11.4092 6.65993C11.0952 5.90426 10.6372 5.36336 10.0662 5.02814C9.51799 4.71723 8.90425 4.58657 8.29418 4.65087C7.04918 4.76337 5.95118 5.56108 5.62418 6.56675C5.57792 6.70829 5.4947 6.8304 5.38568 6.91672C5.27666 7.00301 5.14703 7.04942 5.01417 7.0497C3.94717 7.05197 3.12117 7.33606 2.51717 7.84855C1.99517 8.29172 1.63916 8.91103 1.45116 9.65307C1.28104 10.3515 1.25774 11.0857 1.38316 11.7962C1.49516 12.4303 1.71416 12.9553 1.96517 13.2382L1.97317 13.2462C2.18517 13.4814 2.23017 13.8485 2.08217 14.1382C1.72216 14.845 1.45316 15.8984 1.40916 16.9109C1.35916 18.0677 1.59516 19.0722 2.12817 19.7927L2.14417 19.8143C2.22461 19.9208 2.27633 20.0514 2.29319 20.1905C2.31003 20.3295 2.29127 20.4711 2.23917 20.5984C1.66316 22.0029 1.48616 23.1574 1.67716 24.0665C1.71148 24.2556 1.67954 24.4524 1.58812 24.6149C1.4967 24.7776 1.35302 24.8933 1.18766 24.9374C1.0223 24.9817 0.848322 24.9506 0.702741 24.8512C0.557141 24.7517 0.451463 24.5917 0.408163 24.4051C0.165162 23.2483 0.330162 21.9233 0.881162 20.4302L0.895162 20.3904L0.887162 20.3768C0.616341 19.9222 0.414243 19.4195 0.289162 18.8893L0.284162 18.8677C0.132362 18.2062 0.0726416 17.5218 0.107162 16.8393C0.151162 15.8052 0.385163 14.7462 0.729162 13.8962L0.741162 13.8666L0.739162 13.8644C0.446163 13.3894 0.229162 12.7814 0.109162 12.1087L0.104162 12.0814C-0.0611788 11.1431 -0.0293187 10.1737 0.197162 9.25194C0.459163 8.21218 0.974162 7.31901 1.73316 6.67356C1.79316 6.62243 1.85616 6.57129 1.91916 6.52357C1.76016 4.827 1.80016 3.42134 2.03117 2.35317C2.15817 1.76455 2.34517 1.27365 2.59317 0.890713C2.86317 0.472537 3.20717 0.183905 3.60817 0.0532252C3.87417 -0.0331371 4.14817 -0.0126829 4.40517 0.102088ZM8.52118 10.4315C9.45719 10.4315 10.3212 10.7871 10.9672 11.403C11.5972 12.0019 11.9722 12.8064 11.9722 13.6076C11.9722 14.6166 11.5662 15.403 10.8392 15.9052C10.2192 16.3314 9.38819 16.5382 8.43618 16.5382C7.42718 16.5382 6.56518 16.2439 5.94318 15.7041C5.32618 15.17 4.98017 14.42 4.98017 13.6076C4.98017 12.8042 5.37818 11.9973 6.03618 11.3962C6.70418 10.786 7.58618 10.4315 8.52118 10.4315ZM8.52118 11.4496C7.82742 11.4428 7.15204 11.7031 6.60518 12.1883C6.14418 12.6087 5.88318 13.1371 5.88318 13.6087C5.88318 14.095 6.09318 14.5507 6.49318 14.8973C6.94818 15.2916 7.61718 15.52 8.43618 15.52C9.23519 15.52 9.90919 15.353 10.3682 15.0359C10.8312 14.7178 11.0682 14.2564 11.0682 13.6076C11.0682 13.1269 10.8222 12.5962 10.3852 12.1803C9.90119 11.7201 9.24519 11.4496 8.52118 11.4496ZM9.18319 12.8246L9.18719 12.8292C9.30719 13.0007 9.28219 13.2496 9.13119 13.386L8.83919 13.6473V14.1541C8.83865 14.267 8.79877 14.375 8.72829 14.4544C8.6578 14.5339 8.56246 14.5783 8.46318 14.578C8.3639 14.5783 8.26856 14.5339 8.19808 14.4544C8.12758 14.375 8.0877 14.267 8.08718 14.1541V13.6314L7.81618 13.3837C7.78042 13.3511 7.7507 13.3109 7.72872 13.2652C7.70674 13.2195 7.69294 13.1694 7.6881 13.1176C7.68326 13.0658 7.6875 13.0135 7.70056 12.9636C7.71362 12.9137 7.73524 12.8672 7.76418 12.8269C7.8232 12.7452 7.9082 12.6934 8.0007 12.6825C8.09318 12.6717 8.18572 12.7027 8.25818 12.7689L8.47318 12.9644L8.69318 12.7667C8.76538 12.7018 8.85702 12.6716 8.94854 12.6825C9.04009 12.6933 9.12427 12.7443 9.18319 12.8246ZM4.14317 10.644C4.62117 10.644 5.01017 11.0871 5.01017 11.6337C5.01043 11.8957 4.91917 12.1471 4.75641 12.3327C4.59365 12.5183 4.37273 12.6229 4.14217 12.6235C3.91195 12.6226 3.69143 12.518 3.52893 12.3327C3.36641 12.1474 3.27517 11.8965 3.27517 11.6349C3.27463 11.3729 3.36565 11.1213 3.52821 10.9355C3.69079 10.7497 3.91261 10.6449 4.14317 10.644ZM12.8492 10.644C13.3292 10.644 13.7172 11.0871 13.7172 11.6337C13.7175 11.8957 13.6262 12.1471 13.4634 12.3327C13.3007 12.5183 13.0798 12.6229 12.8492 12.6235C12.619 12.6226 12.3985 12.518 12.236 12.3327C12.0734 12.1474 11.9822 11.8965 11.9822 11.6349C11.9817 11.3729 12.0727 11.1213 12.2352 10.9355C12.3978 10.7497 12.6186 10.6449 12.8492 10.644ZM3.94017 1.47705L3.93717 1.47932C3.82131 1.53657 3.72239 1.63046 3.65217 1.74977L3.64717 1.75659C3.50917 1.97136 3.38917 2.28727 3.29917 2.70203C3.12917 3.48839 3.08317 4.55541 3.17517 5.86335C3.60517 5.7179 4.07417 5.62699 4.57917 5.59404L4.58917 5.5929L4.60817 5.55426C4.65417 5.46108 4.70317 5.37131 4.75617 5.28268C4.87917 4.40655 4.77817 3.35998 4.50317 2.50545C4.36917 2.09182 4.20617 1.76682 4.05017 1.5816C4.01797 1.5431 3.98207 1.5088 3.94317 1.47932L3.94017 1.47705ZM13.1142 1.52251L13.1122 1.52364C13.0733 1.55312 13.0374 1.58741 13.0052 1.62591C12.8492 1.81114 12.6852 2.13727 12.5522 2.5509C12.2622 3.45316 12.1652 4.56905 12.3222 5.47358L12.3802 5.58381L12.3882 5.59972H12.4182C12.9145 5.59988 13.4082 5.68101 13.8842 5.84062C13.9702 4.56337 13.9222 3.51907 13.7562 2.74749C13.6662 2.33272 13.5462 2.01682 13.4072 1.80205L13.4032 1.79523C13.3331 1.67548 13.2342 1.58121 13.1182 1.52364L13.1142 1.52251Z" fill="black"/>
+</svg>
diff --git a/store/agents/ollama/dist/index.html b/store/agents/ollama/dist/index.html
new file mode 100644
index 000000000..4ec0310ac
--- /dev/null
+++ b/store/agents/ollama/dist/index.html
@@ -0,0 +1,59 @@
+<!doctype html>
+<html lang="en">
+<head>
+ <meta charset="utf-8">
+ <meta name="viewport" content="width=device-width, initial-scale=1">
+ <meta name="theme-color" content="#101516">
+ <meta name="description" content="Run, inspect, and benchmark your Ollama models on this Mac using natural language.">
+ <title>Ollama — Your local model space</title>
+ <link rel="icon" type="image/svg+xml" href="/assets/ollama-icon.svg">
+ <link rel="apple-touch-icon" href="/assets/ollama-icon.png">
+ <link rel="stylesheet" href="/styles.css">
+ <script type="module" src="/app.js"></script>
+</head>
+<body>
+ <header class="topbar">
+ <a class="brand" href="/?view=grid" aria-label="Ollama model space"><img class="brand-mark" src="/assets/ollama-icon.svg" width="40" height="40" alt=""><span>Ollama</span><span class="pilot">LOCAL AI</span></a>
+ <div class="top-status"><span class="local-symbol" aria-hidden="true">◈</span> On your machine <span class="separator"></span><span id="connection-status">Connecting</span></div>
+ </header>
+ <main class="workspace">
+ <section class="control-room" aria-label="Ollama control room">
+ <div class="page-heading"><div><p class="eyebrow">OLLAMA / ON THIS MAC</p><h1>Your Ollama models.</h1></div><button id="refresh" class="icon-button" aria-label="Refresh model status" title="Refresh model status">↻</button></div>
+ <div id="runtime-notice" class="runtime-notice" role="status" hidden></div>
+ <div class="stats-row" id="stats-row">
+ <div class="stat"><span>Models on disk</span><strong id="model-count">—</strong></div>
+ <div class="stat"><span>In memory</span><strong id="running-count">—</strong></div>
+ <div class="stat"><span>Last generation</span><strong id="last-speed">— <small>tok/s</small></strong></div>
+ <div class="stat"><span>Model allocation</span><strong id="model-memory">— <small>GiB</small></strong></div>
+ </div>
+ <section class="map-panel" aria-labelledby="map-heading">
+ <div class="panel-heading"><h2 id="map-heading">Ollama runtime</h2><span class="mono live-label" id="map-status">DISCOVERING</span></div>
+ <div id="runtime-map" class="runtime-map"><div class="map-empty">Discovering your Ollama models…</div></div>
+ <div class="map-footer"><span><i class="legend-dot loaded"></i> In memory <i class="legend-dot"></i> On disk</span><span id="machine-info">Apple Silicon</span></div>
+ </section>
+ <section class="models-panel" aria-labelledby="models-heading">
+ <div class="panel-heading"><h2 id="models-heading">Ollama models <span id="models-total" class="count-pill">0</span></h2><span class="secondary">Select a model to inspect</span></div>
+ <div id="models-list" class="models-list"></div>
+ </section>
+ <section class="performance-panel" aria-labelledby="performance-heading">
+ <div class="panel-heading"><h2 id="performance-heading">Performance</h2><a id="export-benchmarks" class="secondary" href="/api/export" download hidden>Export results ↗</a></div>
+ <div id="performance-content" class="performance-content"><div class="empty-small"><span class="empty-icon" aria-hidden="true">▥</span><p>Your first run sets the baseline.</p><span>Benchmark a model to compare speed and response time.</span></div></div>
+ </section>
+ <section class="activity-panel" aria-labelledby="activity-heading"><div class="panel-heading"><h2 id="activity-heading">Activity</h2><span class="secondary">Live from Ollama</span></div><div id="activity-content"></div></section>
+ </section>
+ <aside class="assistant-panel" aria-label="Natural language model controls">
+ <div class="assistant-header"><img class="assistant-avatar" src="/assets/ollama-icon.svg" width="36" height="36" alt=""><div><h2>Ollama operator</h2><span>Tell it what you want to run.</span></div><span class="private-badge">LOCAL</span></div>
+ <div class="conversation" id="conversation" role="log" aria-live="polite" aria-relevant="additions">
+ <div class="assistant-message"><span class="message-label">OLLAMA</span><p>Your Ollama models.<br>At your command.</p><p class="secondary">I can find a model, deploy it with Ollama, and measure how it performs here.</p></div>
+ <div class="suggestions" id="suggestions"><button data-prompt="Run a lightweight model">Run a lightweight model <span>↗</span></button><button data-prompt="Find a small model for coding">Find a model for coding <span>↗</span></button><button data-prompt="Show my models">Show my models <span>↗</span></button></div>
+ </div>
+ <div id="job-panel" class="job-panel" hidden></div>
+ <label class="model-picker" for="selected-model">Chat / follow-up model<select id="selected-model"><option value="">Auto · running model</option></select></label>
+ <form id="command-form" class="composer"><label class="sr-only" for="command-input">Ask the Ollama operator</label><textarea id="command-input" rows="3" placeholder="Run an Ollama model, compare speeds, free up memory…" maxlength="4000"></textarea><div class="composer-footer"><span id="operator-status">Ollama harness</span><button id="send-button" type="submit" aria-label="Send request">↑</button></div></form>
+ <div class="assistant-footer">Ollama inference stays on this Mac.<span>↵ Send · ⇧↵ New line</span></div>
+ </aside>
+ </main>
+ <dialog id="model-dialog" aria-labelledby="detail-title"><div id="model-detail"></div><button class="dialog-close" aria-label="Close model details">×</button></dialog>
+ <div id="toast" class="toast" role="status" hidden></div>
+</body>
+</html>
diff --git a/store/agents/ollama/dist/styles.css b/store/agents/ollama/dist/styles.css
new file mode 100644
index 000000000..cee872f50
--- /dev/null
+++ b/store/agents/ollama/dist/styles.css
@@ -0,0 +1,21 @@
+:root{color-scheme:dark;--bg:#101516;--surface:#171d1e;--surface-2:#1d2526;--line:#303939;--muted:#9ba9a7;--text:#edf3ee;--accent:#c5f87e;--cyan:#7ed8d1;--error:#ffafa4;--mono:ui-monospace,SFMono-Regular,Menlo,monospace;font-family:Inter,-apple-system,BlinkMacSystemFont,"Segoe UI",sans-serif;font-size:16px}
+*{box-sizing:border-box}body{margin:0;background:var(--bg);color:var(--text)}button,textarea,input,select{font:inherit}button{cursor:pointer}button:disabled{cursor:wait;opacity:.55}button,a,textarea,select{outline-offset:4px}button:focus-visible,a:focus-visible{outline:2px solid var(--accent)}button{color:inherit}a{color:inherit}button{border:1px solid var(--line);background:var(--surface);border-radius:8px}button:hover{border-color:#617166}h1,h2,p{margin:0}h2{font-size:1rem;font-weight:600;letter-spacing:-.02em}.mono{font-family:var(--mono)}.secondary{color:var(--muted);font-size:.875rem;line-height:1.6}.topbar{height:76px;border-bottom:1px solid var(--line);display:flex;justify-content:space-between;align-items:center;padding:0 32px}.brand{text-decoration:none;font-size:1rem;font-weight:750;letter-spacing:.15em;display:flex;align-items:center;gap:13px}.brand-mark{background:var(--accent);color:#111b0c;width:30px;height:32px;display:grid;place-items:center;clip-path:polygon(0 0,80% 0,100% 25%,100% 100%,20% 100%,0 75%);font-size:1.125rem;font-weight:900;letter-spacing:-.15em;padding-right:4px}.pilot{font-family:var(--mono);letter-spacing:.05em;font-size:.75rem;font-weight:400;color:var(--muted);padding-left:16px;margin-left:8px;border-left:1px solid var(--line)}.top-status{display:flex;gap:10px;align-items:center;font-size:.8125rem;color:var(--muted)}.local-symbol{color:var(--accent);font-size:1.1rem}.separator{height:16px;width:1px;background:var(--line);margin:0 6px}.workspace{display:grid;grid-template-columns:minmax(0,1fr) 360px;min-height:calc(100vh - 76px);max-width:1900px;margin:auto}.control-room{padding:34px 32px 40px;min-width:0}.page-heading{display:flex;justify-content:space-between;align-items:center;margin-bottom:30px}.eyebrow{font-family:var(--mono);font-size:.75rem;color:var(--accent);letter-spacing:.15em;margin-bottom:10px}h1{font-size:clamp(1.75rem,2.6vw,2.5rem);letter-spacing:-.06em;line-height:1.15;font-weight:550}.icon-button{border-radius:50%;width:38px;height:38px;font-size:1.4rem;background:transparent;color:var(--muted)}.stats-row{display:grid;grid-template-columns:repeat(4,1fr);margin-bottom:30px}.stat{padding:0 22px;border-right:1px solid var(--line);min-width:0}.stat:first-child{padding-left:0}.stat:last-child{border:0}.stat>span{font-size:.8125rem;color:var(--muted);display:block;margin-bottom:9px}.stat strong{font-family:var(--mono);font-size:1.75rem;font-weight:400;letter-spacing:-.07em}.stat small{font-family:inherit;color:var(--muted);font-size:.8rem;letter-spacing:-.02em}.panel-heading{display:flex;justify-content:space-between;align-items:center;gap:12px;margin-bottom:18px}.map-panel{border:1px solid var(--line);background:var(--surface);border-radius:12px;overflow:hidden}.map-panel .panel-heading{padding:20px 20px 0;margin-bottom:0}.live-label{font-size:.6875rem;letter-spacing:.09em;color:var(--muted)}.live-label.online{color:var(--accent)}.runtime-map{min-height:252px;position:relative;padding:25px 22px;background-image:radial-gradient(#3a454238 .8px,transparent .8px);background-size:16px 16px;display:flex;align-items:center;gap:36px}.runtime-node{border:1px solid #596a50;background:#202c21;border-radius:14px;padding:18px;min-width:150px;box-shadow:0 0 40px #c5f87e07;z-index:1;position:relative}.runtime-node::after{content:'';width:37px;height:1px;background:#617751;position:absolute;right:-37px;top:50%}.runtime-node.offline{border-color:var(--line);background:var(--surface-2)}.runtime-node .node-icon{width:32px;height:32px;display:grid;place-items:center;background:var(--accent);color:#233014;border-radius:9px;font-weight:700;margin-bottom:13px}.runtime-node strong{font-size:1rem;display:block;margin-bottom:6px}.runtime-node small{font-family:var(--mono);color:var(--muted);font-size:.75rem}.model-nodes{display:flex;gap:10px;flex-wrap:wrap;align-content:center;flex:1;border-left:1px solid #3c4d3c;padding-left:22px;min-height:120px}.model-node{position:relative;text-align:left;padding:13px 15px;border-radius:8px;min-width:160px;max-width:240px;background:#1b2222}.model-node::before{content:'';position:absolute;left:-23px;top:50%;height:1px;width:22px;background:#3c4d3c}.model-node.active{border-color:#7a995a;background:#263324}.model-node strong{display:block;font-size:.875rem;overflow-wrap:anywhere;line-height:1.4}.model-node span{display:block;color:var(--muted);font-size:.75rem;margin-top:6px}.model-node.active span{color:var(--accent)}.map-empty{margin:auto;color:var(--muted);font-size:.875rem;max-width:330px;line-height:1.7;text-align:center}.empty-node{text-align:left;margin:0;padding:12px 0}.empty-node strong{display:block;color:var(--text);font-size:1rem;margin-bottom:6px}.map-footer{border-top:1px solid var(--line);display:flex;align-items:center;justify-content:space-between;gap:12px;padding:12px 20px;font-family:var(--mono);font-size:.6875rem;color:var(--muted)}.map-footer>span:first-child{display:flex;align-items:center;gap:6px}.legend-dot{width:5px;height:5px;display:inline-block;border-radius:50%;background:#6a7a74;margin-left:12px}.legend-dot:first-child{margin-left:0}.legend-dot.loaded{background:var(--accent)}.models-panel,.performance-panel{margin-top:30px}.count-pill{display:inline-grid;place-items:center;font-family:var(--mono);font-size:.75rem;min-width:23px;height:21px;border-radius:5px;border:1px solid var(--line);color:var(--muted);margin-left:6px}.model-row{width:100%;display:grid;grid-template-columns:minmax(0,1fr) 85px 80px 80px;gap:14px;align-items:center;text-align:left;border-width:0 0 1px;border-radius:0;background:transparent;padding:17px 4px;font-size:.875rem}.model-row:hover{background:var(--surface)}.model-title strong{font-weight:500;overflow-wrap:anywhere;display:block}.model-title small{font-size:.75rem;color:var(--muted);display:block;margin-top:6px}.model-status{font-size:.75rem;color:var(--muted)}.model-status.loaded{color:var(--accent)}.model-numeric{font-family:var(--mono);font-size:.75rem;color:var(--muted)}.empty-small{padding:30px 18px;border:1px dashed var(--line);border-radius:10px;color:var(--muted);text-align:center;font-size:.875rem}.empty-small p{color:var(--text);margin:8px 0;font-size:1rem}.empty-small span:not(.empty-icon){font-size:.8125rem;line-height:1.6}.empty-icon{font-size:1.5rem;color:#667973}.assistant-panel{border-left:1px solid var(--line);background:#131a1b;display:flex;flex-direction:column;min-width:0;position:sticky;top:0;height:calc(100vh - 76px);max-height:1200px}.assistant-header{display:flex;gap:12px;align-items:center;padding:26px 22px;border-bottom:1px solid var(--line)}.assistant-avatar{height:35px;width:35px;display:grid;place-items:center;color:var(--accent);background:#293526;font-size:1.5rem;border-radius:10px}.assistant-header h2{font-size:.9375rem}.assistant-header div>span{font-size:.75rem;color:var(--muted);display:block;margin-top:4px}.private-badge{font-size:.625rem;letter-spacing:.08em;color:var(--muted);border:1px solid var(--line);border-radius:4px;padding:4px 5px;margin-left:auto;font-family:var(--mono)}.conversation{padding:26px 22px;overflow:auto;flex:1;min-height:200px;scroll-behavior:smooth}.message-label{font-family:var(--mono);font-size:.625rem;letter-spacing:.1em;color:var(--accent);display:block;margin-bottom:13px}.assistant-message{font-size:.9375rem;line-height:1.65;margin-bottom:25px}.assistant-message p+p{margin-top:12px}.assistant-message .secondary{font-size:.875rem}.suggestions{display:flex;flex-direction:column;gap:8px;margin-top:25px}.suggestions button{display:flex;justify-content:space-between;align-items:center;text-align:left;font-size:.8125rem;padding:13px 12px;background:transparent;color:#c4d0cb}.suggestions button span{color:var(--accent)}.composer{margin:0 18px;border:1px solid #485441;background:#202829;border-radius:11px;overflow:hidden}.composer:focus-within{border-color:var(--accent)}textarea{color:var(--text);display:block;resize:none;background:transparent;width:100%;border:0;padding:15px 15px 5px;line-height:1.6;outline:none;font-size:.875rem}textarea::placeholder{color:#91a29b}.composer-footer{padding:8px 11px 10px 15px;display:flex;justify-content:space-between;align-items:center}.composer-footer>span{font-size:.6875rem;font-family:var(--mono);color:var(--muted)}.composer-footer button{background:var(--accent);color:#243315;border:0;border-radius:7px;font-size:1.4rem;width:31px;height:31px}.assistant-footer{display:flex;flex-wrap:wrap;gap:8px;justify-content:space-between;padding:14px 20px 20px;font-size:.625rem;color:#98a79f;line-height:1.5}.assistant-footer span{font-size:.625rem}.user-message{padding:12px 14px;background:#263127;border:1px solid #3b4e35;border-radius:9px 9px 2px 9px;margin:15px 0 25px;white-space:pre-wrap;overflow-wrap:anywhere;font-size:.875rem;line-height:1.6}.message-text{white-space:pre-wrap;overflow-wrap:anywhere}.job-panel{padding:15px;margin:0 18px 15px;background:var(--surface-2);border:1px solid var(--line);border-radius:10px;font-size:.8125rem}.job-panel strong{font-size:.8125rem;font-weight:500}.job-panel p{color:var(--muted);font-size:.75rem;margin-top:7px}.job-top{display:flex;justify-content:space-between;align-items:center}.job-top button{font-size:.6875rem;padding:4px 7px}.progress-track{height:4px;background:var(--line);border-radius:4px;margin:12px 0 2px;overflow:hidden}.progress-fill{height:100%;width:0;background:var(--accent);transition:width .3s}.toast{position:fixed;bottom:20px;left:50%;transform:translateX(-50%);padding:12px 18px;background:#33402f;color:var(--text);border:1px solid #60784d;border-radius:8px;max-width:90vw;z-index:10;font-size:.875rem}dialog{background:var(--surface);color:var(--text);border:1px solid var(--line);border-radius:16px;padding:30px;max-width:600px;width:calc(100% - 32px)}dialog::backdrop{background:#000a;backdrop-filter:blur(3px)}.dialog-close{position:absolute;right:12px;top:10px;border:0;background:transparent;font-size:1.5rem}.sr-only{position:absolute;width:1px;height:1px;overflow:hidden;clip:rect(0,0,0,0);white-space:nowrap}.detail-grid{display:grid;grid-template-columns:repeat(2,1fr);gap:16px;margin:22px 0}.detail-grid>div{border:1px solid var(--line);padding:14px;border-radius:8px}.detail-grid span{font-size:.75rem;color:var(--muted);display:block;margin-bottom:7px}.detail-grid strong{font-size:1rem;font-weight:500}.actions{display:flex;flex-wrap:wrap;gap:8px;margin-top:18px}.actions button{padding:9px 13px;font-size:.875rem}.actions button.primary{background:var(--accent);color:#233014;border-color:var(--accent)}.chart-bars{display:flex;flex-direction:column;gap:15px;padding:8px 0}.chart-row{display:grid;grid-template-columns:minmax(120px,160px) 1fr 65px;gap:16px;align-items:center;font-size:.8125rem}.chart-row>span:first-child{overflow-wrap:anywhere}.bar-track{height:22px;background:var(--surface-2);border-radius:4px;overflow:hidden}.bar-fill{height:100%;background:var(--accent);border-radius:3px;min-width:2px}.bar-value{font-family:var(--mono);font-size:.8125rem}.benchmark-meta{color:var(--muted);font-size:.75rem;margin-top:15px;line-height:1.8}.plan-card{border:1px solid #536846;border-radius:10px;background:#222e20;padding:15px;margin-top:12px}.plan-card p{font-size:.8125rem;line-height:1.6}.plan-card .actions{margin-top:10px}.error-text{color:var(--error)}.catalog-card{border:1px solid var(--line);border-radius:9px;padding:13px;margin:10px 0;font-size:.8125rem}.catalog-card strong{display:block;margin-bottom:5px}.catalog-card button{margin-top:10px;padding:6px 10px;background:var(--accent);color:#233014;font-size:.75rem;border-color:var(--accent)}.stream-output{max-height:340px;overflow:auto;border-left:2px solid #526c3f;padding-left:12px;font-size:.875rem}.warning-note{font-size:.75rem;color:#d4cca8;line-height:1.6;margin-top:10px}.mini-label{font-family:var(--mono);font-size:.6875rem;color:var(--muted);letter-spacing:.03em}
+@media(min-width:1450px){.workspace{grid-template-columns:minmax(0,1fr) 420px}.control-room{padding:40px 48px}.runtime-map{min-height:295px}.assistant-header,.conversation{padding:28px}.composer{margin:0 24px}.model-node{min-width:180px}}
+@media(max-width:1080px){.workspace{grid-template-columns:minmax(0,1fr) 320px}.control-room{padding:28px 22px}.topbar{padding:0 22px}.stat{padding:0 12px}.stat strong{font-size:1.5rem}.stat>span{font-size:.75rem}.runtime-map{gap:25px;padding:22px 16px}.runtime-node{min-width:118px;padding:14px}.runtime-node::after{width:26px;right:-26px}.model-node{min-width:130px;padding:12px;max-width:100%}.model-nodes{padding-left:14px}.model-node::before{left:-15px;width:14px}.model-row{grid-template-columns:minmax(0,1fr) 68px 65px;gap:7px}.model-row .model-numeric:last-child{display:none}.map-footer{font-size:.625rem;padding:12px}.top-status{font-size:.75rem}.assistant-header{padding:22px 16px}}
+@media(max-width:820px){.workspace{display:flex;flex-direction:column}.assistant-panel{position:relative;height:auto;max-height:none;border-left:0;border-top:1px solid var(--line);min-height:500px}.control-room{padding:26px 20px}.conversation{max-height:480px}.topbar{height:64px;padding:0 20px}.top-status .separator,.top-status #connection-status{display:none}.pilot{font-size:.625rem;padding-left:10px;margin-left:0}.brand{font-size:.875rem;gap:9px}.runtime-map{min-height:230px}.model-nodes{gap:8px}.model-node{max-width:220px}.stats-row{margin-bottom:24px}.stat{padding:0 10px}.stat>span{font-size:.6875rem}.map-footer{font-size:.625rem}.assistant-footer{font-size:.6875rem}.top-status{font-size:.6875rem;gap:5px}.assistant-header h2{font-size:1rem}.assistant-header div>span{font-size:.875rem}.private-badge{font-size:.75rem}.suggestions button{font-size:.875rem}textarea{font-size:1rem}.assistant-message{font-size:1rem}.model-row{grid-template-columns:minmax(0,1fr) 75px 80px}.model-node strong{font-size:.875rem}.model-node span{font-size:.75rem}}
+@media(max-width:430px){.top-status{display:none}.stat strong{font-size:1.35rem}.stat>span{min-height:30px;line-height:1.4}.stat small{font-size:.6875rem}.runtime-map{gap:20px;padding:18px 12px}.runtime-node{min-width:106px;padding:12px}.runtime-node::after{right:-21px;width:21px}.model-node{min-width:120px}.map-footer{flex-wrap:wrap}.panel-heading>.secondary{font-size:.75rem}.chart-row{grid-template-columns:110px 1fr 52px;gap:8px}}
+@media(prefers-reduced-motion:reduce){*{scroll-behavior:auto!important;transition:none!important;animation:none!important}}
+/* Live control and measurement surfaces. */
+[hidden]{display:none!important}.model-picker{display:flex;flex-direction:column;gap:7px;margin:0 18px 12px;color:var(--muted);font-size:.75rem}.model-picker select{width:100%;color:var(--text);background:var(--surface);border:1px solid var(--line);border-radius:7px;padding:9px;font-size:.875rem}.model-node.selected{box-shadow:0 0 0 2px var(--cyan)}.model-row.selected{background:#1a2826}.job-state{color:var(--muted);margin-left:8px;letter-spacing:0;font-family:inherit;text-transform:lowercase}.run-metrics{font-family:var(--mono);font-size:.75rem;color:var(--cyan);margin-top:12px!important}.stream-output{margin-top:14px;line-height:1.7}.chart-heading{display:flex;justify-content:space-between;gap:12px;align-items:center;flex-wrap:wrap;margin-bottom:10px}.chart-heading h3{font-size:.875rem;margin:0;font-weight:500}.chart-heading>span{font-size:.75rem;color:var(--muted)}.second-chart{margin-top:26px}.bar-fill.latency{background:var(--cyan)}.benchmark-details{margin-top:16px;border-top:1px solid var(--line);padding-top:14px;font-size:.875rem}.benchmark-details summary{cursor:pointer;color:var(--muted)}.sample-record{display:flex;flex-direction:column;gap:5px;padding:18px 0;border-bottom:1px solid var(--line);font-size:.875rem;overflow-wrap:anywhere}.sample-record span{color:var(--muted);font-size:.75rem}.chart-row{grid-template-columns:minmax(125px,170px) 1fr 64px}.bar-value{text-align:right}.bar-value small{font-size:.75rem}.detail-grid strong,#detail-title{overflow-wrap:anywhere}#operator-status{overflow:hidden;text-overflow:ellipsis;white-space:nowrap;max-width:calc(100% - 44px)}.job-top{gap:12px}.job-top button{flex-shrink:0}.assistant-message,.stream-output,.user-message{font-size:1rem}.live-label,.map-footer,.private-badge,.message-label,.composer-footer>span,.assistant-footer,.assistant-footer span,.job-top button,.mini-label,.pilot,.stat>span,.top-status,.stat small{font-size:.75rem}.stat>span,.suggestions button,.top-status{font-size:.875rem}.composer textarea{font-size:1rem}.model-node span,.model-title small,.model-status,.model-numeric{font-size:.8125rem}.map-footer{line-height:1.6;flex-wrap:wrap}.topbar{gap:14px}.assistant-panel{min-height:650px}.job-panel p{overflow-wrap:anywhere}.catalog-card p{line-height:1.6}.model-nodes{min-width:0}.model-node{min-width:0;flex:1 1 150px}button,a,select{-webkit-tap-highlight-color:transparent}
+@media(max-width:1080px){.stat strong{font-size:1.45rem}.stat>span{min-height:32px}.model-row{grid-template-columns:minmax(0,1fr) 76px 76px}.chart-row{grid-template-columns:130px 1fr 60px;gap:10px}}
+@media(max-width:820px){.assistant-panel{min-height:650px}.top-status{font-size:.75rem}.stat>span{font-size:.75rem}.stat small{font-size:.75rem}.model-picker select{font-size:1rem}.assistant-panel{height:auto}.model-nodes{flex-basis:140px}.model-node{min-width:0}.map-footer{font-size:.75rem}}
+@media(max-width:430px){.chart-row{grid-template-columns:100px 1fr 54px;gap:8px}.model-row{grid-template-columns:minmax(0,1fr) 72px 62px}.model-numeric,.model-status{font-size:.75rem}.stats-row{gap:0}.stat{padding:0 7px}.stat strong{font-size:1.3rem}.stat>span{min-height:36px}.runtime-node{min-width:100px}.control-room{padding:24px 16px}.runtime-map{gap:17px}.runtime-node::after{width:18px;right:-18px}.map-footer>span{flex-wrap:wrap}.assistant-header{padding:20px 16px}.conversation{padding:22px 18px}}
+@media(pointer:coarse){button{min-height:44px}.model-picker select{min-height:44px}.composer-footer button{min-width:44px}.icon-button{width:44px;height:44px}}
+.activity-panel{margin-top:32px}.activity-row{display:flex;align-items:flex-start;gap:16px;padding:17px 0;border-top:1px solid var(--line);font-size:.875rem}.activity-state{min-width:78px;font-family:var(--mono);font-size:.75rem;color:var(--muted);padding-top:3px}.activity-row>div{flex:1;min-width:0}.activity-row strong{font-weight:500;overflow-wrap:anywhere}.activity-row p{font-size:.8125rem;line-height:1.6;margin-top:5px;color:var(--muted);overflow-wrap:anywhere}.activity-time{font-size:.75rem;white-space:nowrap;color:var(--muted)}.activity-row button{padding:5px 9px;font-size:.75rem}.embedded-viewer .workspace{display:block;max-width:1440px}.embedded-viewer .assistant-panel{display:none}.embedded-viewer .control-room{padding:34px clamp(20px,4vw,60px)}.embedded-viewer .runtime-map{min-height:240px}.embedded-viewer .topbar{height:64px}.embedded-viewer .workspace{min-height:calc(100vh - 64px)}.embedded-viewer .stats-row{max-width:1050px}.embedded-viewer .model-nodes{flex:1}.embedded-viewer .model-node{flex:0 1 230px}.embedded-viewer .model-row{grid-template-columns:minmax(0,1fr) 100px 110px 100px}.embedded-viewer .runtime-node{min-width:160px}.embedded-viewer .model-row .model-numeric:last-child{display:block}
+@media(max-width:600px){.embedded-viewer .model-row{grid-template-columns:minmax(0,1fr) 76px 76px}.embedded-viewer .model-row .model-numeric:last-child{display:none}.embedded-viewer .runtime-node{min-width:100px}.activity-row{gap:10px}.activity-state{min-width:64px}.activity-time{display:none}.embedded-viewer .model-node{flex-basis:160px}.embedded-viewer .control-room{padding:24px 16px}}
+.embedded-viewer [data-chat]{display:none}
+
+/* Ollama identity: use the official mark at every logo size. */
+.brand{font-size:1.45rem;letter-spacing:-.04em;text-transform:none;gap:12px}.brand .brand-mark{display:block;width:40px;height:40px;padding:0;background:none;clip-path:none;border-radius:10px;object-fit:contain}.brand .pilot{font-size:.65rem;letter-spacing:.13em;margin-left:10px;padding-left:20px}.runtime-node .node-icon{display:block;width:42px;height:42px;padding:0;margin:0 auto 9px;background:none;border-radius:10px;object-fit:contain}.assistant-header .assistant-avatar{padding:0;object-fit:contain;background:none;border-radius:9px}
+
+.runtime-notice{margin:0 0 20px;padding:16px 18px;border:1px solid #614840;border-radius:12px;background:#241e1a}.runtime-notice[hidden]{display:none}.runtime-notice strong{color:var(--error);font-size:.9rem}.runtime-notice p{margin:8px 0;color:var(--text);font-size:.85rem;line-height:1.5}.runtime-notice summary{cursor:pointer;color:var(--muted);font-size:.8rem}.runtime-notice pre{white-space:pre-wrap;overflow-wrap:anywhere;font: .75rem/1.5 var(--mono);color:var(--error)}
diff --git a/store/agents/ollama/harness.json b/store/agents/ollama/harness.json
new file mode 100644
index 000000000..f9a5c1a76
--- /dev/null
+++ b/store/agents/ollama/harness.json
@@ -0,0 +1,28 @@
+{
+ "spec": 1,
+ "id": "autonomous/ollama",
+ "name": "Ollama",
+ "category": "Local AI",
+ "author": "OpenHarness contributors",
+ "description": "Run local language models by talking to Codex. A live viewer shows Ollama models, memory, deployment progress and measured performance on your Mac.",
+ "engine": "codex",
+ "workspace": {
+ "template": "template",
+ "marker": "local-ai.json"
+ },
+ "agent": {
+ "instructions": "agent/AGENTS.md",
+ "env": {
+ "LOCAL_AI": "${dsh}/toolchain/local-ai"
+ }
+ },
+ "toolchain": {
+ "setup": "toolchain/setup.sh",
+ "doctor": "toolchain/doctor.sh"
+ },
+ "viewer": {
+ "command": "./viewer.sh",
+ "url": "http://127.0.0.1:${port}/?view=grid"
+ },
+ "verdict": ".harness/verdict.json"
+}
diff --git a/store/agents/ollama/licenses/ollama.txt b/store/agents/ollama/licenses/ollama.txt
new file mode 100644
index 000000000..8e3dc978a
--- /dev/null
+++ b/store/agents/ollama/licenses/ollama.txt
@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) Ollama
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
diff --git a/store/agents/ollama/src/catalog.mjs b/store/agents/ollama/src/catalog.mjs
new file mode 100644
index 000000000..616380e9d
--- /dev/null
+++ b/store/agents/ollama/src/catalog.mjs
@@ -0,0 +1,6 @@
+export const CATALOG = [
+ { id: 'qwen3:0.6b', name: 'Qwen 3 · 0.6B', purpose: 'A lightweight first model', description: 'Quick to download. Good for trying the harness, short answers, and simple tasks.', downloadGB: 0.523, use: 'general', small: true, source: 'https://ollama.com/library/qwen3:0.6b' },
+ { id: 'qwen2.5-coder:1.5b', name: 'Qwen 2.5 Coder · 1.5B', purpose: 'Small coding assistant', description: 'A compact model for code snippets, explanations, and simple programming tasks.', downloadGB: 0.986, use: 'coding', small: true, source: 'https://ollama.com/library/qwen2.5-coder:1.5b' },
+ { id: 'gemma3:1b', name: 'Gemma 3 · 1B', purpose: 'Everyday text tasks', description: 'A small text-only model for summarizing, rewriting, and everyday questions.', downloadGB: 0.815, use: 'writing', small: true, source: 'https://ollama.com/library/gemma3:1b' },
+];
+export const DEFAULT_MODEL = CATALOG[0].id;
diff --git a/store/agents/ollama/src/controller.mjs b/store/agents/ollama/src/controller.mjs
new file mode 100644
index 000000000..79613900b
--- /dev/null
+++ b/store/agents/ollama/src/controller.mjs
@@ -0,0 +1,191 @@
+import { EventEmitter } from 'node:events';
+import { randomUUID, createHash } from 'node:crypto';
+import os from 'node:os';
+import { setTimeout as delay } from 'node:timers/promises';
+import * as ollama from './ollama.mjs';
+import { resolveIntent, validateAction } from './intents.mjs';
+import { CATALOG } from './catalog.mjs';
+
+export const BENCHMARK_PROMPT = 'Explain how a bicycle works to a curious beginner. Discuss the pedals, chain, gears, wheels, brakes, and balance. Write a clear explanation of at least 250 words without a heading.';
+export const BENCHMARK_SETTINGS = { context: 2048, maxTokens: 128, seed: 42, temperature: 0, thinking: false };
+export const median = values => { const v = values.filter(Number.isFinite).sort((a, b) => a - b); return v.length ? (v[Math.floor((v.length - 1) / 2)] + v[Math.ceil((v.length - 1) / 2)]) / 2 : null; };
+const terminal = job => !['queued', 'running'].includes(job.status);
+export const safeError = error => /TLS handshake timeout|fetch failed/.test(error.message) ? 'The model download or runtime connection timed out. Check your connection and retry; partial downloads can resume.' : String(error.message || error).replace(/https?:\/\/[^\s"<>]+/g, url => url.split('?')[0]).slice(0, 700);
+
+export class Controller extends EventEmitter {
+ constructor(store, adapter = ollama, profile = {}) {
+ super();
+ this.profile = { id: 'ollama', name: 'Ollama', catalog: CATALOG, ...profile };
+ this.store = store; this.adapter = adapter; this.queue = []; this.aborts = new Map(); this.actions = new Map(); this.processing = false; this.closing = false;
+ this.system = { platform: os.platform(), arch: os.arch(), chip: os.cpus()[0]?.model || 'Local machine', totalMemory: os.totalmem(), cores: os.cpus().length };
+ this.runtime = { id: this.profile.id, name: this.profile.name, online: false, models: [] };
+ this.updatedAt = null;
+ }
+ snapshot() { return { runtime: this.runtime, system: { ...this.system, freeMemory: os.freemem() }, benchmarks: this.store.data.benchmarks, loadTests: this.store.data.loadTests, runs: this.store.data.runs, jobs: this.store.data.jobs, catalog: this.profile.catalog, updatedAt: this.updatedAt }; }
+ async refresh() {
+ if (this.refreshing) return this.refreshing;
+ this.refreshing = this.adapter.discover().then(runtime => { this.runtime = runtime; this.updatedAt = new Date().toISOString(); this.emit('state', this.snapshot()); }).finally(() => { this.refreshing = null; });
+ return this.refreshing;
+ }
+ async command(text, { selectedModel, history } = {}) {
+ await this.refresh();
+ const intent = (this.profile.resolveIntent || resolveIntent)(text, { models: this.runtime.models, selectedModel, profile: this.profile });
+ if (intent.kind === 'action') {
+ if (intent.action.type === 'chat') intent.action.history = history;
+ return { kind: 'job', job: await this.submit(intent.action) };
+ }
+ if (intent.kind === 'inventory') {
+ const models = this.runtime.models;
+ return { kind: 'reply', message: !this.runtime.online ? `${this.profile.name} is not running. Say “start ${this.profile.name}” or “run a lightweight model”.` : models.length ? `${models.length} model${models.length === 1 ? '' : 's'} on disk, ${models.filter(m => m.running).length} in memory.\n${models.map(m => `${m.name} — ${m.running ? 'in memory' : 'on disk'}`).join('\n')}` : `${this.profile.name} is ready. Say “run a lightweight model” to download your first model.` };
+ }
+ return intent;
+ }
+ async submit(input) {
+ if (this.closing) throw new Error('Harness is shutting down. Try again after it restarts.');
+ if (this.queue.length >= 8) throw new Error('There are already eight jobs waiting. Let them finish or cancel one.');
+ const action = this.profile.validateAction ? this.profile.validateAction(input) : validateAction(input, this.profile.validateModel);
+ const job = { id: randomUUID(), type: action.type, model: action.model || null, models: action.models || [], status: 'queued', progress: 0, message: 'Waiting for the model operator', createdAt: new Date().toISOString(), output: '', result: null };
+ this.store.data.jobs.unshift(job);
+ this.store.data.jobs = this.store.data.jobs.filter((j, i) => i < 40 || !terminal(j));
+ this.actions.set(job.id, action); this.queue.push(job.id);
+ await this.store.save(); this.emit('job', job);
+ void this.drain();
+ return job;
+ }
+ async cancel(id) {
+ const job = this.store.data.jobs.find(j => j.id === id);
+ if (!job) throw new Error('Job not found.');
+ if (terminal(job)) return job;
+ this.aborts.get(id)?.abort(new Error('Cancelled by you.'));
+ if (job.status === 'queued') { job.status = 'cancelled'; job.message = 'Cancelled before starting.'; job.completedAt = new Date().toISOString(); this.queue = this.queue.filter(queued => queued !== id); this.actions.delete(id); await this.store.save(); }
+ else job.message = 'Cancelling…';
+ this.emit('job', job); return job;
+ }
+ async drain() {
+ if (this.processing) return;
+ this.processing = true;
+ try {
+ while (this.queue.length && !this.closing) {
+ const id = this.queue.shift(); const job = this.store.data.jobs.find(j => j.id === id); const action = this.actions.get(id);
+ if (!job || job.status !== 'queued') continue;
+ const abort = new AbortController(); this.aborts.set(id, abort);
+ job.status = 'running'; job.startedAt = new Date().toISOString();
+ const progress = (message, value, extra) => { job.message = message; if (Number.isFinite(value)) job.progress = Math.max(0, Math.min(100, value)); if (extra) job.download = extra; this.emit('job', job); };
+ try {
+ await this.store.save();
+ progress(`Connecting to ${this.profile.name}`, 1);
+ await this.adapter.ensureRunning(this.store.directory, { signal: abort.signal, onProgress: progress });
+ await this.refresh();
+ job.result = await this.execute(action, { signal: abort.signal, progress, job });
+ abort.signal.throwIfAborted();
+ job.status = 'succeeded'; job.progress = 100; job.message = job.result.message;
+ } catch (error) {
+ job.status = abort.signal.aborted ? 'cancelled' : 'failed';
+ job.message = abort.signal.aborted ? 'Cancelled. Any downloaded files stay cached; retrying can resume the download.' : safeError(error);
+ } finally {
+ job.completedAt = new Date().toISOString(); this.aborts.delete(id); this.actions.delete(id);
+ try { await this.refresh(); await this.store.save(); } catch (error) { job.persistenceError = error.message; }
+ this.emit('job', job);
+ }
+ }
+ } finally { this.processing = false; }
+ }
+ async deploy(model, { signal, progress }, load = true, context) {
+ await this.refresh();
+ if (!this.runtime.models.some(m => m.id === model)) {
+ progress(`Inspecting ${model} before downloading`, 2);
+ await this.adapter.pullModel(model, { signal, onProgress: (message, value, extra) => progress(message, 3 + (value || 0) * 0.75, extra) });
+ }
+ await this.adapter.localModelDetails(model, { signal });
+ if (load) {
+ progress(`Loading ${model} into memory`, 82);
+ await this.adapter.loadModel(model, { signal, context: context ?? (this.profile.preserveServingConfig ? undefined : 4096) });
+ for (let i = 0; i < 15; i++) { await this.refresh(); if (this.runtime.models.some(m => m.id === model && m.running)) return; await delay(200, undefined, { signal }); }
+ throw new Error(`${this.profile.name} accepted the load request, but the model did not appear in memory. Check the runtime log and try again.`);
+ }
+ await this.refresh();
+ if (!this.runtime.models.some(m => m.id === model)) throw new Error(`${this.profile.name} did not report the model as installed after the download.`);
+ }
+ async execute(action, ctx) {
+ const { signal, progress, job } = ctx;
+ if (action.model) job.activeModel = action.model;
+ if (action.type === 'start') return { message: `${this.profile.name} ${this.runtime.version} is ready. ${this.runtime.models.length} models found on disk.` };
+ if (action.type === 'serve') {
+ await this.deploy(action.model, ctx, false);
+ progress(`Starting ${action.model} with the requested serving settings`, 80);
+ await this.adapter.loadModel(action.model, { ...action, signal });
+ await this.refresh();
+ return { model: action.model, serving: this.runtime.serving, message: `${action.model} is serving · ${this.runtime.serving.context.toLocaleString()} context · ${this.runtime.serving.maxSequences} parallel sequences · prefix cache ${this.runtime.serving.prefixCaching ? 'on' : 'off'}.` };
+ }
+ if (action.type === 'load_test') {
+ await this.deploy(action.model, ctx);
+ progress('Warming the server before the concurrent test', 5);
+ await this.adapter.generate(action.model, [{ role: 'user', content: BENCHMARK_PROMPT }], { ...BENCHMARK_SETTINGS, maxTokens: 32, signal });
+ const serving = { ...this.runtime.serving }; const samples = []; let next = 0; let complete = 0;
+ const began = performance.now();
+ const worker = async () => {
+ while (next < action.requests) {
+ signal.throwIfAborted(); const index = next++;
+ try {
+ const { metrics } = await this.adapter.generate(action.model, [{ role: 'user', content: `${BENCHMARK_PROMPT} Reference number: ${index + 1}.` }], { ...BENCHMARK_SETTINGS, signal });
+ samples.push({ index, status: 'succeeded', ...metrics });
+ } catch (error) { signal.throwIfAborted(); samples.push({ index, status: 'failed', error: safeError(error) }); }
+ complete++; progress(`${complete} of ${action.requests} requests completed · concurrency ${action.concurrency}`, 10 + complete / action.requests * 85);
+ }
+ };
+ const outcomes = await Promise.allSettled(Array.from({ length: action.concurrency }, worker)); signal.throwIfAborted();
+ const rejected = outcomes.find(outcome => outcome.status === 'rejected'); if (rejected) throw rejected.reason;
+ const totalMs = performance.now() - began;
+ const passed = samples.filter(s => s.status === 'succeeded');
+ const latency = passed.map(s => s.totalMs).sort((a, b) => a - b);
+ const result = { id: randomUUID(), at: new Date().toISOString(), model: action.model, digest: this.runtime.models.find(m => m.id === action.model)?.digest, runtime: { name: this.profile.name, version: this.runtime.version, backend: this.runtime.backend, backendVersion: this.runtime.backendVersion }, hardware: this.system, concurrency: action.concurrency, requests: action.requests, succeeded: passed.length, failed: samples.length - passed.length, totalMs, aggregateTokensPerSecond: passed.reduce((sum, s) => sum + s.tokens, 0) / (totalMs / 1000), medianLatencyMs: median(latency), p95LatencyMs: latency.length ? latency[Math.ceil(latency.length * 0.95) - 1] : null, medianFirstTokenMs: median(passed.map(s => s.firstTokenMs)), settings: { ...BENCHMARK_SETTINGS, context: serving.context }, serving, prompt: BENCHMARK_PROMPT, promptVariation: 'Reference number: {1-based request index}.', mode: 'one discarded warmup; repeated prompt prefixes may be cached; client end-to-end measurements', samples: samples.sort((a, b) => a.index - b.index) };
+ this.store.data.loadTests.unshift(result); this.store.data.loadTests = this.store.data.loadTests.slice(0, 200); await this.store.save();
+ if (!passed.length) throw new Error(`All ${action.requests} requests failed. The load-test record contains each error.`);
+ return { model: action.model, loadTestId: result.id, message: `${passed.length}/${action.requests} requests succeeded at concurrency ${action.concurrency} · ${result.aggregateTokensPerSecond.toFixed(1)} aggregate tokens/s · ${Math.round(result.p95LatencyMs)} ms p95 request latency · ${result.failed} errors.` };
+ }
+ if (['deploy', 'download'].includes(action.type)) {
+ await this.deploy(action.model, ctx, action.type === 'deploy');
+ return { model: action.model, message: action.type === 'deploy' ? `${action.model} is ready on this Mac. Ask a question, or say “benchmark it”. ${this.profile.readyMessage || 'Idle models unload after 15 minutes; their files stay on disk.'}` : `${action.model} is downloaded. Say “run ${action.model}” to load it into memory.` };
+ }
+ if (['unload', 'unload_all'].includes(action.type)) {
+ const targets = action.type === 'unload_all' ? this.runtime.models.filter(m => m.running).map(m => m.id) : [action.model];
+ for (const model of targets) { progress(`Releasing ${model} from memory`, 30); await this.adapter.unloadModel(model, { signal }); }
+ await this.refresh();
+ if (targets.some(model => this.runtime.models.some(m => m.id === model && m.running))) throw new Error('The runtime still reports a requested model in memory. It may be in use by another client.');
+ return { message: targets.length ? `${targets.length === 1 ? targets[0] : `${targets.length} models`} unloaded. Downloaded files are still on disk.` : 'No models were in memory.' };
+ }
+ if (action.type === 'chat') {
+ await this.deploy(action.model, ctx);
+ progress(`Generating with ${action.model}`, 90);
+ let lastEmit = 0;
+ const generated = await this.adapter.generate(action.model, [...action.history, { role: 'user', content: action.prompt }], { signal, onToken: text => { job.output += text; if (Date.now() - lastEmit > 80) { this.emit('job', job); lastEmit = Date.now(); } } });
+ const run = { id: randomUUID(), at: new Date().toISOString(), ...generated.metrics };
+ this.store.data.runs.unshift(run); this.store.data.runs = this.store.data.runs.slice(0, 200);
+ return { model: action.model, metrics: generated.metrics, message: `Response from ${action.model}.` };
+ }
+ if (action.type === 'benchmark') {
+ const results = [];
+ for (const [modelIndex, model] of action.models.entries()) {
+ job.activeModel = model;
+ await this.deploy(model, ctx, true, BENCHMARK_SETTINGS.context);
+ const metadata = this.runtime.models.find(m => m.id === model);
+ progress(`Warming up ${model} (not included in results)`, 5 + modelIndex / action.models.length * 90);
+ await this.adapter.generate(model, [{ role: 'user', content: BENCHMARK_PROMPT }], { ...BENCHMARK_SETTINGS, maxTokens: 32, signal });
+ const samples = [];
+ for (let i = 0; i < 3; i++) {
+ progress(`${model} · measured run ${i + 1} of 3`, 10 + (modelIndex + i / 3) / action.models.length * 85);
+ const { metrics } = await this.adapter.generate(model, [{ role: 'user', content: BENCHMARK_PROMPT }], { ...BENCHMARK_SETTINGS, signal });
+ if (!Number.isFinite(metrics.tokensPerSecond) || !Number.isFinite(metrics.firstTokenMs)) throw new Error(`${this.profile.name} did not return usable timing data; no benchmark was recorded.`);
+ samples.push(metrics);
+ }
+ await this.refresh();
+ const result = { id: randomUUID(), model, digest: metadata?.digest || null, at: new Date().toISOString(), runtime: { name: this.profile.name, version: this.runtime.version, mlxVersion: this.runtime.mlxVersion, backend: this.runtime.backend, backendVersion: this.runtime.backendVersion }, hardware: this.system, settings: BENCHMARK_SETTINGS, prompt: BENCHMARK_PROMPT, promptHash: createHash('sha256').update(BENCHMARK_PROMPT).digest('hex'), mode: this.profile.benchmarkMode || 'warm; repeated prompt may use prefix cache', samples, medianTokensPerSecond: median(samples.map(s => s.tokensPerSecond)), medianFirstTokenMs: median(samples.map(s => s.firstTokenMs)), medianFirstOutputMs: median(samples.map(s => s.firstOutputMs)), residentBytes: this.runtime.models.find(m => m.id === model)?.resident?.size || null };
+ this.store.data.benchmarks.unshift(result); this.store.data.benchmarks = this.store.data.benchmarks.slice(0, 200); results.push(result);
+ await this.store.save();
+ }
+ return { model: action.models.at(-1), benchmarkIds: results.map(r => r.id), message: results.map(r => `${r.model}: ${r.medianTokensPerSecond.toFixed(1)} tokens/s · ${Math.round(r.medianFirstTokenMs)} ms to first token (median of 3 warm runs).`).join('\n') };
+ }
+ throw new Error('Unknown action.');
+ }
+ async close() { this.closing = true; for (const abort of this.aborts.values()) abort.abort(new Error('Harness shutting down.')); await this.store.save(); await this.adapter.close?.(); }
+}
diff --git a/store/agents/ollama/src/intents.mjs b/store/agents/ollama/src/intents.mjs
new file mode 100644
index 000000000..7aa35e8df
--- /dev/null
+++ b/store/agents/ollama/src/intents.mjs
@@ -0,0 +1,91 @@
+import { CATALOG, DEFAULT_MODEL } from './catalog.mjs';
+import { validateModel } from './ollama.mjs';
+
+export const HELP = 'Try “run a lightweight model”, “find a model for coding”, “benchmark it”, “ask it: explain unified memory”, or “free up memory”. You can also name any local Ollama model, such as qwen3:0.6b.';
+
+// Only explicit operational requests enter the executor. Model replies are text,
+// never instructions. There is no shell-command generation or eval here.
+export function resolveIntent(input, { models = [], selectedModel, profile = {} } = {}) {
+ const catalog = profile.catalog || CATALOG;
+ const defaultModel = profile.defaultModel || DEFAULT_MODEL;
+ const normalize = profile.validateModel || validateModel;
+ const help = profile.help || HELP;
+ if (typeof input !== 'string' || !input.trim() || input.length > 4000) throw new Error('Enter a request between 1 and 4,000 characters.');
+ const text = input.trim();
+ const lower = text.toLowerCase();
+ const selected = selectedModel ? normalize(selectedModel) : null;
+ const preferred = selected || models.find(m => m.running)?.id || models[0]?.id;
+ const action = (type, props = {}) => ({ kind: 'action', action: { type, ...props } });
+ const reply = message => ({ kind: 'reply', message });
+ const choose = () => reply('Which model? Select one in the model map, or include its name in your request.');
+ // Prompt contents must never be parsed as commands or model identifiers.
+ const ask = text.match(/^(?:(?:please |can you )?)(?:ask|tell|chat with)\s+(.+?)\s*:\s*([\s\S]+)$/i);
+ if (ask) {
+ const target = /^(it|me|us|the model|my model)$/i.test(ask[1]) ? preferred : normalize(ask[1].trim());
+ return target ? action('chat', { model: target, prompt: ask[2] }) : choose();
+ }
+ const names = [...new Set([...models.map(m => m.id), ...catalog.map(m => m.id)])];
+ const mentioned = names.filter(name => lower.includes(name.toLowerCase()));
+ for (const match of text.matchAll(profile.modelPattern || /\b(?:[a-z0-9][a-z0-9._-]*\/)?[a-z0-9][a-z0-9._-]*:[a-z0-9][a-z0-9._-]*\b/g)) {
+ if (!mentioned.includes(match[0]) && !/^\d+:\d+$/.test(match[0])) mentioned.push(normalize(match[0]));
+ }
+ const modelForTask = /cod(?:e|ing)|programming/.test(lower) ? catalog[1].id : /writing|summari[sz]|rewrite/.test(lower) ? catalog[2].id : defaultModel;
+ const lead = lower.replace(/^(?:please\s+|(?:can|could|would) you\s+|i(?:'d| would) like (?:you )?to\s+|i want (?:you )?to\s+)+/, '');
+ if (/\b(don['’]?t|do not|never|not yet|cancel|abort)\b/.test(lower) && /\b(run|load|download|deploy|start|stop|unload|benchmark|compare|cancel|abort)\b/.test(lower)) return reply('I haven’t started an action. Use the Cancel button on an active job to stop it.');
+ if (/^(how (?:do|would|can|should)|what (?:if|happens)|why|explain|help)\b/.test(lead)) return reply(help);
+ if (/^(show|list|what|which|check|status|refresh)\b/.test(lead) && /models?|running|loaded|status|memory|installed/.test(lead) || /^(status|models|refresh)[.!?]?$/.test(lead)) return { kind: 'inventory' };
+ if (/^(find|recommend|suggest|pick|choose|what['’]?s|what is)\b/.test(lead) && /model|coding|writing/.test(lead)) return { kind: 'catalog', message: 'Start small, then choose a larger model once you have a baseline. These downloads run locally on this Mac.', models: /coding|code|programming/.test(lower) ? [catalog[1]] : /writing|summari[sz]/.test(lower) ? [catalog[2]] : catalog };
+ if (/^(delete|remove|erase)\b/.test(lead)) return reply('This pilot keeps downloaded models on disk. To release their memory, say “unload it” or “free up memory”.');
+ if (/^(free (?:up )?(?:my |the )?memory|unload all(?: models)?|stop all(?: models)?|release (?:all )?memory)[.!?]?$/.test(lead)) return action('unload_all');
+ if (/^(start|launch|open) (?:the )?(?:ollama|mlx(?:-lm)?|runtime|server)[.!?]?$/.test(lead)) return action('start');
+
+ const operation = lead.match(/^(run|deploy|load|start|download|pull|install|unload|stop|benchmark|test|measure|compare)\b/);
+ if (operation) {
+ if (/\b(?:and then|then|after that)\b/.test(lead) || /\band\s+(run|deploy|download|stop|unload|benchmark|ask)\b/.test(lead)) return reply('Let’s do one operation at a time. Start with the first request, then ask for the next step when it finishes.');
+ const verb = operation[1];
+ const rest = lead.slice(verb.length).trim().replace(/[.!?]+$/, '');
+ let model = mentioned[0];
+ if (!model && /^[a-z0-9][a-z0-9._-]*(?:\/[a-z0-9][a-z0-9._-]*)?$/.test(rest) && !/^(it|this|that|model|ollama|speed|performance|all)$/.test(rest)) {
+ model = names.find(name => name.split(':')[0] === rest) || normalize(rest);
+ }
+ if (!model && /\b(it|this model|that model|my model|the model|the running model)\b/.test(rest)) model = preferred;
+ if (['benchmark', 'test', 'measure', 'compare'].includes(verb)) {
+ if (verb === 'compare' && /\ball\b/.test(rest)) return models.length > 4 ? reply('This pilot compares up to four models at a time. Name the models to compare.') : models.length ? action('benchmark', { models: models.map(m => m.id) }) : choose();
+ const targets = mentioned.length ? mentioned : model ? [model] : preferred ? [preferred] : [];
+ if (verb === 'compare' && targets.length < 2) return reply('Name two models from this framework to compare. Each gets the same prompt and settings.');
+ return targets.length ? action('benchmark', { models: targets }) : choose();
+ }
+ if (['unload', 'stop'].includes(verb)) return model ? action('unload', { model }) : choose();
+ if (!model && /\b(lightweight|small|tiny|fast|coding|writing|starter|first)\b/.test(rest)) model = modelForTask;
+ if (!model) return reply('Name the model you want, or say “run a lightweight model” and I’ll use Qwen 3 0.6B.');
+ return action(['download', 'pull', 'install'].includes(verb) ? 'download' : 'deploy', { model });
+ }
+ if (/\b(benchmark|speed|fast|tokens per second|performance)\b/.test(lower) && /^(how|test|measure)\b/.test(lead)) return preferred ? action('benchmark', { models: [preferred] }) : choose();
+ if (/^(chat|talk)(?: with| to)?(?: it| the model)?[.!?]?$/.test(lead)) return preferred ? reply(`${preferred} is selected. Ask a question below, or say “ask it: …”.`) : choose();
+ if (/^(run|deploy|download|load|start|stop|unload|benchmark|compare|install|delete|remove)\b/.test(lead) || /\b(shell|terminal|sudo|rm -rf)\b/.test(lower)) return reply(help);
+ return preferred ? action('chat', { model: preferred, prompt: text }) : reply(help);
+}
+
+export function validateAction(input, normalize = validateModel) {
+ if (!input || typeof input !== 'object' || Array.isArray(input)) throw new Error('Provide a model action.');
+ const { type } = input;
+ if (!['start', 'download', 'deploy', 'unload', 'unload_all', 'benchmark', 'chat'].includes(type)) throw new Error('That action is not supported by this pilot.');
+ const action = { type };
+ if (['download', 'deploy', 'unload', 'chat'].includes(type)) action.model = normalize(input.model);
+ if (type === 'benchmark') {
+ if (!Array.isArray(input.models) || !input.models.length || input.models.length > 4) throw new Error('Benchmark between one and four models at a time.');
+ action.models = [...new Set(input.models.map(normalize))];
+ }
+ if (type === 'chat') {
+ if (typeof input.prompt !== 'string' || !input.prompt.trim() || input.prompt.length > 4000) throw new Error('Enter a prompt between 1 and 4,000 characters.');
+ action.prompt = input.prompt.trim();
+ action.history = [];
+ if (Array.isArray(input.history)) {
+ action.history = input.history.slice(-6).map(m => {
+ if (!['user', 'assistant'].includes(m.role) || typeof m.content !== 'string' || m.content.length > 6000) throw new Error('Invalid chat history.');
+ return { role: m.role, content: m.content };
+ });
+ }
+ }
+ return action;
+}
diff --git a/store/agents/ollama/src/json-worker.mjs b/store/agents/ollama/src/json-worker.mjs
new file mode 100644
index 000000000..7129ce6a1
--- /dev/null
+++ b/store/agents/ollama/src/json-worker.mjs
@@ -0,0 +1,80 @@
+import { spawn } from 'node:child_process';
+import { access, mkdir, open } from 'node:fs/promises';
+import { constants } from 'node:fs';
+import { createInterface } from 'node:readline';
+import { randomUUID } from 'node:crypto';
+import { join } from 'node:path';
+
+// Private, request-correlated JSON-lines transport. It never invokes a shell.
+export class JsonWorker {
+ constructor({ python, script, directory, env = {}, name }) {
+ Object.assign(this, { python, script, directory, env, name });
+ this.child = null; this.ready = false; this.pending = new Map(); this.state = {}; this.closed = false;
+ }
+ async start() {
+ if (this.closed) throw new Error(`${this.name} is closing.`);
+ if (this.stopping) await this.stopping;
+ if (this.child && this.ready) return;
+ if (this.starting) return this.starting;
+ this.starting = this.launch().finally(() => { this.starting = null; });
+ return this.starting;
+ }
+ async launch() {
+ await access(this.python, constants.X_OK).catch(() => { throw new Error(`${this.name} needs setup. Run this harness's setup command.`); });
+ await mkdir(this.directory, { recursive: true });
+ const log = await open(join(this.directory, 'model-cache.log'), 'a', 0o600);
+ const child = spawn(this.python, ['-u', this.script], { stdio: ['pipe', 'pipe', log.fd], env: { ...process.env, ...this.env } });
+ await log.close(); this.child = child; this.ready = false;
+ child.stdin.on('error', () => {});
+ await new Promise((resolve, reject) => {
+ const timer = setTimeout(() => { this.stop(child); reject(new Error(`${this.name} startup timed out.`)); }, 45000);
+ const fail = error => { clearTimeout(timer); reject(error); };
+ child.once('error', error => { if (this.child === child) this.child = null; fail(error); });
+ const lines = createInterface({ input: child.stdout });
+ lines.on('line', line => {
+ if (this.child !== child) return;
+ try {
+ if (line.length > 8 * 1024 * 1024) throw new Error('Oversized worker event.');
+ const event = JSON.parse(line);
+ if (event.event === 'ready') { clearTimeout(timer); this.ready = true; this.state = event; resolve(); return; }
+ if (event.state) this.state = { ...this.state, ...event.state };
+ const pending = this.pending.get(event.id); if (!pending) return;
+ if (event.event === 'done') pending.resolve(event.result);
+ else if (event.event === 'error') pending.reject(new Error(event.message));
+ else pending.onEvent(event);
+ } catch (error) { this.stop(child); fail(error); }
+ });
+ child.once('close', (code, signal) => {
+ clearTimeout(timer); lines.close();
+ if (this.child === child) { this.child = null; this.ready = false; }
+ const error = new Error(`${this.name} stopped (${signal || code}).`);
+ for (const pending of this.pending.values()) if (pending.child === child) pending.reject(error);
+ fail(error);
+ });
+ });
+ }
+ stop(child = this.child) {
+ if (!child || this.stopping) return this.stopping;
+ this.ready = false;
+ this.stopping = new Promise(resolve => {
+ const timer = setTimeout(() => child.kill('SIGKILL'), 1500);
+ child.once('close', () => { clearTimeout(timer); resolve(); });
+ child.kill('SIGTERM');
+ }).finally(() => { this.stopping = null; });
+ return this.stopping;
+ }
+ async request(action, body = {}, { signal, onEvent = () => {}, timeout = 180000 } = {}) {
+ signal?.throwIfAborted(); await this.start(); signal?.throwIfAborted();
+ const child = this.child; const id = randomUUID();
+ return new Promise((resolve, reject) => {
+ let settled = false;
+ const finish = (fn, value) => { if (settled) return; settled = true; clearTimeout(timer); signal?.removeEventListener('abort', abort); this.pending.delete(id); fn(value); };
+ const abort = () => { this.stop(child); finish(reject, signal.reason || new Error('Cancelled.')); };
+ const timer = setTimeout(() => { this.stop(child); finish(reject, new Error(`${this.name} ${action} timed out.`)); }, timeout);
+ this.pending.set(id, { child, resolve: value => finish(resolve, value), reject: error => finish(reject, error), onEvent });
+ signal?.addEventListener('abort', abort, { once: true });
+ child.stdin.write(JSON.stringify({ id, action, ...body }) + '\n', error => { if (error) finish(reject, error); });
+ });
+ }
+ async close() { this.closed = true; await this.starting?.catch(() => {}); await this.stop(); }
+}
diff --git a/store/agents/ollama/src/mlx.mjs b/store/agents/ollama/src/mlx.mjs
new file mode 100644
index 000000000..a347b2ea1
--- /dev/null
+++ b/store/agents/ollama/src/mlx.mjs
@@ -0,0 +1,156 @@
+import { spawn } from 'node:child_process';
+import { access, mkdir, open } from 'node:fs/promises';
+import { constants } from 'node:fs';
+import { createInterface } from 'node:readline';
+import { join } from 'node:path';
+import { randomUUID } from 'node:crypto';
+import { totalmem } from 'node:os';
+import { performance } from 'node:perf_hooks';
+
+export function validateMlxModel(model) {
+ if (typeof model !== 'string' || model.length > 180 || !/^[A-Za-z0-9][A-Za-z0-9_.-]*\/[A-Za-z0-9][A-Za-z0-9_.-]*$/.test(model) || model.includes('..')) throw new Error('Use a Hugging Face model ID, such as mlx-community/Qwen3-0.6B-4bit.');
+ return model;
+}
+
+export const MLX_CATALOG = [
+ { id: 'mlx-community/Qwen3-0.6B-4bit', name: 'Qwen 3 · 0.6B', purpose: 'A lightweight first model', description: 'A small MLX model for trying local chat and measuring this Mac.', downloadGB: 0.4, use: 'general', small: true },
+ { id: 'mlx-community/Qwen2.5-Coder-1.5B-Instruct-4bit', name: 'Qwen 2.5 Coder · 1.5B', purpose: 'Small coding assistant', description: 'A compact MLX coding model for explanations and short snippets.', downloadGB: 1.0, use: 'coding', small: true },
+ { id: 'mlx-community/Qwen2.5-0.5B-Instruct-4bit', name: 'Qwen 2.5 · 0.5B', purpose: 'Small baseline', description: 'A compact instruction model for short answers and a second performance baseline.', downloadGB: 0.3, use: 'writing', small: true },
+].map(model => ({ ...model, source: `https://huggingface.co/${model.id}` }));
+
+export const MLX_PROFILE = {
+ id: 'mlx-lm', name: 'MLX-LM', catalog: MLX_CATALOG, defaultModel: MLX_CATALOG[0].id,
+ validateModel: validateMlxModel, modelPattern: /\b[A-Za-z0-9][A-Za-z0-9_.-]*\/[A-Za-z0-9][A-Za-z0-9_.-]*\b/g,
+ help: 'Try “run a lightweight model”, “benchmark it”, “ask it: explain unified memory”, or “free up memory”. Use Hugging Face MLX model IDs, such as mlx-community/Qwen3-0.6B-4bit.',
+ benchmarkMode: 'warm model; fresh prompt cache for each sample',
+ readyMessage: 'MLX-LM keeps one model loaded per workspace. Switching models releases the previous model; idle models unload after 15 minutes.',
+};
+
+export class MlxAdapter {
+ constructor({ packageRoot, dataDir, python, interpreterArgs = ['-u'] } = {}) {
+ this.python = python || join(packageRoot, '.venv/bin/python');
+ this.interpreterArgs = interpreterArgs;
+ this.workerPath = join(packageRoot, 'worker.py');
+ this.dataDir = dataDir;
+ this.child = null; this.starting = null; this.pending = new Map(); this.generation = 0;
+ this.current = { models: [] }; this.version = null; this.mlxVersion = null; this.expiresAt = null;
+ this.installed = false; this.closed = false;
+ }
+ async ensureRunning(_directory, { signal } = {}) {
+ signal?.throwIfAborted();
+ if (this.closed) throw new Error('The MLX-LM workspace is closing.');
+ if (this.stopping) await this.stopping;
+ signal?.throwIfAborted();
+ if (this.closed) throw new Error('The MLX-LM workspace is closing.');
+ if (this.child && this.ready) return;
+ if (this.starting) return this.starting;
+ this.starting = this.startWorker().finally(() => { this.starting = null; });
+ return this.starting;
+ }
+ async startWorker() {
+ try { await access(this.python, constants.X_OK); this.installed = true; }
+ catch { throw new Error('MLX-LM needs setup. Run this harness’s setup command to install its isolated Python environment.'); }
+ await mkdir(this.dataDir, { recursive: true });
+ const log = await open(join(this.dataDir, 'mlx-lm.log'), 'a', 0o600);
+ const child = spawn(this.python, [...this.interpreterArgs, this.workerPath], { stdio: ['pipe', 'pipe', log.fd], env: { ...process.env, HF_HUB_DISABLE_TELEMETRY: '1', TOKENIZERS_PARALLELISM: 'false', HARNESS_MLX_REGISTRY: join(this.dataDir, 'mlx-models.json') } });
+ await log.close();
+ this.child = child; this.ready = false;
+ child.stdin.on('error', () => {}); // The write callback rejects the corresponding request.
+ const generation = ++this.generation;
+ await new Promise((resolve, reject) => {
+ const timer = setTimeout(() => { child.kill('SIGKILL'); reject(new Error('MLX-LM did not start within 45 seconds. Check .harness/mlx-lm.log.')); }, 45000);
+ const startupFailure = error => { clearTimeout(timer); reject(error); };
+ child.once('error', startupFailure);
+ const lines = createInterface({ input: child.stdout });
+ lines.on('line', line => {
+ try {
+ if (line.length > 8 * 1024 * 1024) throw new Error('MLX-LM returned an oversized event.');
+ const event = JSON.parse(line);
+ if (this.generation !== generation) return;
+ if (event.event === 'ready') {
+ clearTimeout(timer); this.ready = true; this.current = event;
+ this.version = event.version; this.mlxVersion = event.mlxVersion;
+ resolve(); return;
+ }
+ if (event.state) this.current = { ...this.current, ...event.state };
+ const pending = this.pending.get(event.id); if (!pending) return;
+ if (event.event === 'error') pending.reject(new Error(event.message));
+ else if (event.event === 'done') pending.resolve(event.result);
+ else pending.onEvent(event);
+ } catch (error) { child.kill('SIGKILL'); startupFailure(error); }
+ });
+ child.once('exit', (code, signal) => {
+ clearTimeout(timer); lines.close();
+ const error = new Error(`MLX-LM worker stopped (${signal || code}). Its loaded model was released. Check .harness/mlx-lm.log if unexpected.`);
+ if (this.generation === generation) { this.child = null; this.ready = false; this.expiresAt = null; this.current = { ...this.current, models: this.current.models.map(m => ({ ...m, running: false, resident: null })) }; }
+ for (const request of this.pending.values()) if (request.child === child) request.reject(error);
+ startupFailure(error);
+ });
+ });
+ }
+ stopWorker(child, signal = 'SIGTERM') {
+ if (child !== this.child) return;
+ this.ready = false;
+ if (this.stopping) return;
+ this.stopping = new Promise(resolve => {
+ const timer = setTimeout(() => child.kill('SIGKILL'), 1500);
+ child.once('close', () => { clearTimeout(timer); resolve(); });
+ child.kill(signal);
+ }).finally(() => { this.stopping = null; });
+ }
+ async request(action, body = {}, { signal, onEvent = () => {}, timeout = 180000 } = {}) {
+ await this.ensureRunning(undefined, { signal }); signal?.throwIfAborted();
+ const child = this.child; const id = randomUUID();
+ return new Promise((resolve, reject) => {
+ let settled = false;
+ const finish = (fn, value) => { if (settled) return; settled = true; clearTimeout(timer); signal?.removeEventListener('abort', abort); this.pending.delete(id); fn(value); };
+ const abort = () => { this.stopWorker(child); finish(reject, signal.reason || new Error('Cancelled.')); };
+ const timer = setTimeout(() => { this.stopWorker(child, 'SIGKILL'); finish(reject, new Error(`MLX-LM ${action} timed out. Its worker was stopped to release memory.`)); }, timeout);
+ this.pending.set(id, { child, resolve: value => finish(resolve, value), reject: error => finish(reject, error), onEvent });
+ signal?.addEventListener('abort', abort, { once: true });
+ child.stdin.write(JSON.stringify({ id, action, ...body }) + '\n', error => { if (error) finish(reject, error); });
+ });
+ }
+ async discover() {
+ let error;
+ try {
+ await this.ensureRunning();
+ if (!this.pending.size) {
+ const loaded = this.current.models.find(m => m.running);
+ if (loaded && this.expiresAt && Date.now() > this.expiresAt) await this.unloadModel(loaded.id);
+ await this.request('inventory');
+ }
+ } catch (e) { error = e.message; }
+ return { id: 'mlx-lm', name: 'MLX-LM', online: Boolean(this.child && this.ready), installed: this.installed, version: this.version, mlxVersion: this.mlxVersion, error,
+ endpoint: 'Private local worker', cacheRoot: this.current.cacheRoot, memoryScope: 'MLX active allocations; one model per workspace',
+ models: this.current.models.map(m => ({ ...m, resident: m.resident ? { ...m.resident, expires_at: this.expiresAt ? new Date(this.expiresAt).toISOString() : null } : null })) };
+ }
+ async pullModel(model, { signal, onProgress = () => {} } = {}) {
+ return this.request('download', { model: validateMlxModel(model), memoryBudget: Math.floor(totalmem() * 0.8) }, { signal, timeout: 30 * 60 * 1000, onEvent: event => { if (event.event === 'progress') onProgress(event.message, event.percent, { completed: event.completed, total: event.total }); } });
+ }
+ localModelDetails(model, options) { return this.request('details', { model: validateMlxModel(model) }, options); }
+ async loadModel(model, { signal, context = 4096 } = {}) {
+ const result = await this.request('load', { model: validateMlxModel(model), context }, { signal });
+ this.expiresAt = Date.now() + 15 * 60 * 1000; return result;
+ }
+ async unloadModel(model, { signal } = {}) {
+ const result = await this.request('unload', { model: validateMlxModel(model) }, { signal });
+ this.expiresAt = null; return result;
+ }
+ async generate(model, messages, { signal, onToken = () => {}, ...options } = {}) {
+ const started = performance.now(); let firstTokenMs = null; let firstOutputMs = null;
+ const result = await this.request('generate', { model: validateMlxModel(model), messages, options }, { signal, onEvent: event => {
+ if (event.event === 'first_token' && firstTokenMs === null) firstTokenMs = performance.now() - started;
+ if (event.event === 'token') { if (firstOutputMs === null && /\S/.test(event.text)) firstOutputMs = performance.now() - started; onToken(event.text); }
+ } });
+ this.expiresAt = Date.now() + 15 * 60 * 1000;
+ result.metrics.firstTokenMs = firstTokenMs; result.metrics.firstOutputMs = firstOutputMs;
+ return result;
+ }
+ async close() {
+ this.closed = true;
+ if (this.starting) await this.starting.catch(() => {});
+ if (this.child) this.stopWorker(this.child);
+ await this.stopping;
+ }
+}
diff --git a/store/agents/ollama/src/ollama.mjs b/store/agents/ollama/src/ollama.mjs
new file mode 100644
index 000000000..17063efec
--- /dev/null
+++ b/store/agents/ollama/src/ollama.mjs
@@ -0,0 +1,187 @@
+import { access, mkdir, open, statfs } from 'node:fs/promises';
+import { constants } from 'node:fs';
+import { homedir, totalmem } from 'node:os';
+import { join } from 'node:path';
+import { spawn } from 'node:child_process';
+import { setTimeout as delay } from 'node:timers/promises';
+import { performance } from 'node:perf_hooks';
+
+export const OLLAMA_URL = 'http://127.0.0.1:11434';
+export const DEFAULT_CONTEXT = 4096;
+let starting;
+
+export function validateModel(model) {
+ if (typeof model !== 'string' || model.length > 180 || !/^[a-zA-Z0-9][a-zA-Z0-9._-]*(?:\/[a-zA-Z0-9][a-zA-Z0-9._-]*)?(?::[a-zA-Z0-9][a-zA-Z0-9._-]*)?$/.test(model) || model.includes('..')) throw new Error('Use an Ollama model name, such as qwen3:0.6b.');
+ if (/(?:^|[-:])cloud(?:$|[-:])/i.test(model)) throw new Error('This harness runs local models. Choose a model without a cloud tag.');
+ return model.includes(':') ? model : `${model}:latest`;
+}
+
+export async function* readNDJSON(body) {
+ const decoder = new TextDecoder();
+ let buffer = '';
+ for await (const chunk of body) {
+ buffer += typeof chunk === 'string' ? chunk : decoder.decode(chunk, { stream: true });
+ if (buffer.length > 8 * 1024 * 1024) throw new Error('The runtime returned an oversized event.');
+ let newline;
+ while ((newline = buffer.indexOf('\n')) !== -1) {
+ const line = buffer.slice(0, newline).trim(); buffer = buffer.slice(newline + 1);
+ if (line) { const event = JSON.parse(line); if (event.error) throw new Error(event.error); yield event; }
+ }
+ }
+ buffer += decoder.decode();
+ if (buffer.trim()) { const event = JSON.parse(buffer); if (event.error) throw new Error(event.error); yield event; }
+}
+
+export async function streamRequest(path, body, { signal, timeout = 30 * 60 * 1000, onEvent = () => {} } = {}) {
+ const response = await fetch(`${OLLAMA_URL}${path}`, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ ...body, stream: true }), signal: signal ? AbortSignal.any([signal, AbortSignal.timeout(timeout)]) : AbortSignal.timeout(timeout) });
+ if (!response.ok) { const message = await response.text(); throw new Error(message.slice(0, 600) || `Ollama returned ${response.status}`); }
+ let last;
+ for await (const event of readNDJSON(response.body)) { last = event; await onEvent(event); }
+ return last;
+}
+
+export async function ensureRunning(dataDir, { signal, onProgress = () => {} } = {}) {
+ try { return await ollamaJSON('/api/version', { timeout: 1500, signal }); } catch {}
+ signal?.throwIfAborted();
+ if (starting) return starting;
+ starting = (async () => {
+ const binary = await findOllama();
+ if (!binary) throw new Error('Ollama is not installed. Install the official Ollama app from ollama.com, then try again.');
+ onProgress('Starting Ollama on this Mac');
+ await mkdir(dataDir, { recursive: true });
+ const log = await open(join(dataDir, 'ollama.log'), 'a', 0o600);
+ let failure;
+ const child = spawn(binary, ['serve'], { env: { ...process.env, OLLAMA_HOST: '127.0.0.1:11434', OLLAMA_NO_CLOUD: '1' }, detached: true, stdio: ['ignore', log.fd, log.fd] });
+ child.on('error', error => { failure = error; });
+ child.unref();
+ await log.close();
+ const deadline = Date.now() + 30000;
+ while (Date.now() < deadline) {
+ if (failure) throw failure;
+ signal?.throwIfAborted();
+ try { return await ollamaJSON('/api/version', { timeout: 1000, signal }); } catch {}
+ if (child.exitCode !== null) throw new Error(`Ollama exited during startup. Details are in ${join(dataDir, 'ollama.log')}.`);
+ await delay(300, undefined, { signal });
+ }
+ throw new Error('Ollama did not become ready within 30 seconds. Check the runtime log and try again.');
+ })().finally(() => { starting = null; });
+ return starting;
+}
+
+export async function modelManifest(model, { signal } = {}) {
+ const normalized = validateModel(model);
+ const [path, tag] = normalized.split(':');
+ const repository = path.includes('/') ? path : `library/${path}`;
+ const url = `https://registry.ollama.ai/v2/${repository}/manifests/${tag}`;
+ const response = await fetch(url, { signal: signal ? AbortSignal.any([signal, AbortSignal.timeout(12000)]) : AbortSignal.timeout(12000), headers: { Accept: 'application/vnd.docker.distribution.manifest.v2+json' } });
+ if (!response.ok) throw new Error(response.status === 404 ? `The model ${model} was not found in Ollama's registry. Check its name and tag.` : `Could not inspect the download size (${response.status}). Try again when the registry is reachable.`);
+ const manifest = await response.json();
+ const layers = [...(manifest.layers || []), ...(manifest.config ? [manifest.config] : [])];
+ if (!layers.length || layers.some(layer => !Number.isSafeInteger(layer.size) || layer.size < 0)) throw new Error('The model registry returned an invalid manifest.');
+ return { bytes: layers.reduce((total, layer) => total + layer.size, 0), repository, tag };
+}
+
+export async function pullModel(model, options = {}) {
+ model = validateModel(model);
+ const manifest = await modelManifest(model, options);
+ if (manifest.bytes > totalmem() * 0.8) throw new Error(`This model download is ${(manifest.bytes / 1024 ** 3).toFixed(1)} GiB, too large for the pilot's memory budget on this Mac. Choose a smaller quantization or model.`);
+ try {
+ const disk = await statfs(process.env.OLLAMA_MODELS || join(homedir(), '.ollama'));
+ if (disk.bavail * disk.bsize < manifest.bytes + 2 * 1024 ** 3) throw new Error('There is not enough free disk space for this download plus 2 GiB of headroom.');
+ } catch (error) { if (!['ENOENT', 'EPERM', 'EACCES'].includes(error.code)) throw error; }
+ const layers = new Map();
+ const last = await streamRequest('/api/pull', { model }, { ...options, onEvent: event => {
+ if (event.digest && event.total) layers.set(event.digest, { total: event.total, completed: event.completed || 0 });
+ const known = [...layers.values()];
+ const completed = known.reduce((sum, layer) => sum + layer.completed, 0);
+ options.onProgress?.(event.status || 'Downloading model', Math.min(99, Math.floor(completed / manifest.bytes * 100)), { completed, total: manifest.bytes });
+ } });
+ if (last?.status !== 'success') throw new Error('The model download ended before Ollama confirmed success. You can retry to resume it.');
+ return manifest;
+}
+
+export async function localModelDetails(model, options = {}) {
+ const details = await ollamaJSON('/api/show', { method: 'POST', body: { model: validateModel(model) }, timeout: 15000, ...options });
+ if (details.remote_host || details.remote_model) throw new Error('This model forwards inference to a remote service. Choose a local model for this harness.');
+ if (Array.isArray(details.capabilities) && !details.capabilities.includes('completion')) throw new Error('This pilot supports language generation models. The selected model does not expose completion capability.');
+ return details;
+}
+
+export async function loadModel(model, { signal, context = DEFAULT_CONTEXT, keepAlive = '15m' } = {}) {
+ await localModelDetails(model, { signal });
+ return ollamaJSON('/api/generate', { method: 'POST', body: { model, prompt: '', stream: false, keep_alive: keepAlive, options: { num_ctx: context } }, timeout: 180000, signal });
+}
+
+export async function unloadModel(model, { signal } = {}) {
+ return ollamaJSON('/api/generate', { method: 'POST', body: { model: validateModel(model), stream: false, keep_alive: 0 }, timeout: 30000, signal });
+}
+
+export async function generate(model, messages, { signal, maxTokens = 512, context = DEFAULT_CONTEXT, keepAlive = '15m', seed = 42, temperature = 0.3, thinking = true, onToken = () => {} } = {}) {
+ const details = await localModelDetails(model, { signal });
+ const start = performance.now();
+ let firstTokenMs = null;
+ let firstOutputMs = null;
+ let text = '';
+ const body = { model, messages, options: { num_predict: maxTokens, num_ctx: context, temperature, seed }, keep_alive: keepAlive };
+ if (details.capabilities?.includes('thinking')) body.think = /^gpt-oss(?::|$)/.test(model) ? 'low' : Boolean(thinking);
+ const last = await streamRequest('/api/chat', body, { signal, timeout: 180000, onEvent: event => {
+ const piece = event.message?.content || '';
+ if (firstTokenMs === null && (piece || event.message?.thinking)) firstTokenMs = performance.now() - start;
+ if (piece) { if (firstOutputMs === null) firstOutputMs = performance.now() - start; text += piece; onToken(piece); }
+ } });
+ if (!last?.done) throw new Error('The model response ended unexpectedly. Partial output was not counted as a successful run.');
+ const elapsedMs = performance.now() - start;
+ return { text, metrics: {
+ model, ...responseMetrics(last), firstTokenMs, firstOutputMs, elapsedMs,
+ context, maxTokens, temperature, seed, thinking: body.think ?? null,
+ } };
+}
+
+export function responseMetrics(last) {
+ return {
+ tokens: last.eval_count ?? null,
+ promptTokens: last.prompt_eval_count ?? null,
+ tokensPerSecond: last.eval_duration > 0 && Number.isFinite(last.eval_count) ? last.eval_count / (last.eval_duration / 1e9) : null,
+ promptTokensPerSecond: last.prompt_eval_duration > 0 && Number.isFinite(last.prompt_eval_count) ? last.prompt_eval_count / (last.prompt_eval_duration / 1e9) : null,
+ loadMs: Number.isFinite(last.load_duration) ? last.load_duration / 1e6 : null,
+ evalMs: Number.isFinite(last.eval_duration) ? last.eval_duration / 1e6 : null,
+ doneReason: last.done_reason || null,
+ };
+}
+
+export async function findOllama() {
+ const candidates = [process.env.OLLAMA_BIN, '/usr/local/bin/ollama', '/opt/homebrew/bin/ollama', '/Applications/Ollama.app/Contents/Resources/ollama', ...String(process.env.PATH || '').split(':').map(p => join(p, 'ollama'))].filter(Boolean);
+ for (const candidate of candidates) {
+ try { await access(candidate, constants.X_OK); return candidate; } catch {}
+ }
+ return null;
+}
+
+export async function ollamaJSON(path, { method = 'GET', body, timeout = 5000, signal } = {}) {
+ const response = await fetch(`${OLLAMA_URL}${path}`, {
+ method, headers: { 'Content-Type': 'application/json' },
+ body: body === undefined ? undefined : JSON.stringify(body),
+ signal: signal ? AbortSignal.any([signal, AbortSignal.timeout(timeout)]) : AbortSignal.timeout(timeout),
+ });
+ const result = await response.json();
+ if (!response.ok || result.error) throw new Error(result.error || `Ollama returned ${response.status}`);
+ return result;
+}
+
+export async function discover() {
+ const binary = await findOllama();
+ try {
+ const [version, tags, running] = await Promise.all([ollamaJSON('/api/version'), ollamaJSON('/api/tags'), ollamaJSON('/api/ps')]);
+ return {
+ id: 'ollama', name: 'Ollama', online: true, installed: Boolean(binary), version: version.version, endpoint: OLLAMA_URL,
+ models: (tags.models || []).map(m => ({
+ id: m.name, name: m.name, size: m.size, digest: m.digest, modifiedAt: m.modified_at,
+ family: m.details?.family, parameters: m.details?.parameter_size, quantization: m.details?.quantization_level,
+ running: Boolean((running.models || []).find(r => r.name === m.name || r.model === m.name)),
+ resident: (running.models || []).find(r => r.name === m.name || r.model === m.name) || null,
+ })).filter(m => !/(?:^|[-:])cloud(?:$|[-:])/i.test(m.name)),
+ };
+ } catch (error) {
+ return { id: 'ollama', name: 'Ollama', online: false, installed: Boolean(binary), version: null, endpoint: OLLAMA_URL, models: [], error: error.message };
+ }
+}
diff --git a/store/agents/ollama/src/server.mjs b/store/agents/ollama/src/server.mjs
new file mode 100644
index 000000000..06984c602
--- /dev/null
+++ b/store/agents/ollama/src/server.mjs
@@ -0,0 +1,143 @@
+import http from 'node:http';
+import { readFile, writeFile, rename } from 'node:fs/promises';
+import { fileURLToPath, pathToFileURL } from 'node:url';
+import { join, dirname, resolve } from 'node:path';
+import { randomBytes, timingSafeEqual } from 'node:crypto';
+import { Store } from './store.mjs';
+import { Controller } from './controller.mjs';
+import { createViewerRelay } from './viewer-relay.mjs';
+
+export const ROOT = dirname(dirname(fileURLToPath(import.meta.url)));
+const assets = { '/': ['index.html', 'text/html'], '/app.js': ['app.js', 'text/javascript'], '/styles.css': ['styles.css', 'text/css'], '/assets/ollama-icon.svg': ['assets/ollama-icon.svg', 'image/svg+xml'], '/assets/ollama-icon.png': ['assets/ollama-icon.png', 'image/png'], '/favicon.ico': ['assets/ollama-icon.png', 'image/png'], '/assets/mlx-icon.svg': ['assets/mlx-icon.svg', 'image/svg+xml'], '/assets/mlx-icon.png': ['assets/mlx-icon.png', 'image/png'] };
+assets['/assets/vllm-icon.png'] = ['assets/vllm-icon.png', 'image/png'];
+const json = (res, status, value) => { res.writeHead(status, { 'Content-Type': 'application/json; charset=utf-8' }); res.end(JSON.stringify(value)); };
+
+async function readBody(req) {
+ if (!/^application\/json(?:\s*;|$)/i.test(req.headers['content-type'] || '')) throw Object.assign(new Error('Use application/json for control requests.'), { status: 415 });
+ if (Number(req.headers['content-length']) > 65536) throw Object.assign(new Error('Request too large.'), { status: 413 });
+ const chunks = []; let size = 0;
+ for await (const chunk of req) { size += chunk.length; if (size > 65536) throw Object.assign(new Error('Request too large.'), { status: 413 }); chunks.push(chunk); }
+ try { return JSON.parse(Buffer.concat(chunks).toString('utf8')); } catch { throw Object.assign(new Error('Invalid JSON request.'), { status: 400 }); }
+}
+
+export function createServer(controller, { assetRoot = join(ROOT, 'dist') } = {}) {
+ const token = randomBytes(32).toString('hex');
+ const events = new Set();
+ const sendEvent = (res, type, data) => { if (res.writableLength > 1024 * 1024) { res.destroy(); events.delete(res); } else if (!res.destroyed) res.write(`event: ${type}\ndata: ${JSON.stringify(data)}\n\n`); };
+ const onState = state => { for (const res of events) sendEvent(res, 'state', state); };
+ const onJob = job => { for (const res of events) sendEvent(res, 'job', job); };
+ controller.on('state', onState); controller.on('job', onJob);
+ const server = http.createServer(async (req, res) => {
+ try {
+ const port = server.address()?.port;
+ const hosts = [`127.0.0.1:${port}`, `localhost:${port}`];
+ if (!hosts.includes(req.headers.host)) return json(res, 403, { error: 'Local requests only.' });
+ const origin = `http://${req.headers.host}`;
+ if (req.headers.origin && req.headers.origin !== origin || req.headers['sec-fetch-site'] === 'cross-site') return json(res, 403, { error: 'Cross-origin control is disabled.' });
+ const url = new URL(req.url, origin);
+ res.setHeader('X-Content-Type-Options', 'nosniff');
+ res.setHeader('Referrer-Policy', 'no-referrer');
+ res.setHeader('Cache-Control', 'no-store');
+ res.setHeader('Content-Security-Policy', "default-src 'self'; script-src 'self'; style-src 'self' 'unsafe-inline'; connect-src 'self'; img-src 'self' data:; frame-ancestors 'self'; base-uri 'none'; form-action 'self'");
+ if (req.method === 'GET' && url.pathname === '/api/health') return json(res, 200, { application: 'local-ai-harnesses', protocol: 1, workspace: controller.workspace, runtime: controller.profile?.id || 'ollama', pid: process.pid });
+ if (req.method === 'POST') {
+ const submitted = Buffer.from(String(req.headers['x-harness-token'] || ''));
+ const expected = Buffer.from(token);
+ if (submitted.length !== expected.length || !timingSafeEqual(submitted, expected)) return json(res, 403, { error: 'Refresh the viewer before sending a control request.' });
+ const body = await readBody(req);
+ if (!body || typeof body !== 'object' || Array.isArray(body)) return json(res, 400, { error: 'Expected a JSON object.' });
+ if (url.pathname === '/api/command') return json(res, 200, await controller.command(body.text, { selectedModel: body.selectedModel, history: body.history }));
+ if (url.pathname === '/api/action') return json(res, 202, { kind: 'job', job: await controller.submit(body) });
+ if (url.pathname === '/api/cancel') return json(res, 200, { job: await controller.cancel(body.id) });
+ }
+ if (req.method === 'GET' && url.pathname === '/api/state') { await controller.refresh(); return json(res, 200, { ...controller.snapshot(), token }); }
+ if (req.method === 'GET' && url.pathname === '/api/jobs') return json(res, 200, { jobs: controller.snapshot().jobs });
+ if (req.method === 'GET' && url.pathname === '/api/events') {
+ if (events.size >= 32) return json(res, 503, { error: 'Too many viewer connections.' });
+ res.writeHead(200, { 'Content-Type': 'text/event-stream', Connection: 'keep-alive', 'X-Accel-Buffering': 'no' });
+ res.write('retry: 2000\n\n'); events.add(res); sendEvent(res, 'state', controller.snapshot());
+ req.on('close', () => events.delete(res)); return;
+ }
+ if (req.method === 'GET' && url.pathname === '/api/export') {
+ res.setHeader('Content-Disposition', `attachment; filename="${controller.profile?.id || 'ollama'}-benchmarks.json"`);
+ return json(res, 200, { exportedAt: new Date().toISOString(), system: controller.system, benchmarks: controller.snapshot().benchmarks, loadTests: controller.snapshot().loadTests });
+ }
+ if (req.method === 'GET' && Object.hasOwn(assets, url.pathname)) {
+ let [file, type] = assets[url.pathname];
+ if (url.pathname === '/favicon.ico' && controller.profile?.id === 'mlx-lm') file = 'assets/mlx-icon.png';
+ if (url.pathname === '/favicon.ico' && controller.profile?.id === 'vllm') file = 'assets/vllm-icon.png';
+ const data = await readFile(join(assetRoot, file));
+ res.writeHead(200, { 'Content-Type': `${type}; charset=utf-8` }); return res.end(data);
+ }
+ json(res, 404, { error: 'Not found.' });
+ } catch (error) { if (!res.headersSent) json(res, error.status || 400, { error: error.message }); else res.end(); }
+ });
+ const heartbeat = setInterval(() => { for (const res of events) res.write(': heartbeat\n\n'); }, 15000); heartbeat.unref();
+ server.on('close', () => { clearInterval(heartbeat); controller.off('state', onState); controller.off('job', onJob); });
+ server.closeEvents = () => { for (const res of events) res.end(); events.clear(); };
+ server.requestTimeout = 30000;
+ return server;
+}
+
+export async function start(port = Number(process.env.HARNESS_VIEWER_PORT || process.env.HARNESS_PORT || 4310), options = {}) {
+ if (!Number.isInteger(port) || port < 0 || port > 65535) throw new Error('The viewer port must be between 0 and 65535.');
+ const workspace = resolve(process.env.HARNESS_WORKSPACE || options.workspaceRoot || ROOT);
+ const runtime = options.profile?.id || 'ollama';
+ let marker;
+ try { marker = JSON.parse(await readFile(join(workspace, 'local-ai.json'), 'utf8')); }
+ catch (error) { if (error.code !== 'ENOENT') throw error; }
+ if (marker?.runtime && marker.runtime !== runtime) throw new Error(`This workspace belongs to ${marker.runtime}. Open a separate ${runtime} workspace.`);
+ let existing;
+ let conflictingRuntime;
+ try {
+ const endpoint = JSON.parse(await readFile(join(workspace, '.harness/endpoint.json'), 'utf8'));
+ if (endpoint.workspace === workspace && Number.isInteger(endpoint.port) && endpoint.port > 0 && endpoint.port <= 65535) {
+ const health = await (await fetch(`http://127.0.0.1:${endpoint.port}/api/health`, { signal: AbortSignal.timeout(1200) })).json();
+ if (health.application === 'local-ai-harnesses' && health.workspace === workspace && health.pid === endpoint.pid) {
+ if ((health.runtime || 'ollama') === runtime) existing = endpoint;
+ else conflictingRuntime = health.runtime || 'ollama';
+ }
+ }
+ } catch {}
+ if (conflictingRuntime) throw new Error(`This workspace is already serving ${conflictingRuntime}. Open a separate ${runtime} workspace.`);
+ if (existing) {
+ if (existing.port === port) throw new Error(`This workspace is already running at http://127.0.0.1:${port}. Open that viewer instead of starting a second copy.`);
+ const relay = createViewerRelay(existing.port);
+ await new Promise((resolve, reject) => { relay.once('error', reject); relay.listen(port, '127.0.0.1', resolve); });
+ console.log(`Local: http://127.0.0.1:${relay.address().port} (shared workspace service)`);
+ const stopRelay = () => { relay.closeAllConnections(); relay.close(); };
+ process.once('SIGINT', stopRelay); process.once('SIGTERM', stopRelay);
+ return { server: relay, controller: null };
+ }
+ const store = await new Store(join(workspace, '.harness')).init();
+ const controller = new Controller(store, options.adapter, options.profile);
+ controller.workspace = workspace;
+ const server = createServer(controller, options);
+ await new Promise((resolve, reject) => { server.once('error', reject); server.listen(port, '127.0.0.1', resolve); });
+ const boundPort = server.address().port;
+ await writeFile(join(store.directory, 'endpoint.json'), JSON.stringify({ port: boundPort, pid: process.pid, workspace, runtime, startedAt: new Date().toISOString() }), { mode: 0o600 });
+ console.log(`Local: http://127.0.0.1:${boundPort}`);
+ let verdictWriting = Promise.resolve();
+ let lastVerdict;
+ const verdict = async snapshot => {
+ const last = snapshot.jobs[0];
+ const failed = last?.status === 'failed';
+ const affected = last?.activeModel || last?.model || last?.models?.join(', ') || controller.profile.name;
+ const failure = failed ? `${affected} · ${last.type} failed: ${last.message}` : undefined;
+ const data = { spec: 1, ready: snapshot.runtime.online && !failed, summary: failed ? failure : snapshot.runtime.online ? `${snapshot.runtime.models.length} local models · ${controller.profile.name} connected` : `${controller.profile.name} is ready to start`, findings: failed ? [{ severity: 'error', kind: 'runtime', message: failure }] : [], updatedAt: new Date().toISOString() };
+ const key = JSON.stringify({ ...data, updatedAt: undefined });
+ if (key === lastVerdict) return;
+ lastVerdict = key;
+ verdictWriting = verdictWriting.catch(() => {}).then(async () => { const temp = join(store.directory, 'verdict.json.tmp'); await writeFile(temp, JSON.stringify(data, null, 2), { mode: 0o600 }); await rename(temp, join(store.directory, 'verdict.json')); });
+ await verdictWriting;
+ };
+ controller.on('state', snapshot => { void verdict(snapshot).catch(() => {}); });
+ controller.on('job', () => { void verdict(controller.snapshot()).catch(() => {}); });
+ await controller.refresh();
+ const poll = setInterval(() => { void controller.refresh().catch(error => console.error('Discovery:', error.message)); }, 5000); poll.unref();
+ let stopping = false;
+ const stop = async () => { if (stopping) return; stopping = true; clearInterval(poll); server.closeEvents(); server.close(); await controller.close(); };
+ process.once('SIGINT', stop); process.once('SIGTERM', stop);
+ return { server, controller };
+}
+if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) await start();
diff --git a/store/agents/ollama/src/store.mjs b/store/agents/ollama/src/store.mjs
new file mode 100644
index 000000000..59957fa1b
--- /dev/null
+++ b/store/agents/ollama/src/store.mjs
@@ -0,0 +1,21 @@
+import { mkdir, readFile, writeFile, rename } from 'node:fs/promises';
+import { join } from 'node:path';
+
+export class Store {
+ constructor(directory) { this.directory = directory; this.file = join(directory, 'state.json'); this.data = { benchmarks: [], loadTests: [], runs: [], jobs: [] }; this.writing = Promise.resolve(); }
+ async init() {
+ await mkdir(this.directory, { recursive: true });
+ try {
+ const parsed = JSON.parse(await readFile(this.file, 'utf8'));
+ for (const key of ['benchmarks', 'loadTests', 'runs', 'jobs']) if (Array.isArray(parsed[key])) this.data[key] = parsed[key];
+ this.data.jobs = this.data.jobs.map(job => ['queued', 'running'].includes(job.status) ? { ...job, status: 'interrupted', message: 'Harness restarted before this job completed. Retry the action to continue.', completedAt: new Date().toISOString() } : { ...job, message: String(job.message || '').replace(/https?:\/\/[^\s"<>]+/g, url => url.split('?')[0]).slice(0, 700) });
+ } catch (error) { if (error.code !== 'ENOENT') throw new Error(`Could not read saved Harness data: ${error.message}`); }
+ await this.save();
+ return this;
+ }
+ save() {
+ const content = JSON.stringify(this.data, null, 2);
+ this.writing = this.writing.catch(() => {}).then(async () => { const temp = `${this.file}.tmp`; await writeFile(temp, content, { mode: 0o600 }); await rename(temp, this.file); });
+ return this.writing;
+ }
+}
diff --git a/store/agents/ollama/src/viewer-relay.mjs b/store/agents/ollama/src/viewer-relay.mjs
new file mode 100644
index 000000000..7aa643ddd
--- /dev/null
+++ b/store/agents/ollama/src/viewer-relay.mjs
@@ -0,0 +1,26 @@
+import http from 'node:http';
+
+// Multiple views of one workspace share one controller and job queue. The native
+// host assigns a fresh viewer port; this loopback relay serves that port without
+// opening a second writer against the workspace's history.
+export function createViewerRelay(targetPort) {
+ if (!Number.isInteger(targetPort) || targetPort < 1 || targetPort > 65535) throw new Error('Invalid local service port.');
+ const server = http.createServer((req, res) => {
+ const port = server.address().port;
+ const host = req.headers.host;
+ const reject = (status, message) => { res.writeHead(status, { 'Content-Type': 'application/json' }); res.end(JSON.stringify({ error: message })); };
+ if (![`localhost:${port}`, `127.0.0.1:${port}`].includes(host)) return reject(403, 'Local requests only.');
+ if (req.headers.origin && req.headers.origin !== `http://${host}` || req.headers['sec-fetch-site'] === 'cross-site') return reject(403, 'Cross-origin control is disabled.');
+ if (!req.url.startsWith('/') || req.url.startsWith('//')) return reject(400, 'Invalid local path.');
+ const headers = { ...req.headers, host: `127.0.0.1:${targetPort}` };
+ if (headers.origin) headers.origin = `http://127.0.0.1:${targetPort}`;
+ const upstream = http.request({ hostname: '127.0.0.1', port: targetPort, path: req.url, method: req.method, headers }, response => {
+ res.writeHead(response.statusCode || 502, response.headers); response.pipe(res);
+ });
+ upstream.on('error', () => { if (!res.headersSent) reject(502, 'The workspace service stopped. Reopen this viewer to reconnect.'); else res.end(); });
+ req.on('aborted', () => upstream.destroy()); res.on('close', () => upstream.destroy());
+ req.pipe(upstream);
+ });
+ server.requestTimeout = 30000;
+ return server;
+}
diff --git a/store/agents/ollama/src/vllm.mjs b/store/agents/ollama/src/vllm.mjs
new file mode 100644
index 000000000..8603b27f4
--- /dev/null
+++ b/store/agents/ollama/src/vllm.mjs
@@ -0,0 +1,218 @@
+import { spawn } from 'node:child_process';
+import { mkdir, open, readFile } from 'node:fs/promises';
+import net from 'node:net';
+import { join } from 'node:path';
+import { randomBytes } from 'node:crypto';
+import { totalmem } from 'node:os';
+import { setTimeout as delay } from 'node:timers/promises';
+import { JsonWorker } from './json-worker.mjs';
+import { MLX_CATALOG, validateMlxModel } from './mlx.mjs';
+import { resolveIntent, validateAction } from './intents.mjs';
+
+export const DEFAULT_SERVING = Object.freeze({ context: 4096, maxSequences: 4, memoryFraction: 0.1, prefixCaching: true });
+const bounded = (value, name, min, max, integer = true) => {
+ if (!Number.isFinite(value) || (integer && !Number.isInteger(value)) || value < min || value > max) throw new Error(`${name} must be ${integer ? 'an integer ' : ''}between ${min} and ${max}.`);
+ return value;
+};
+export function validateServing(input = {}, base = DEFAULT_SERVING) {
+ const config = { ...base };
+ if (input.context !== undefined) config.context = bounded(input.context, 'Context', 512, 16384);
+ if (input.maxSequences !== undefined) config.maxSequences = bounded(input.maxSequences, 'Parallel sequences', 1, 8);
+ if (input.memoryFraction !== undefined) config.memoryFraction = bounded(input.memoryFraction, 'Metal memory fraction', 0.05, 0.8, false);
+ if (input.prefixCaching !== undefined) {
+ if (typeof input.prefixCaching !== 'boolean') throw new Error('Prefix caching must be true or false.');
+ config.prefixCaching = input.prefixCaching;
+ }
+ return config;
+}
+export function validateVllmAction(input) {
+ if (!input || !['serve', 'load_test'].includes(input.type)) return validateAction(input, validateMlxModel);
+ const action = { type: input.type, model: validateMlxModel(input.model) };
+ if (input.type === 'serve') {
+ const allowed = ['type', 'model', ...Object.keys(DEFAULT_SERVING)];
+ if (Object.keys(input).some(key => !allowed.includes(key))) throw new Error('Unknown serving setting.');
+ validateServing(input);
+ for (const key of Object.keys(DEFAULT_SERVING)) if (input[key] !== undefined) action[key] = input[key];
+ } else {
+ if (Object.keys(input).some(key => !['type', 'model', 'concurrency', 'requests'].includes(key))) throw new Error('Unknown load-test setting.');
+ action.concurrency = bounded(input.concurrency ?? 4, 'Concurrency', 1, 8);
+ action.requests = bounded(input.requests ?? 8, 'Requests', action.concurrency, 32);
+ }
+ return action;
+}
+export const VLLM_PROFILE = {
+ id: 'vllm', name: 'vLLM', catalog: MLX_CATALOG, defaultModel: MLX_CATALOG[0].id,
+ preserveServingConfig: true,
+ validateModel: validateMlxModel, validateAction: validateVllmAction,
+ modelPattern: /\b[A-Za-z0-9][A-Za-z0-9_.-]*\/[A-Za-z0-9][A-Za-z0-9_.-]*\b/g,
+ help: 'Try “run a lightweight model”, “benchmark it”, “test 4 concurrent requests”, or “free up memory”. Ask the agent to change context, parallel sequences, Metal memory budget or prefix caching.',
+ benchmarkMode: 'warm server; repeated prompt may use prefix cache; client end-to-end token throughput, including prompt processing and HTTP',
+ readyMessage: 'This workspace owns one vLLM server using the Metal backend. Idle servers stop after 15 minutes.',
+ resolveIntent(text, options) {
+ const match = typeof text === 'string' && text.trim().match(/^(?:please )?(?:test|benchmark) (\d+) concurrent requests[.!?]?$/i);
+ if (match) {
+ const model = options.selectedModel || options.models.find(m => m.running)?.id;
+ return model ? { kind: 'action', action: { type: 'load_test', model, concurrency: Number(match[1]), requests: Math.max(8, Number(match[1])) } } : { kind: 'reply', message: 'Load or select a model first.' };
+ }
+ if (typeof text === 'string' && /^(?:please )?start vllm[.!?]?$/i.test(text.trim())) return { kind: 'action', action: { type: 'start' } };
+ return resolveIntent(text, options);
+ },
+};
+
+// Decode streaming UTF-8 before splitting frames. Chunk boundaries are not tokens.
+export async function readCompletion(body, onToken = () => {}, now = () => performance.now()) {
+ const start = now(); const decoder = new TextDecoder(); let buffer = ''; let text = ''; let usage; let finishReason; let firstTokenMs = null; let firstOutputMs = null; let done = false;
+ const frame = data => {
+ if (data === '[DONE]') { done = true; return; }
+ const event = JSON.parse(data);
+ if (event.error) throw new Error(event.error.message || 'vLLM generation failed.');
+ if (event.usage) usage = event.usage;
+ for (const choice of event.choices || []) {
+ const delta = choice.delta || {};
+ const content = delta.content || '';
+ const reasoning = delta.reasoning_content || delta.reasoning || '';
+ if ((content.length || reasoning.length) && firstTokenMs === null) firstTokenMs = now() - start;
+ if (/\S/.test(content) && firstOutputMs === null) firstOutputMs = now() - start;
+ if (content) { text += content; onToken(content); }
+ if (choice.finish_reason) finishReason = choice.finish_reason;
+ }
+ };
+ const consume = () => {
+ let boundary;
+ while ((boundary = buffer.indexOf('\n\n')) >= 0) {
+ const raw = buffer.slice(0, boundary); buffer = buffer.slice(boundary + 2);
+ const data = raw.split('\n').filter(line => line.startsWith('data:')).map(line => line.slice(5).trimStart()).join('\n');
+ if (data) frame(data);
+ }
+ if (buffer.length > 4 * 1024 * 1024) throw new Error('Oversized vLLM stream frame.');
+ };
+ for await (const bytes of body) { buffer += decoder.decode(bytes, { stream: true }).replace(/\r/g, ''); consume(); }
+ buffer += decoder.decode(); consume();
+ if (!done || !Number.isInteger(usage?.completion_tokens) || !Number.isInteger(usage?.prompt_tokens)) throw new Error('vLLM returned an incomplete stream or missing token usage; no measurement was recorded.');
+ return { text, usage, firstTokenMs, firstOutputMs, finishReason };
+}
+
+async function freePort() {
+ const server = net.createServer();
+ await new Promise((resolve, reject) => { server.once('error', reject); server.listen(0, '127.0.0.1', resolve); });
+ const port = server.address().port;
+ await new Promise((resolve, reject) => server.close(error => error ? reject(error) : resolve()));
+ return port;
+}
+export function parsePrometheus(text) {
+ const sums = {};
+ for (const line of text.split('\n')) {
+ const match = line.match(/^(vllm:[a-z_]+)(?:\{.*\})?\s+([\d.eE+-]+)$/);
+ if (match && Number.isFinite(Number(match[2]))) sums[match[1]] = (sums[match[1]] || 0) + Number(match[2]);
+ }
+ return { runningRequests: sums['vllm:num_requests_running'] ?? null, waitingRequests: sums['vllm:num_requests_waiting'] ?? null, cacheUsage: sums['vllm:kv_cache_usage_perc'] ?? null, prefixQueries: sums['vllm:prefix_cache_queries_total'] ?? null, prefixHits: sums['vllm:prefix_cache_hits_total'] ?? null };
+}
+export class VllmAdapter {
+ constructor({ packageRoot, dataDir, python } = {}) {
+ Object.assign(this, { packageRoot, dataDir }); this.python = python || join(packageRoot, '.venv/bin/python');
+ this.worker = new JsonWorker({ python: this.python, script: join(packageRoot, 'cache.py'), directory: dataDir, name: 'vLLM model cache', env: { HARNESS_MODEL_REGISTRY: join(dataDir, 'vllm-models.json'), HF_HUB_DISABLE_TELEMETRY: '1' } });
+ this.config = { ...DEFAULT_SERVING }; this.server = null; this.loaded = null; this.inflight = 0; this.closed = false; this.expiresAt = null;
+ }
+ async ensureRunning(_directory, { signal } = {}) { signal?.throwIfAborted(); if (this.closed) throw new Error('vLLM is closing.'); await this.worker.start(); signal?.throwIfAborted(); }
+ async discover() {
+ try {
+ await this.ensureRunning();
+ if (this.loaded && !this.inflight && Date.now() > this.expiresAt) await this.stopServer();
+ // Avoid putting inventory requests behind a long model download.
+ if (!this.worker.pending.size) await this.worker.request('inventory', {}, { timeout: 10000 });
+ let rss = null; let metrics = {};
+ if (this.loaded && this.server) {
+ const record = this.server;
+ try {
+ metrics = parsePrometheus(await (await this.fetchNative('/metrics', { signal: AbortSignal.timeout(1500) }, record)).text());
+ if (!this.worker.pending.size) rss = (await this.worker.request('memory', { pid: record.child.pid }, { timeout: 5000 })).rssBytes;
+ } catch {} // Scrape delay does not change server health or invent measurements.
+ }
+ const versions = this.worker.state.versions;
+ const models = (this.worker.state.models || []).map(model => ({ ...model, running: model.id === this.loaded, resident: model.id === this.loaded ? { size: rss, context_length: this.config.context, expires_at: new Date(this.expiresAt).toISOString() } : null }));
+ return { id: 'vllm', name: 'vLLM', online: true, version: versions?.vllm, backend: 'Metal', backendVersion: versions?.['vllm-metal'], mlxVersion: versions?.mlx, models, serving: { ...this.config, status: this.loaded ? 'serving' : this.server ? 'starting' : 'idle', model: this.loaded, processRssBytes: rss, ...metrics } };
+ } catch (error) { return { id: 'vllm', name: 'vLLM', online: false, models: [], error: error.message, backend: 'Metal', serving: { ...this.config, status: 'offline' } }; }
+ }
+ async localModelDetails(model, { signal } = {}) { return this.worker.request('details', { model: validateMlxModel(model) }, { signal }); }
+ async pullModel(model, { signal, onProgress = () => {} } = {}) {
+ return this.worker.request('download', { model: validateMlxModel(model), memoryBudget: Math.floor(totalmem() * 0.8) }, { signal, timeout: 2 * 60 * 60 * 1000, onEvent: event => { if (event.event === 'progress') onProgress(event.message, event.percent, { completed: event.completed, total: event.total }); } });
+ }
+ async fetchNative(path, options = {}, record = this.server) {
+ if (!record) throw new Error('The vLLM server is not running.');
+ const response = await fetch(`http://127.0.0.1:${record.port}${path}`, { ...options, headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${record.key}`, ...options.headers } });
+ if (!response.ok) throw new Error(`vLLM HTTP ${response.status}: ${(await response.text()).slice(0, 700)}`);
+ return response;
+ }
+ async loadModel(model, options = {}) {
+ const config = validateServing(options, this.config); validateMlxModel(model); options.signal?.throwIfAborted();
+ if (this.loaded === model && JSON.stringify(config) === JSON.stringify(this.config)) { this.touch(); return; }
+ if (this.inflight) throw new Error('Wait for current requests before changing the server.');
+ await this.stopServer();
+ if (this.closed) throw new Error('vLLM is closing.');
+ const details = await this.localModelDetails(model, options);
+ const port = await freePort(); const key = randomBytes(32).toString('hex');
+ await mkdir(this.dataDir, { recursive: true });
+ const log = await open(join(this.dataDir, 'vllm.log'), 'a', 0o600);
+ const args = ['-u', join(this.packageRoot, 'server.py'), 'serve', details.path, '--host', '127.0.0.1', '--port', String(port), '--served-model-name', model, '--tokenizer', details.path, '--max-model-len', String(config.context), '--max-num-seqs', String(config.maxSequences), '--max-num-batched-tokens', String(config.context), '--gpu-memory-utilization', String(config.memoryFraction), config.prefixCaching ? '--enable-prefix-caching' : '--no-enable-prefix-caching', '--middleware', 'guard.LocalOnlyMiddleware'];
+ if (details.family === 'qwen3') args.push('--reasoning-parser', 'qwen3');
+ options.signal?.throwIfAborted();
+ const child = spawn(this.python, args, { detached: true, stdio: ['ignore', log.fd, log.fd], cwd: this.packageRoot, env: { ...process.env, PYTHONPATH: this.packageRoot, VLLM_API_KEY: key, HARNESS_VLLM_PORT: String(port), HARNESS_PARENT_PID: String(process.pid), HF_HUB_OFFLINE: '1', HF_HUB_DISABLE_TELEMETRY: '1', TOKENIZERS_PARALLELISM: 'false', VLLM_NO_USAGE_STATS: '1', DO_NOT_TRACK: '1', VLLM_METAL_MEMORY_FRACTION: 'auto', VLLM_MLX_DEVICE: 'gpu' } });
+ const record = { child, port, key, exited: false, error: null };
+ record.exit = new Promise(resolve => { child.once('error', error => { record.error = error; }); child.once('close', () => { record.exited = true; if (child.pid) { try { process.kill(-child.pid, 'SIGKILL'); } catch {} } if (this.server === record) { this.server = null; this.loaded = null; } resolve(); }); });
+ this.server = record; this.config = config; await log.close();
+ const abort = () => { void this.stopServer(record); };
+ options.signal?.addEventListener('abort', abort, { once: true });
+ try {
+ const deadline = Date.now() + 180000;
+ while (Date.now() < deadline) {
+ options.signal?.throwIfAborted();
+ if (record.exited) {
+ const tail = (await readFile(join(this.dataDir, 'vllm.log'), 'utf8')).slice(-2500);
+ throw new Error(`vLLM did not start. ${record.error?.message || tail}`);
+ }
+ try {
+ const response = await this.fetchNative('/v1/models', { signal: AbortSignal.timeout(1000) }, record);
+ const inventory = await response.json();
+ if (inventory.data?.some(item => item.id === model)) { options.signal?.throwIfAborted(); this.loaded = model; this.touch(); return; }
+ } catch (error) { if (options.signal?.aborted) throw error; }
+ await delay(300, undefined, { signal: options.signal });
+ }
+ throw new Error('vLLM startup exceeded three minutes. Check .harness/vllm.log.');
+ } catch (error) { await this.stopServer(record); throw error; }
+ finally { options.signal?.removeEventListener('abort', abort); }
+ }
+ touch() { this.expiresAt = Date.now() + 15 * 60 * 1000; }
+ async stopServer(record = this.server) {
+ if (!record) return;
+ if (record.stopping) return record.stopping;
+ if (this.server === record) this.loaded = null;
+ record.stopping = (async () => {
+ // Only this detached, harness-owned process group is eligible for termination.
+ const kill = signal => { if (!record.child.pid) return; try { process.kill(-record.child.pid, signal); } catch (error) { if (error.code !== 'ESRCH') throw error; } };
+ if (!record.exited) kill('SIGTERM');
+ const timer = setTimeout(() => { try { kill('SIGKILL'); } catch {} }, 4000);
+ try { await record.exit; } finally { clearTimeout(timer); }
+ // The close handler reaps remaining group members once, including after crashes.
+ if (this.server === record) { this.server = null; this.loaded = null; }
+ })();
+ return record.stopping;
+ }
+ async unloadModel(model) { if (this.loaded === model || !this.loaded) await this.stopServer(); }
+ async generate(model, messages, { signal, onToken, maxTokens = 512, temperature = 0.7, seed = 42, thinking = false } = {}) {
+ signal?.throwIfAborted();
+ if (this.loaded !== model) await this.loadModel(model, { signal });
+ const record = this.server; this.inflight++; this.touch();
+ const requestSignal = AbortSignal.any([...(signal ? [signal] : []), AbortSignal.timeout(180000)]);
+ const start = performance.now(); const abort = () => { void this.stopServer(record); };
+ signal?.addEventListener('abort', abort, { once: true });
+ try {
+ const response = await this.fetchNative('/v1/chat/completions', { method: 'POST', signal: requestSignal, body: JSON.stringify({ model, messages, max_tokens: maxTokens, temperature, seed, stream: true, stream_options: { include_usage: true }, chat_template_kwargs: { enable_thinking: thinking } }) }, record);
+ const headersMs = performance.now() - start;
+ const completion = await readCompletion(response.body, onToken);
+ signal?.throwIfAborted();
+ const totalMs = performance.now() - start;
+ return { text: completion.text, metrics: { model, tokens: completion.usage.completion_tokens, promptTokens: completion.usage.prompt_tokens, tokensPerSecond: completion.usage.completion_tokens / (totalMs / 1000), firstTokenMs: completion.firstTokenMs === null ? null : headersMs + completion.firstTokenMs, firstOutputMs: completion.firstOutputMs === null ? null : headersMs + completion.firstOutputMs, totalMs, thinking, doneReason: completion.finishReason, metric: 'client_end_to_end', serving: { ...this.config } } };
+ } finally { signal?.removeEventListener('abort', abort); if (requestSignal.aborted) await this.stopServer(record); this.inflight--; this.touch(); }
+ }
+ async close() { this.closed = true; await this.stopServer(); await this.worker.close(); }
+}
diff --git a/store/agents/ollama/store.json b/store/agents/ollama/store.json
new file mode 100644
index 000000000..5ff1c2890
--- /dev/null
+++ b/store/agents/ollama/store.json
@@ -0,0 +1,7 @@
+{
+ "homepage": "https://ollama.com",
+ "upstream": "https://github.com/ollama/ollama",
+ "license": "MIT",
+ "tagline": "Start building with open models.",
+ "screenshots": []
+}
diff --git a/store/agents/ollama/template/local-ai.json b/store/agents/ollama/template/local-ai.json
new file mode 100644
index 000000000..9f799878a
--- /dev/null
+++ b/store/agents/ollama/template/local-ai.json
@@ -0,0 +1 @@
+{ "spec": 1, "runtime": "ollama", "scope": "local-language-models", "notes": "Model weights stay in Ollama's existing cache. Benchmark results belong to this workspace." }
diff --git a/store/agents/ollama/toolchain/doctor.sh b/store/agents/ollama/toolchain/doctor.sh
new file mode 100755
index 000000000..f600c4ee3
--- /dev/null
+++ b/store/agents/ollama/toolchain/doctor.sh
@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+set -euo pipefail
+cd "$(dirname "$0")/.."
+. toolchain/node.sh
+"$LOCAL_AI_NODE" --input-type=module <<'JS'
+import { findOllama } from './src/ollama.mjs';
+const major = Number(process.versions.node.split('.')[0]);
+if (major < 22) { console.error('miss Node.js 22 or newer'); process.exit(1); }
+console.log(`ok Node ${process.versions.node}`);
+const binary = await findOllama();
+if (!binary) { console.error('miss Ollama — install the official macOS app from https://ollama.com/download/mac'); process.exit(1); }
+console.log(`ok Ollama CLI (${binary})`);
+console.log('ok Local viewer and operator; no npm dependencies');
+JS
diff --git a/store/agents/ollama/toolchain/local-ai b/store/agents/ollama/toolchain/local-ai
new file mode 100755
index 000000000..8a7b30032
--- /dev/null
+++ b/store/agents/ollama/toolchain/local-ai
@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+set -euo pipefail
+package_dir="$(cd "$(dirname "$0")/.." && pwd)"
+export HARNESS_WORKSPACE="${HARNESS_WORKSPACE:-$PWD}"
+. "$package_dir/toolchain/node.sh"
+exec "$LOCAL_AI_NODE" "$package_dir/bin/harness.mjs" "$@"
diff --git a/store/agents/ollama/toolchain/node.sh b/store/agents/ollama/toolchain/node.sh
new file mode 100644
index 000000000..df3e4ee05
--- /dev/null
+++ b/store/agents/ollama/toolchain/node.sh
@@ -0,0 +1,15 @@
+# Use this Mac's Node or the interpreter managed by Harness, including GUI launches.
+local_ai_node_candidate="$(command -v node 2>/dev/null || true)"
+if [ -n "$local_ai_node_candidate" ] && "$local_ai_node_candidate" -e 'process.exit(Number(process.versions.node.split(".")[0]) >= 22 ? 0 : 1)' >/dev/null 2>&1; then
+ LOCAL_AI_NODE="$local_ai_node_candidate"
+else
+ local_ai_node_record="${ADAPTER_RUNTIME_DIR:-${HOME}/.harness/runtime}/current-node"
+ local_ai_node_candidate="$(cat "$local_ai_node_record" 2>/dev/null || true)"
+ if [ -n "$local_ai_node_candidate" ] && [ -x "${local_ai_node_candidate%/*}/node" ] && "${local_ai_node_candidate%/*}/node" -e 'process.exit(Number(process.versions.node.split(".")[0]) >= 22 ? 0 : 1)' >/dev/null 2>&1; then
+ LOCAL_AI_NODE="${local_ai_node_candidate%/*}/node"
+ else
+ echo 'miss Node.js 22 or newer — update Harness or install Node.js 22' >&2
+ return 1
+ fi
+fi
+export LOCAL_AI_NODE
diff --git a/store/agents/ollama/toolchain/setup.sh b/store/agents/ollama/toolchain/setup.sh
new file mode 100755
index 000000000..6b8eaf2b8
--- /dev/null
+++ b/store/agents/ollama/toolchain/setup.sh
@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+set -euo pipefail
+cd "$(dirname "$0")/.."
+exec ./toolchain/doctor.sh
diff --git a/store/agents/ollama/viewer.sh b/store/agents/ollama/viewer.sh
new file mode 100755
index 000000000..d4d4e1e17
--- /dev/null
+++ b/store/agents/ollama/viewer.sh
@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+set -euo pipefail
+cd "$(dirname "$0")"
+. toolchain/node.sh
+exec "$LOCAL_AI_NODE" src/server.mjs
diff --git a/store/agents/vllm/.gitignore b/store/agents/vllm/.gitignore
new file mode 100644
index 000000000..bfeee06c9
--- /dev/null
+++ b/store/agents/vllm/.gitignore
@@ -0,0 +1,10 @@
+.harness/
+.venv/
+.runtime/
+__pycache__/
+*.pyc
+*.log
+node_modules/
+.DS_Store
+!/dist/
+!/dist/**
diff --git a/store/agents/vllm/LICENSE b/store/agents/vllm/LICENSE
new file mode 100644
index 000000000..a499cd17e
--- /dev/null
+++ b/store/agents/vllm/LICENSE
@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Autonomous, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
diff --git a/store/agents/vllm/README.md b/store/agents/vllm/README.md
new file mode 100644
index 000000000..a24a20bd3
--- /dev/null
+++ b/store/agents/vllm/README.md
@@ -0,0 +1,111 @@
+# vLLM harness
+
+One **vLLM** harness, using **vLLM Metal** as its Apple Silicon backend. Install with
+`harness dsh install autonomous/vllm` in **Local AI**, alongside Grid, Ollama and MLX-LM. The agent operates
+the native framework; the branded viewer shows live state and actual measurements.
+
+Open **Local AI → vLLM → New project** in Harness. Try:
+
+- “Run a lightweight model and ask it why bicycles need brakes.”
+- “Use a 4,096-token context and allow four parallel requests.”
+- “Compare one request at a time with four concurrent requests.”
+- “Turn prefix caching off and measure again.”
+- “Unload the model but keep its files.”
+
+The shared viewer adds serving settings, active/waiting requests, KV cache usage,
+server process RSS, concurrent throughput, median/p95 latency, error counts and raw
+samples. It uses the official vLLM icon. The embedded view leaves conversation to
+the host agent; the standalone command panel has a bounded phrase parser.
+
+## Native integration
+
+The adapter starts the official `vllm serve` CLI, with a workspace-owned process
+group, local cached weights, and the vLLM Metal platform plugin. It consumes native
+OpenAI-compatible streaming responses and `/metrics` telemetry. A separate small
+Python helper inventories and downloads safetensors snapshots through Hugging Face.
+No generated shell code or model-provided commands are executed.
+
+One server/model is active per workspace. Switching models or serving settings
+restarts that server. Cancellation, viewer shutdown, parent death and the 15-minute
+idle timeout stop owned processes. Other applications are outside its ownership.
+Each native API route is authenticated and restricted to loopback Host/Origin;
+the transient inference key is never exposed in viewer state or command arguments.
+
+The Mac runtime is pinned to vLLM **0.29.0**, vLLM Metal **0.29.0**, MLX **0.32.1**
+and MLX-LM commit `9e6acca691e64d6d8bb808c328fcdea459099cca`. The official core
+wheel is labeled `0.29.0+cpu`; MetalPlatform selects GPU inference. The doctor tests
+an actual native Metal cache kernel. Do not upgrade MLX independently of the plugin.
+
+## Serving controls
+
+| Setting | Default | Supported range |
+| --- | --- | --- |
+| Context | 4,096 tokens | 512–16,384 |
+| Parallel sequences | 4 | 1–8 |
+| Metal memory budget fraction | 0.1 | 0.05–0.8 |
+| Prefix caching | On | On/off |
+
+The memory fraction uses **Metal's recommended working-set limit**, not physical
+RAM. It covers the plugin's model/cache memory planning. Large models may need a
+larger explicit budget. `VLLM_METAL_MEMORY_FRACTION=auto` makes this pinned release
+honor the native `--gpu-memory-utilization` flag.
+
+`serve` merges supplied settings with the current configuration. Chat preserves
+them. They reset to defaults on viewer restart; every measurement records its actual
+configuration. The ordinary three-run benchmark sets context to 2,048 tokens.
+
+## Measurements
+
+Token rates use native completion-token counts divided by **client end-to-end time**,
+including prompt processing and HTTP. They are not native decode tokens/s and should
+not be ranked against Ollama or MLX-LM decode rates. Streaming chunks are never counted
+as tokens. First-token and first-visible-output times are separate.
+
+Concurrency tests use 1–8 clients and up to 32 total requests, with one discarded
+warmup, a recorded prompt with numbered suffixes, temperature 0, seed 42, thinking off
+when supported and up to 128 generated tokens. Aggregate throughput divides successful
+output tokens by the whole measured batch time. Failures retain individual error
+samples. p95 uses nearest rank; these short tests are exploratory, not capacity claims.
+Repeated prefixes can be cached. Compare identical model, context and cache settings.
+
+Server RSS sums owned processes and may count shared pages more than once; it is
+not Metal allocation or physical VRAM. Server gauges may lag requests. Missing values
+remain unknown. Exports include hardware, runtime/backend versions, revisions and samples.
+
+## Setup and development
+
+Requirements: Apple Silicon, macOS 15+, native arm64 Python 3.12 and Node 22+.
+Dependencies live in this package's `.venv`; model files stay in the standard Hugging
+Face cache. `requirements.lock` pins all dependencies and hashes the official wheels.
+
+```sh
+# From this package directory:
+./toolchain/setup.sh
+./toolchain/doctor.sh
+./viewer.sh
+./toolchain/local-ai status
+./toolchain/local-ai action-json '{"type":"serve","model":"mlx-community/Qwen3-0.6B-4bit","maxSequences":4,"prefixCaching":true}'
+./toolchain/local-ai action-json '{"type":"load_test","model":"mlx-community/Qwen3-0.6B-4bit","concurrency":4,"requests":8}'
+```
+
+The standalone viewer is `http://127.0.0.1:4312/?view=grid`. Native Harness assigns its
+own port. Commands must use that workspace's `HARNESS_WORKSPACE`; discovery verifies
+both workspace and framework. Shared code changes are packaged with
+`npm run build:vllm` from `store/tools/local-ai`. `npm test` runs isolated checks;
+`npm run test:vllm-live` exercises this package workspace with the cached demo model.
+
+This first version qualifies Qwen3 0.6B 4-bit on this M2 Max. Other cached text models
+are candidates; model support depends on the pinned backend. No CUDA parity, remote
+serving, training, multimodal inference or model deletion is claimed.
+
+See [findings and sources](../../../docs/research/local-ai-harnesses/vllm/findings.md),
+[live verification](../../../docs/research/local-ai-harnesses/vllm/verification.json) and
+[raw benchmarks](../../../docs/research/local-ai-harnesses/vllm/benchmarks.json).
+
+## Credit and stewardship
+
+The native framework and official mark belong to [the vLLM and vLLM Metal contributors](https://github.com/vllm-project/vllm-metal).
+OpenHarness contributors maintain this independent integration, its agent instructions
+and viewer. The harness is MIT-licensed; see [LICENSE](LICENSE). Upstream notices
+and license texts are recorded in [THIRD_PARTY.md](THIRD_PARTY.md) and `licenses/`.
+Model weights are downloaded separately under their respective model licenses.
diff --git a/store/agents/vllm/THIRD_PARTY.md b/store/agents/vllm/THIRD_PARTY.md
new file mode 100644
index 000000000..82bcf9c19
--- /dev/null
+++ b/store/agents/vllm/THIRD_PARTY.md
@@ -0,0 +1,10 @@
+# Attribution
+
+This independently maintained OpenHarness integration is MIT-licensed; see LICENSE.
+It is not an official application of the upstream framework. Upstream names and
+marks identify the tools being operated and remain their owners’ marks.
+
+Official logo source and retrieval date are recorded in assets/README.md.
+Upstream framework license texts are included in licenses/. The setup scripts use
+the upstream distributions; Python and runtime dependencies retain their own licenses.
+Model weights are not included and retain their individual model licenses.
diff --git a/store/agents/vllm/agent/AGENTS.md b/store/agents/vllm/agent/AGENTS.md
new file mode 100644
index 000000000..4327cadd7
--- /dev/null
+++ b/store/agents/vllm/agent/AGENTS.md
@@ -0,0 +1,68 @@
+# vLLM model operator
+
+You are the natural-language interface for this vLLM workspace. Its left viewer
+shows live server state and measured performance. Use `$LOCAL_AI` for managed work.
+Start with status, then translate the user's intent into these typed operations:
+
+```sh
+"$LOCAL_AI" status
+"$LOCAL_AI" action deploy mlx-community/Qwen3-0.6B-4bit
+"$LOCAL_AI" action chat mlx-community/Qwen3-0.6B-4bit "Explain unified memory in three sentences."
+"$LOCAL_AI" action-json '{"type":"serve","model":"mlx-community/Qwen3-0.6B-4bit","context":4096,"maxSequences":4,"memoryFraction":0.1,"prefixCaching":true}'
+"$LOCAL_AI" action-json '{"type":"load_test","model":"mlx-community/Qwen3-0.6B-4bit","concurrency":1,"requests":8}'
+"$LOCAL_AI" action-json '{"type":"load_test","model":"mlx-community/Qwen3-0.6B-4bit","concurrency":4,"requests":8}'
+"$LOCAL_AI" action benchmark mlx-community/Qwen3-0.6B-4bit
+"$LOCAL_AI" action unload_all
+"$LOCAL_AI" cancel JOB_ID
+```
+
+This is **one vLLM harness**; **Metal is its backend on this Mac**. The native
+vLLM CLI starts the official API server and scheduler, and vLLM Metal runs MLX and
+Metal paged-attention kernels. `$VLLM --help` provides native CLI documentation.
+Normal operations go through `$LOCAL_AI` so they have progress, cancellation and
+measurements. Do not launch a second unmanaged server or another chat/agent pane.
+
+The package uses isolated Python 3.12 with pinned official vLLM/vLLM Metal 0.29.0
+Mac wheels, MLX 0.32.1 and the compatible MLX-LM commit. Do not independently upgrade
+MLX: its ABI must match the plugin. The core wheel's `+cpu` version suffix does not
+mean CPU inference; the doctor must confirm MetalPlatform and a native GPU kernel.
+If missing, run this package's setup command. Do not modify global Python.
+
+Use Hugging Face `owner/repository` IDs. Default first model:
+`mlx-community/Qwen3-0.6B-4bit`. Start small, verify an actual answer, then report it
+working. Cached text models are candidates, not a claim of universal compatibility.
+No remote code, model deletion, cloud inference, training or multimodal operations.
+
+`serve` merges provided settings with the current in-memory configuration and
+restarts the owned server when needed. Bounds: context 512–16,384; parallel sequences
+1–8; memoryFraction 0.05–0.8; prefixCaching boolean. Defaults: 4,096 / 4 / 0.1 / on.
+The fraction is a budget based on Metal's recommended working-set limit, **not a
+percentage of physical RAM**. Larger weights may need a larger explicit budget.
+Configuration is visible and recorded per measurement, but resets to defaults when
+the viewer restarts. Chat preserves serving settings and caps output at 512 tokens.
+Changing model releases the previous one. Idle servers stop after 15 minutes.
+
+Load tests send 1–8 concurrent client requests, 1–32 total (at least concurrency),
+with 128 output tokens, temperature 0, seed 42, reasoning disabled when supported,
+one discarded warmup, and a numbered variation of the recorded benchmark prompt.
+They report aggregate output tokens / entire measured batch time, median/p95 request
+latency, first token, success/error counts, settings and all samples. p95 uses nearest
+rank; small samples are exploratory. Prefix caching may reuse shared prompt prefixes.
+Keep cache/context/model settings identical when comparing concurrency levels.
+
+The ordinary benchmark uses three warm runs with 2,048-token context and changes the
+server context to that size. All vLLM token rates are **client end-to-end throughput**
+including prompt processing and HTTP, calculated from native token usage, never SSE
+chunks. Do not rank these against MLX-LM native decode tokens/s or Ollama decode rate.
+First token includes reasoning when enabled; visible output is separately timed.
+Small demonstration models are not evidence of answer quality.
+
+RSS sums owned server processes and may count shared pages more than once. It is not
+GPU allocation or an independent VRAM pool. KV cache usage is server telemetry, which
+can lag the request stream. Missing measurements are unknown. Preserve raw failures.
+
+The inference API is a private authenticated loopback service with every route
+guarded; never expose its key. Cancellation stops this workspace's process group and
+retains downloads. Other apps and servers are outside its ownership. Parent death
+also stops the owned server. `.harness` stores state, logs, endpoint and measurements;
+weights stay in the shared Hugging Face cache. Never edit data to conceal a failure.
diff --git a/store/agents/vllm/assets/README.md b/store/agents/vllm/assets/README.md
new file mode 100644
index 000000000..ab27be24c
--- /dev/null
+++ b/store/agents/vllm/assets/README.md
@@ -0,0 +1,7 @@
+# vLLM identity
+
+`vllm-icon.png` is the official vLLM mark from
+https://github.com/vllm-project/vllm/blob/main/docs/assets/logos/vllm-logo-only-light.png
+retrieved September 20, 2026. Used to identify the locally installed framework;
+this harness is an independent integration. The Metal backend is part of the vLLM
+harness and does not have a separate app identity.
diff --git a/store/agents/vllm/assets/vllm-icon.png b/store/agents/vllm/assets/vllm-icon.png
new file mode 100644
index 000000000..7aaf17487
Binary files /dev/null and b/store/agents/vllm/assets/vllm-icon.png differ
diff --git a/store/agents/vllm/bin/harness.mjs b/store/agents/vllm/bin/harness.mjs
new file mode 100755
index 000000000..16b63570d
--- /dev/null
+++ b/store/agents/vllm/bin/harness.mjs
@@ -0,0 +1,54 @@
+#!/usr/bin/env node
+import { readFile } from 'node:fs/promises';
+import { resolve, join, dirname } from 'node:path';
+import { fileURLToPath } from 'node:url';
+const args = process.argv.slice(2);
+const runtimeName = process.env.LOCAL_AI_NAME || 'Ollama';
+const runtime = process.env.LOCAL_AI_RUNTIME || 'ollama';
+if (!args.length) { console.log(`Talk to the ${runtimeName} agent, or use:\n local-ai "run a lightweight model"\n local-ai status\n local-ai action deploy MODEL\n local-ai action benchmark MODEL [MODEL…]\n local-ai action chat MODEL "PROMPT"\n local-ai action unload MODEL\n local-ai action-json '{"type":"serve","model":"OWNER/REPO","maxSequences":4}' (vLLM)\n local-ai cancel JOB_ID`); process.exit(0); }
+const packageRoot = dirname(dirname(fileURLToPath(import.meta.url)));
+const workspace = resolve(process.env.HARNESS_WORKSPACE || packageRoot);
+let port = Number(process.env.HARNESS_PORT || ({ 'mlx-lm': 4311, vllm: 4312 }[runtime] || 4310));
+try { const endpoint = JSON.parse(await readFile(join(workspace, '.harness', 'endpoint.json'), 'utf8')); if (Number.isInteger(endpoint.port) && endpoint.port > 0 && endpoint.port <= 65535 && endpoint.workspace === workspace) port = endpoint.port; } catch {}
+const base = `http://127.0.0.1:${port}`;
+const get = async path => { const response = await fetch(`${base}${path}`, { signal: AbortSignal.timeout(15000) }); const value = await response.json(); if (!response.ok) throw new Error(value.error || `HTTP ${response.status}`); return value; };
+let currentJob;
+try {
+ const health = await get('/api/health');
+ if (health.application !== 'local-ai-harnesses' || health.workspace !== workspace || (health.runtime || 'ollama') !== runtime) throw new Error(`This endpoint belongs to another workspace or framework. Open the ${runtimeName} viewer for this workspace.`);
+ const state = await get('/api/state');
+ const post = async (path, body) => {
+ const response = await fetch(`${base}${path}`, { method: 'POST', headers: { 'Content-Type': 'application/json', 'X-Harness-Token': state.token }, body: JSON.stringify(body), signal: AbortSignal.timeout(15000) });
+ const value = await response.json(); if (!response.ok) throw new Error(value.error || `HTTP ${response.status}`); return value;
+ };
+ if (args[0] === 'status' && args.length === 1) {
+ const { token, ...publicState } = state; console.log(JSON.stringify(publicState, null, 2)); process.exit(0);
+ }
+ if (args[0] === 'cancel') { const { job } = await post('/api/cancel', { id: args[1] }); console.log(JSON.stringify({ id: job.id, status: job.status, message: job.message }, null, 2)); process.exit(0); }
+ let result;
+ if (args[0] === 'action-json') {
+ if (args.length !== 2) throw new Error('Pass one JSON action object.');
+ result = await post('/api/action', JSON.parse(args[1]));
+ } else if (args[0] === 'action') {
+ const type = args[1];
+ const body = type === 'benchmark' ? { type, models: args.slice(2) } : type === 'chat' ? { type, model: args[2], prompt: args.slice(3).join(' ') } : { type, model: args[2] };
+ result = await post('/api/action', body);
+ } else result = await post('/api/command', { text: args.join(' ') });
+ if (!result.job) { console.log(result.message || JSON.stringify(result, null, 2)); if (result.models) console.log(result.models.map(m => `${m.id}: ${m.description}`).join('\n')); process.exit(0); }
+ currentJob = result.job.id;
+ console.log(`Job ${currentJob}`);
+ process.once('SIGINT', async () => { try { await post('/api/cancel', { id: currentJob }); console.log('Cancellation requested.'); } finally { process.exit(130); } });
+ let last;
+ while (true) {
+ const { jobs } = await get('/api/jobs');
+ const job = jobs.find(j => j.id === currentJob);
+ if (!job) throw new Error('The job could not be found. Check the viewer.');
+ if (job.message !== last) { console.log(job.message); last = job.message; }
+ if (!['queued', 'running'].includes(job.status)) {
+ if (job.output) console.log(job.output);
+ if (job.result?.metrics) console.log(JSON.stringify(job.result.metrics, null, 2));
+ process.exit(job.status === 'succeeded' ? 0 : 1);
+ }
+ await new Promise(resolve => setTimeout(resolve, 1000));
+ }
+} catch (error) { console.error(error.cause?.code === 'ECONNREFUSED' ? `The viewer is not running. Open the ${runtimeName} harness viewer.` : `${error.message}${currentJob ? ` (job ${currentJob}; check its status before retrying)` : ''}`); process.exitCode = 1; }
diff --git a/store/agents/vllm/cache.py b/store/agents/vllm/cache.py
new file mode 100644
index 000000000..ebdb8c8c3
--- /dev/null
+++ b/store/agents/vllm/cache.py
@@ -0,0 +1,160 @@
+"""Private metadata/download worker; inference belongs to the native vLLM server."""
+import importlib.metadata
+import json
+import os
+from pathlib import Path
+import re
+import shutil
+import sys
+import uuid
+
+protocol = sys.stdout
+sys.stdout = sys.stderr
+os.environ.setdefault("HF_HUB_DISABLE_TELEMETRY", "1")
+from huggingface_hub import HfApi, hf_hub_download
+from huggingface_hub.constants import HF_HUB_CACHE
+import psutil
+
+root = Path(HF_HUB_CACHE)
+registry = Path(os.environ["HARNESS_MODEL_REGISTRY"])
+try:
+ revisions = json.loads(registry.read_text())
+ if not isinstance(revisions, dict):
+ revisions = {}
+except (OSError, ValueError):
+ revisions = {}
+
+
+def emit(event):
+ protocol.write(json.dumps(event, ensure_ascii=False, allow_nan=False) + "\n")
+ protocol.flush()
+
+
+def validate(repo):
+ if not isinstance(repo, str) or len(repo) > 180 or not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]*/[A-Za-z0-9][A-Za-z0-9_.-]*", repo) or ".." in repo:
+ raise ValueError("Use a Hugging Face owner/repository model ID.")
+ return repo
+
+
+def snapshot(repo):
+ base = root / ("models--" + validate(repo).replace("/", "--"))
+ candidates = [revisions.get(repo, "")]
+ try:
+ candidates.append((base / "refs/main").read_text().strip())
+ except OSError:
+ pass
+ for revision in candidates:
+ if isinstance(revision, str) and re.fullmatch(r"[a-f0-9]{40,64}", revision):
+ path = base / "snapshots" / revision
+ if path.is_dir():
+ return path
+ paths = sorted((base / "snapshots").glob("*"), key=lambda p: p.stat().st_mtime, reverse=True)
+ if not paths:
+ raise ValueError(f"{repo} is not downloaded.")
+ return paths[0]
+
+
+def describe(repo):
+ path = snapshot(repo)
+ config = json.loads((path / "config.json").read_text())
+ if "vision_config" in config or "diffusion" in str(config.get("model_type", "")):
+ raise ValueError("This harness currently supports text language models.")
+ weights = list(path.glob("*.safetensors"))
+ if not weights or not (path / "tokenizer_config.json").is_file():
+ raise ValueError("Incomplete language-model snapshot. Download to resume.")
+ index = path / "model.safetensors.index.json"
+ if index.exists():
+ expected = set(json.loads(index.read_text())["weight_map"].values())
+ if any(Path(name).name != name or not (path / name).is_file() for name in expected):
+ raise ValueError("Incomplete model shards. Download to resume.")
+ quant = config.get("quantization") or config.get("quantization_config") or {}
+ template = json.loads((path / "tokenizer_config.json").read_text()).get("chat_template", "")
+ return {"id": repo, "name": repo.split("/")[-1], "path": str(path), "digest": path.name,
+ "size": sum(p.stat().st_size for p in path.iterdir() if p.is_file()),
+ "family": config.get("model_type"), "parameters": None,
+ "quantization": f"{quant['bits']}-bit" if quant.get("bits") else config.get("torch_dtype", "Unquantized"),
+ "supportsThinking": "enable_thinking" in str(template), "running": False, "resident": None}
+
+
+def inventory():
+ repos = set(revisions)
+ for base in root.glob("models--*"):
+ parts = base.name.split("--", 2)
+ if len(parts) == 3:
+ repos.add(parts[1] + "/" + parts[2])
+ models = []
+ for repo in sorted(repos):
+ try:
+ models.append(describe(repo))
+ except (OSError, ValueError, KeyError):
+ pass
+ return {"models": models}
+
+
+def download(repo, request_id, budget):
+ info = HfApi().model_info(repo, files_metadata=True)
+ suffixes = {".json", ".safetensors", ".model", ".tiktoken", ".txt", ".jinja"}
+ files = [f for f in info.siblings if Path(f.rfilename).suffix in suffixes]
+ if not any(f.rfilename.endswith(".safetensors") for f in files):
+ raise ValueError("Choose a text model with safetensors weights.")
+ if any(f.size is None or f.size < 0 or Path(f.rfilename).is_absolute() or ".." in Path(f.rfilename).parts for f in files):
+ raise ValueError("Invalid model file metadata.")
+ total = sum(f.size for f in files)
+ if total > budget:
+ raise ValueError("The model exceeds this harness's 80%-of-RAM download budget.")
+ root.mkdir(parents=True, exist_ok=True)
+ if shutil.disk_usage(root).free < total + 2 * 1024**3:
+ raise ValueError("Insufficient disk space; reserve the model size plus 2 GiB.")
+ complete = 0
+ for i, file in enumerate(files):
+ emit({"id": request_id, "event": "progress", "message": f"Downloading {file.rfilename} · {i+1}/{len(files)} files", "percent": int(complete / max(total, 1) * 100), "completed": complete, "total": total})
+ hf_hub_download(repo, file.rfilename, revision=info.sha, cache_dir=str(root))
+ complete += file.size
+ revisions[repo] = info.sha
+ registry.parent.mkdir(parents=True, exist_ok=True)
+ temporary = registry.with_suffix(f".{uuid.uuid4().hex}.tmp")
+ temporary.write_text(json.dumps(revisions))
+ temporary.replace(registry)
+ return describe(repo)
+
+
+def memory(pid):
+ try:
+ parent = psutil.Process(pid)
+ if parent.ppid() != os.getppid():
+ return None # Only inspect a sibling server owned by this controller.
+ rss = 0
+ for process in [parent] + parent.children(recursive=True):
+ try:
+ rss += process.memory_info().rss
+ except (psutil.NoSuchProcess, psutil.AccessDenied):
+ pass
+ return rss
+ except (psutil.NoSuchProcess, psutil.AccessDenied):
+ return None
+
+
+versions = {name: importlib.metadata.version(name) for name in ["vllm", "vllm-metal", "mlx", "mlx-lm"]}
+emit({"event": "ready", "versions": versions, "cacheRoot": str(root), **inventory()})
+for line in sys.stdin:
+ request_id = None
+ try:
+ if len(line) > 100000:
+ raise ValueError("Request too large.")
+ message = json.loads(line)
+ request_id = message["id"]
+ action = message["action"]
+ repo = validate(message["model"]) if "model" in message else None
+ if action == "inventory":
+ result = inventory()
+ elif action == "details":
+ result = describe(repo)
+ elif action == "download":
+ result = download(repo, request_id, int(message["memoryBudget"]))
+ elif action == "memory":
+ result = {"rssBytes": memory(int(message["pid"]))}
+ else:
+ raise ValueError("Unknown cache action.")
+ emit({"id": request_id, "event": "done", "result": result, "state": inventory()})
+ except Exception as error:
+ emit({"id": request_id, "event": "error", "message": str(error)[:1500]})
diff --git a/store/agents/vllm/dist/app.js b/store/agents/vllm/dist/app.js
new file mode 100644
index 000000000..37487b249
--- /dev/null
+++ b/store/agents/vllm/dist/app.js
@@ -0,0 +1,242 @@
+const $ = selector => document.querySelector(selector);
+const escapeHTML = value => String(value ?? '').replace(/[&<>"']/g, c => ({ '&': '&amp;', '<': '&lt;', '>': '&gt;', '"': '&quot;', "'": '&#39;' }[c]));
+const gib = bytes => Number.isFinite(bytes) ? (bytes / 1024 ** 3).toFixed(1) : '—';
+const number = (value, digits = 1) => Number.isFinite(value) ? value.toFixed(digits) : '—';
+const active = job => ['queued', 'running'].includes(job.status);
+let state;
+let selectedModel = sessionStorage.getItem('harness-model') || '';
+let pending = false;
+let modelSignature = '';
+let benchmarkSignature = '';
+let initialized = false;
+let lastReceived = Date.now();
+const jobElements = new Map();
+const jobSignatures = new Map();
+const chatHistory = new Map();
+const chatRequests = new Map();
+const completed = new Set();
+const embedded = new URLSearchParams(location.search).get('view') === 'grid';
+document.body.classList.toggle('embedded-viewer', embedded);
+
+function toast(text) { const el = $('#toast'); el.textContent = text; el.hidden = false; clearTimeout(toast.timer); toast.timer = setTimeout(() => { el.hidden = true; }, 4500); }
+function scrollConversation() { const el = $('#conversation'); el.scrollTop = el.scrollHeight; }
+function appendMessage(text, user = false) {
+ const el = document.createElement('div'); el.className = user ? 'user-message' : 'assistant-message';
+ if (user) el.textContent = text;
+ else el.innerHTML = `<span class="message-label">vLLM</span><p class="message-text">${escapeHTML(text)}</p>`;
+ $('#conversation').append(el); scrollConversation(); return el;
+}
+async function api(path, body) {
+ if (!state?.token) await refresh();
+ const response = await fetch(path, { method: 'POST', headers: { 'Content-Type': 'application/json', 'X-Harness-Token': state?.token || '' }, body: JSON.stringify(body) });
+ const result = await response.json();
+ if (!response.ok) { if (response.status === 403) void refresh(); throw new Error(result.error || 'The request failed.'); }
+ return result;
+}
+async function refresh() {
+ try {
+ const response = await fetch('/api/state'); if (!response.ok) throw new Error('Could not reach the vLLM viewer');
+ applyState(await response.json());
+ } catch (error) { $('#connection-status').textContent = 'vLLM viewer disconnected'; $('#operator-status').textContent = 'Reconnect to send a request'; return false; }
+ return true;
+}
+function applyState(next) {
+ lastReceived = Date.now(); state = { ...state, ...next };
+ if (selectedModel && !state.runtime.models.some(m => m.id === selectedModel)) selectedModel = '';
+ if (!selectedModel) selectedModel = state.runtime.models.find(m => m.running)?.id || '';
+ render();
+ if (!initialized) {
+ [...state.jobs].slice(0, 4).reverse().forEach(job => showJob(job, true)); initialized = true;
+ }
+ for (const job of state.jobs) if (jobElements.has(job.id) || active(job)) showJob(job);
+ renderActiveJob();
+ renderActivity();
+}
+function selectModel(model) {
+ selectedModel = model; sessionStorage.setItem('harness-model', model);
+ $('#selected-model').value = model;
+ $('#operator-status').textContent = model || 'vLLM harness';
+ document.querySelectorAll('[data-model]').forEach(el => el.classList.toggle('selected', el.dataset.model === model));
+}
+function render() {
+ const { runtime, system, benchmarks, runs } = state;
+ const models = runtime.models || [];
+ const loaded = models.filter(m => m.running);
+ const lastJob = state.jobs[0];
+ const notice = $('#runtime-notice');
+ notice.hidden = lastJob?.status !== 'failed';
+ if (!notice.hidden) {
+ const affected = lastJob.activeModel || lastJob.model || lastJob.models?.join(', ') || 'vLLM';
+ const explanation = /tensor.*size overflow/i.test(lastJob.message)
+ ? "vLLM could not read this model’s weights. Other models can still run."
+ : 'The last operation failed. Check the details before retrying.';
+ notice.innerHTML = `<strong>${escapeHTML(affected)} · ${escapeHTML(lastJob.type)} failed</strong><p>${escapeHTML(explanation)}</p><details><summary>Error details</summary><pre>${escapeHTML(lastJob.message)}</pre></details>`;
+ }
+ $('#connection-status').textContent = runtime.online ? 'vLLM connected' : 'vLLM offline';
+ $('#model-count').textContent = runtime.online ? models.length : '—';
+ $('#models-total').textContent = models.length;
+ $('#running-count').textContent = runtime.online ? loaded.length : '—';
+ $('#model-memory').innerHTML = `${runtime.online ? gib(loaded.reduce((sum, m) => sum + (m.resident?.size || 0), 0)) : '—'} <small>GiB</small>`;
+ const recent = runs[0];
+ $('#last-speed').innerHTML = `${number(recent?.tokensPerSecond)} <small>tok/s</small>`;
+ $('#last-speed').title = recent ? `${recent.model} · ${new Date(recent.at).toLocaleString()}` : 'A chat response will appear here';
+ $('#map-status').textContent = runtime.online ? 'LIVE CONNECTION' : 'READY TO START';
+ $('#map-status').classList.toggle('online', runtime.online);
+ $('#machine-info').textContent = `${system.chip} · ${Math.round(system.totalMemory / 1024 ** 3)} GiB`;
+ const signature = JSON.stringify({ online: runtime.online, version: runtime.version, models, selectedModel });
+ if (signature !== modelSignature) {
+ modelSignature = signature;
+ $('#runtime-map').innerHTML = `<div class="runtime-node ${runtime.online ? '' : 'offline'}"><img class="node-icon" src="/assets/vllm-icon.png" width="42" height="42" alt=""><strong>vLLM</strong><small>${runtime.online ? `v${escapeHTML(runtime.version)}` : 'Not running'}</small></div><div class="model-nodes">${models.length ? models.map(m => `<button class="model-node ${m.running ? 'active' : ''} ${selectedModel === m.id ? 'selected' : ''}" data-model="${escapeHTML(m.id)}"><strong>${escapeHTML(m.name)}</strong><span>${m.running ? 'In memory' : 'On disk'} · ${gib(m.size)} GiB</span></button>`).join('') : `<div class="map-empty empty-node"><strong>Your first vLLM model starts here.</strong>Ask the operator to run your first model.<br>Its connection will appear here.</div>`}</div>`;
+ $('#models-list').innerHTML = models.length ? models.map(m => `<button class="model-row ${selectedModel === m.id ? 'selected' : ''}" data-model="${escapeHTML(m.id)}"><span class="model-title"><strong>${escapeHTML(m.name)}</strong><small>${escapeHTML([m.family, m.parameters, m.quantization].filter(Boolean).join(' · '))}</small></span><span class="model-status ${m.running ? 'loaded' : ''}">${m.running ? 'In memory' : 'On disk'}</span><span class="model-numeric">${gib(m.size)} GiB</span><span class="model-numeric">vLLM</span></button>`).join('') : `<div class="empty-small"><p>${runtime.online ? 'No models downloaded yet.' : 'Start vLLM to discover your models.'}</p><span>Say “run a lightweight model” to get started.</span></div>`;
+ $('#selected-model').innerHTML = `<option value="">Auto · running model</option>${models.map(m => `<option value="${escapeHTML(m.id)}">${escapeHTML(m.name)}</option>`).join('')}`;
+ $('#selected-model').value = selectedModel;
+ $('#operator-status').textContent = selectedModel || 'vLLM harness';
+ }
+ const benchSignature = JSON.stringify(benchmarks.map(b => b.id));
+ if (benchSignature !== benchmarkSignature) { benchmarkSignature = benchSignature; renderBenchmarks(); }
+ $('#export-benchmarks').hidden = !benchmarks.length && !state.loadTests?.length; renderServing(); renderLoadTests();
+}
+function renderBenchmarks() {
+ const latest = [...new Map([...state.benchmarks].reverse().map(b => [b.model, b])).values()].sort((a, b) => b.medianTokensPerSecond - a.medianTokensPerSecond);
+ if (!latest.length) return;
+ const maxSpeed = Math.max(...latest.map(b => b.medianTokensPerSecond), 1);
+ const maxLatency = Math.max(...latest.map(b => b.medianFirstTokenMs), 1);
+ const bars = (key, max, unit) => latest.map(b => `<div class="chart-row"><span>${escapeHTML(b.model.split("/").at(-1))}</span><div class="bar-track"><div class="bar-fill ${key === 'medianFirstTokenMs' ? 'latency' : ''}" style="width:${Math.max(0, b[key] / max * 100)}%"></div></div><span class="bar-value">${number(b[key], key === 'medianFirstTokenMs' ? 0 : 1)} <small>${unit}</small></span></div>`).join('');
+ $('#performance-content').innerHTML = `<div class="chart-heading"><h3>Request throughput</h3><span>tokens/s · higher is faster</span></div><div class="chart-bars" role="group" aria-label="Measured median end-to-end request throughput">${bars('medianTokensPerSecond', maxSpeed, '')}</div><div class="chart-heading second-chart"><h3>Time to first token</h3><span>milliseconds · lower is faster</span></div><div class="chart-bars" role="group" aria-label="Measured median time to first token">${bars('medianFirstTokenMs', maxLatency, '')}</div><p class="benchmark-meta">Latest benchmark per model · median of 3 warm runs<br>Same prompt · 2,048-token context · up to 128 generated tokens · seed 42<br>Thinking off when supported; actual mode is recorded per run.<br>Repeated prompts can use the prefix cache. Token throughput includes prompt processing and HTTP; it is not native decode speed or answer quality.</p><details class="benchmark-details"><summary>Samples and measurement details</summary>${latest.map(b => `<div class="sample-record"><strong>${escapeHTML(b.model)}</strong><span>${new Date(b.at).toLocaleString()} · vLLM ${escapeHTML(b.runtime.version)}</span><span>Speed: ${b.samples.map(s => number(s.tokensPerSecond)).join(' / ')} tokens/s</span><span>First token: ${b.samples.map(s => number(s.firstTokenMs, 0)).join(' / ')} ms</span><span>Tokens generated: ${b.samples.map(s => s.tokens).join(' / ')} · thinking: ${escapeHTML(String(b.samples[0].thinking ?? 'not reported'))} · server RSS: ${gib(b.residentBytes)} GiB</span></div>`).join('')}</details>`;
+}
+function showJob(job, historical = false) {
+ let el = jobElements.get(job.id);
+ if (!el) {
+ el = document.createElement('div'); el.className = 'assistant-message job-message';
+ jobElements.set(job.id, el); $('#conversation').append(el);
+ }
+ const signature = JSON.stringify(job);
+ if (jobSignatures.get(job.id) === signature) return;
+ jobSignatures.set(job.id, signature);
+ const failed = ['failed', 'cancelled', 'interrupted'].includes(job.status);
+ const metrics = job.result?.metrics;
+ const label = job.type === 'chat' ? job.model : 'vLLM';
+ el.innerHTML = `<span class="message-label">${escapeHTML(label)} <span class="job-state ${failed ? 'error-text' : ''}">${escapeHTML(job.status)}</span></span><p class="message-text ${failed ? 'error-text' : ''}">${escapeHTML(job.message)}</p>${job.output ? `<div class="stream-output message-text">${escapeHTML(job.output)}</div>` : ''}${metrics ? `<p class="run-metrics">${number(metrics.tokensPerSecond)} tok/s · ${number(metrics.firstTokenMs, 0)} ms first token · ${metrics.tokens ?? '—'} tokens${metrics.thinking ? ` · ${number(metrics.firstOutputMs, 0)} ms to visible output · reasoning on` : ''}${metrics.peakMemoryBytes ? ` · ${gib(metrics.peakMemoryBytes)} GiB peak allocation` : ''}${metrics.doneReason === 'length' ? ' · output limit reached' : ''}</p>` : ''}${job.persistenceError ? '<p class="error-text">Could not save this result to disk.</p>' : ''}${job.status === 'succeeded' && job.type === 'deploy' ? `<div class="actions"><button data-prompt="Benchmark ${escapeHTML(job.model)}">Benchmark it</button><button data-chat="${escapeHTML(job.model)}">Ask a question</button></div>` : ''}${job.status === 'failed' ? '<p class="secondary">You can retry the request after addressing the error.</p>' : ''}`;
+ if (job.status === 'succeeded' && !completed.has(job.id)) {
+ completed.add(job.id);
+ if (!historical && ['deploy', 'serve', 'chat', 'benchmark', 'load_test'].includes(job.type) && job.result?.model) selectModel(job.result.model);
+ const request = chatRequests.get(job.id);
+ if (request) {
+ const history = chatHistory.get(job.model) || [];
+ history.push({ role: 'user', content: request }, { role: 'assistant', content: job.output });
+ chatHistory.set(job.model, history.slice(-6).map(m => ({ ...m, content: m.content.slice(0, 6000) }))); chatRequests.delete(job.id);
+ }
+ }
+ if (!historical) scrollConversation();
+}
+function renderActiveJob() {
+ const running = state.jobs.find(j => j.status === 'running') || state.jobs.find(j => j.status === 'queued');
+ const panel = $('#job-panel'); panel.hidden = !running;
+ if (!running) return;
+ const queued = state.jobs.filter(j => j.status === 'queued').length;
+ panel.innerHTML = `<div class="job-top"><strong>${escapeHTML(running.type === 'chat' ? 'Generating' : running.type.replaceAll('_', ' '))}</strong><button data-cancel="${escapeHTML(running.id)}">Cancel</button></div><div class="progress-track" role="progressbar" aria-label="Job progress" aria-valuemin="0" aria-valuemax="100" aria-valuenow="${Math.round(running.progress)}"><div class="progress-fill" style="width:${running.progress}%"></div></div><p>${escapeHTML(running.message)}${queued ? ` · ${queued} queued` : ''}</p>`;
+}
+function receiveJob(job) {
+ if (!state) return;
+ state.jobs = [job, ...state.jobs.filter(j => j.id !== job.id)];
+ showJob(job); renderActiveJob(); renderActivity();
+}
+function renderActivity() {
+ const jobs = state.jobs.slice(0, 6);
+ $('#activity-content').innerHTML = jobs.length ? jobs.map(job => `<div class="activity-row"><span class="activity-state ${job.status === 'failed' ? 'error-text' : ''}">${escapeHTML(job.status)}</span><div><strong>${escapeHTML(job.model || job.models.join(', ') || 'vLLM')}</strong><p>${escapeHTML(job.message)}</p></div>${active(job) ? `<button data-cancel="${escapeHTML(job.id)}">Cancel</button>` : '<span class="activity-time">' + new Date(job.completedAt || job.createdAt).toLocaleTimeString([], { hour: '2-digit', minute: '2-digit' }) + '</span>'}</div>`).join('') : '<p class="secondary">Ask the agent to run a model. Its progress will appear here.</p>';
+}
+async function sendCommand(text) {
+ if (pending || !text.trim()) return;
+ pending = true; $('#send-button').disabled = true; $('#suggestions').hidden = true;
+ appendMessage(text, true); $('#command-input').value = '';
+ try {
+ const result = await api('/api/command', { text, selectedModel: selectedModel || undefined, history: chatHistory.get(selectedModel) || [] });
+ if (result.job) {
+ if (result.job.type === 'chat') { const prompt = text.match(/^(?:ask|tell|chat with)\s+.+?\s*:\s*([\s\S]+)$/i)?.[1] || text; chatRequests.set(result.job.id, prompt); }
+ receiveJob(result.job);
+ } else {
+ const el = appendMessage(result.message || 'Choose a model to get started.');
+ if (result.models) el.insertAdjacentHTML('beforeend', result.models.map(m => `<div class="catalog-card"><strong>${escapeHTML(m.name)}</strong><p>${escapeHTML(m.description)}</p><p class="secondary">About ${number(m.downloadGB, 2)} GB to download</p><button data-action="deploy" data-target="${escapeHTML(m.id)}">Run this model</button></div>`).join(''));
+ }
+ } catch (error) { appendMessage(error.message).classList.add('error-text'); }
+ finally { pending = false; $('#send-button').disabled = false; scrollConversation(); }
+}
+async function runAction(type, model) {
+ $('#model-dialog').close();
+ const action = type === 'benchmark' ? { type, models: [model] } : { type, model };
+ appendMessage(`${type === 'deploy' ? 'Run' : type[0].toUpperCase() + type.slice(1)} ${model}`, true);
+ try { const result = await api('/api/action', action); receiveJob(result.job); return { jobId: result.job.id, status: result.job.status }; }
+ catch (error) { appendMessage(error.message).classList.add('error-text'); throw error; }
+}
+function inspectModel(id) {
+ const model = state.runtime.models.find(m => m.id === id); if (!model) return;
+ selectModel(id);
+ const resident = model.resident;
+ $('#model-detail').innerHTML = `<p class="eyebrow">vLLM / MODEL DETAILS</p><h2 id="detail-title">${escapeHTML(model.name)}</h2><p class="secondary">${model.running ? 'Loaded and ready for inference' : 'Downloaded and available to load'}</p><div class="detail-grid"><div><span>Model parameters</span><strong>${escapeHTML(model.parameters || 'Unknown')}</strong></div><div><span>Quantization</span><strong>${escapeHTML(model.quantization || 'Unknown')}</strong></div><div><span>Size on disk</span><strong>${gib(model.size)} GiB</strong></div><div><span>Server process RSS</span><strong>${resident ? `${gib(resident.size)} GiB` : 'Not loaded'}</strong></div><div><span>Context window</span><strong>${resident?.context_length?.toLocaleString() || 'Set when loaded'}</strong></div><div><span>Idle expiry</span><strong>${resident?.expires_at ? new Date(resident.expires_at).toLocaleTimeString() : '—'}</strong></div></div><p class="secondary">RSS sums this workspace’s server processes and may count shared pages more than once. It is not GPU allocation. On Apple Silicon, CPU and GPU share unified memory; this is not a separate physical VRAM pool.</p><div class="actions"><button class="primary" data-action="deploy" data-target="${escapeHTML(id)}">${model.running ? 'Keep ready for 15m' : 'Load model'}</button><button data-action="benchmark" data-target="${escapeHTML(id)}">Benchmark</button><button data-action="load_test" data-target="${escapeHTML(id)}">Test 4 concurrent</button><button data-chat="${escapeHTML(id)}">Chat</button>${model.running ? `<button data-action="unload" data-target="${escapeHTML(id)}">Unload</button>` : ''}</div>`;
+ $('#model-dialog').showModal();
+}
+
+$('#refresh').addEventListener('click', async () => { if (await refresh()) toast('Model status refreshed'); });
+$('#selected-model').addEventListener('change', event => selectModel(event.target.value));
+$('#command-form').addEventListener('submit', event => { event.preventDefault(); void sendCommand($('#command-input').value); });
+$('#command-input').addEventListener('keydown', event => { if (event.key === 'Enter' && !event.shiftKey && !event.isComposing) { event.preventDefault(); $('#command-form').requestSubmit(); } });
+$('.dialog-close').addEventListener('click', () => $('#model-dialog').close());
+document.addEventListener('click', event => {
+ const button = event.target.closest('button'); if (!button) return;
+ if (button.dataset.prompt) void sendCommand(button.dataset.prompt);
+ if (button.dataset.model) inspectModel(button.dataset.model);
+ if (button.dataset.action) void runAction(button.dataset.action, button.dataset.target).catch(() => {});
+ if (button.dataset.chat) { selectModel(button.dataset.chat); $('#model-dialog').close(); $('#command-input').value = 'Ask it: '; $('#command-input').focus(); }
+ if (button.dataset.cancel) void api('/api/cancel', { id: button.dataset.cancel }).then(result => receiveJob(result.job)).catch(error => toast(error.message));
+});
+
+function renderServing() {
+ const serving = state.runtime.serving || {};
+ $('#serving-state').textContent = `${serving.status || 'idle'} · Metal ${state.runtime.backendVersion || ''}`;
+ const item = (label, value) => `<div class="serving-stat"><span>${escapeHTML(label)}</span><strong>${escapeHTML(value)}</strong></div>`;
+ $('#serving-content').innerHTML = `<div class="serving-grid">${item('Context', serving.context?.toLocaleString() || '—')}${item('Parallel sequences', serving.maxSequences ?? '—')}${item('Metal memory budget', Number.isFinite(serving.memoryFraction) ? `${Math.round(serving.memoryFraction * 100)}%` : '—')}${item('Prefix cache', serving.prefixCaching === undefined ? '—' : serving.prefixCaching ? 'On' : 'Off')}${item('Requests running / waiting', `${serving.runningRequests ?? '—'} / ${serving.waitingRequests ?? '—'}`)}${item('KV cache used', Number.isFinite(serving.cacheUsage) ? `${(serving.cacheUsage * 100).toFixed(1)}%` : '—')}</div><p class="benchmark-meta">Ask the agent to change these settings. The memory budget uses Metal’s recommended working-set limit. Server telemetry can lag active requests.</p>${serving.model ? `<div class="actions"><button data-load-concurrency="1" data-target="${escapeHTML(serving.model)}">Measure 1 request at a time</button><button data-load-concurrency="4" data-target="${escapeHTML(serving.model)}">Measure 4 concurrent requests</button></div>` : ''}`;
+ $('#model-memory').innerHTML = `${Number.isFinite(serving.processRssBytes) ? gib(serving.processRssBytes) : '—'} <small>GiB</small>`;
+ $('#model-memory').title = 'Summed server process RSS. Shared pages may be counted more than once; this is not GPU allocation.';
+ $('#last-speed').title += ' · client end-to-end throughput';
+}
+let loadTestSignature = '';
+function renderLoadTests() {
+ const results = (state.loadTests || []).slice(0, 6);
+ const signature = JSON.stringify(results.map(test => test.id));
+ if (signature === loadTestSignature) return;
+ loadTestSignature = signature;
+ if (!results.length) {
+ $('#concurrency-content').innerHTML = '<div class="empty-small"><span class="empty-icon" aria-hidden="true">⇉</span><p>See what happens when requests overlap.</p><span>Run a model, then ask “test 4 concurrent requests”. Real throughput, latency and errors appear here.</span></div>';
+ return;
+ }
+ const max = Math.max(1, ...results.map(test => test.aggregateTokensPerSecond));
+ $('#concurrency-content').innerHTML = results.map(test => `<article class="load-result"><div class="load-title"><div><span class="concurrency-badge">${test.concurrency}×</span><strong>${escapeHTML(test.model.split('/').at(-1))}</strong></div><span class="secondary">${new Date(test.at).toLocaleTimeString([], { hour: '2-digit', minute: '2-digit' })}</span></div><div class="load-bar"><div class="bar-track"><div class="bar-fill" style="width:${Math.max(0, test.aggregateTokensPerSecond / max * 100)}%"></div></div><strong>${number(test.aggregateTokensPerSecond)} <small>tok/s</small></strong></div><div class="load-metrics"><span><b>${number(test.medianLatencyMs, 0)}</b> ms median</span><span><b>${number(test.p95LatencyMs, 0)}</b> ms p95</span><span><b>${test.succeeded}/${test.requests}</b> completed</span><span class="${test.failed ? 'error-text' : ''}"><b>${test.failed}</b> errors</span></div><details class="load-details"><summary>Settings &amp; samples</summary><p>${test.serving.context.toLocaleString()} context · ${test.serving.maxSequences} parallel sequences · ${Math.round(test.serving.memoryFraction * 100)}% Metal budget · prefix cache ${test.serving.prefixCaching ? 'on' : 'off'}<br>${escapeHTML(test.mode)}<br>${new Date(test.at).toLocaleString()} · vLLM ${escapeHTML(test.runtime.version)} · Metal ${escapeHTML(test.runtime.backendVersion)}</p>${test.samples.map(sample => `<p>Request ${sample.index + 1}: ${sample.status === 'succeeded' ? `${sample.tokens} tokens · ${number(sample.totalMs, 0)} ms · ${number(sample.firstTokenMs, 0)} ms first token` : escapeHTML(sample.error)}</p>`).join('')}</details></article>`).join('') + '<p class="benchmark-meta">Aggregate output tokens ÷ measured batch time. Latency includes queueing, prompt processing and HTTP. Compare identical model and serving settings; shared prefixes may be cached. These small samples are exploratory. Raw samples are included in Export results.</p>';
+}
+document.addEventListener('click', event => {
+ const button = event.target.closest('button[data-load-concurrency]');
+ if (!button) return;
+ void api('/api/action', { type: 'load_test', model: button.dataset.target, concurrency: Number(button.dataset.loadConcurrency), requests: 8 }).then(result => receiveJob(result.job)).catch(error => toast(error.message));
+});
+if (document.modelContext?.registerTool) {
+ try {
+ await document.modelContext.registerTool({ name: 'vllm_serving_action', title: 'Configure or measure vLLM serving', description: 'Apply the serving settings shown in this viewer, or run a bounded concurrent request test. Returns a job ID.', inputSchema: { type: 'object', properties: { type: { enum: ['serve', 'load_test'] }, model: { type: 'string' }, context: { type: 'integer', minimum: 512, maximum: 16384 }, maxSequences: { type: 'integer', minimum: 1, maximum: 8 }, memoryFraction: { type: 'number', minimum: 0.05, maximum: 0.8 }, prefixCaching: { type: 'boolean' }, concurrency: { type: 'integer', minimum: 1, maximum: 8 }, requests: { type: 'integer', minimum: 1, maximum: 32 } }, required: ['type', 'model'], additionalProperties: false }, annotations: { readOnlyHint: false, untrustedContentHint: true }, async execute(input) { if (!['serve', 'load_test'].includes(input?.type)) throw new Error('Unsupported serving action.'); const result = await api('/api/action', input); receiveJob(result.job); return { jobId: result.job.id, status: result.job.status }; } });
+ } catch (error) { console.info('vLLM WebMCP unavailable:', error.message); }
+}
+
+await refresh();
+const events = new EventSource('/api/events');
+events.addEventListener('state', event => { try { applyState(JSON.parse(event.data)); } catch { void refresh(); } });
+events.addEventListener('job', event => { lastReceived = Date.now(); receiveJob(JSON.parse(event.data)); });
+events.addEventListener('error', () => { $('#connection-status').textContent = 'Reconnecting…'; });
+setInterval(() => { if (Date.now() - lastReceived > 12000) void refresh(); }, 10000);
+
+// The same visible UI actions are available to supported agent browsers.
+const lifecycle = new AbortController();
+if (document.modelContext?.registerTool) {
+ const register = tool => document.modelContext.registerTool(tool, { signal: lifecycle.signal });
+ try {
+ await register({ name: 'harness_inventory', title: 'Inspect local models', description: 'Read vLLM models, residency, recent benchmarks and job status from this viewer.', inputSchema: { type: 'object', properties: {}, additionalProperties: false }, annotations: { readOnlyHint: true, untrustedContentHint: true }, async execute() { if (!await refresh()) throw new Error('The workspace service is disconnected.'); return { runtime: state.runtime, benchmarks: state.benchmarks, jobs: state.jobs.map(({ id, type, model, status, message }) => ({ id, type, model, status, message })) }; } });
+ await register({ name: 'harness_start_action', title: 'Control a local model', description: 'Start one local vLLM action. Deploy downloads a missing model. Benchmark runs three warm measurements. Returns a job ID; inspect inventory to see completion.', inputSchema: { type: 'object', properties: { type: { type: 'string', enum: ['deploy', 'download', 'unload', 'benchmark'] }, model: { type: 'string', description: 'Exact Hugging Face text model ID, including its owner.' } }, required: ['type', 'model'], additionalProperties: false }, annotations: { readOnlyHint: false, untrustedContentHint: true }, async execute(input) { if (!input || !['deploy', 'download', 'unload', 'benchmark'].includes(input.type) || typeof input.model !== 'string') throw new Error('Provide an allowed action and model.'); return runAction(input.type, input.model); } });
+ } catch (error) { console.info('WebMCP tools unavailable:', error.message); }
+}
+window.addEventListener('pagehide', () => { lifecycle.abort(); events.close(); });
diff --git a/store/agents/vllm/dist/assets/README.md b/store/agents/vllm/dist/assets/README.md
new file mode 100644
index 000000000..ab27be24c
--- /dev/null
+++ b/store/agents/vllm/dist/assets/README.md
@@ -0,0 +1,7 @@
+# vLLM identity
+
+`vllm-icon.png` is the official vLLM mark from
+https://github.com/vllm-project/vllm/blob/main/docs/assets/logos/vllm-logo-only-light.png
+retrieved September 20, 2026. Used to identify the locally installed framework;
+this harness is an independent integration. The Metal backend is part of the vLLM
+harness and does not have a separate app identity.
diff --git a/store/agents/vllm/dist/assets/vllm-icon.png b/store/agents/vllm/dist/assets/vllm-icon.png
new file mode 100644
index 000000000..7aaf17487
Binary files /dev/null and b/store/agents/vllm/dist/assets/vllm-icon.png differ
diff --git a/store/agents/vllm/dist/index.html b/store/agents/vllm/dist/index.html
new file mode 100644
index 000000000..fa7ca2e0c
--- /dev/null
+++ b/store/agents/vllm/dist/index.html
@@ -0,0 +1,61 @@
+<!doctype html>
+<html lang="en">
+<head>
+ <meta charset="utf-8">
+ <meta name="viewport" content="width=device-width, initial-scale=1">
+ <meta name="theme-color" content="#10151d">
+ <meta name="description" content="Run, inspect, and benchmark your vLLM models on this Mac using natural language.">
+ <title>vLLM — Your local model space</title>
+ <link rel="icon" type="image/png" href="/assets/vllm-icon.png">
+ <link rel="apple-touch-icon" href="/assets/vllm-icon.png">
+ <link rel="stylesheet" href="/styles.css">
+ <script type="module" src="/app.js"></script>
+</head>
+<body>
+ <header class="topbar">
+ <a class="brand" href="/?view=grid" aria-label="vLLM model space"><img class="brand-mark" src="/assets/vllm-icon.png" width="40" height="40" alt=""><span>vLLM</span><span class="pilot">LOCAL AI</span></a>
+ <div class="top-status"><span class="local-symbol" aria-hidden="true">◈</span> On your machine <span class="separator"></span><span id="connection-status">Connecting</span></div>
+ </header>
+ <main class="workspace">
+ <section class="control-room" aria-label="vLLM control room">
+ <div class="page-heading"><div><p class="eyebrow">vLLM / METAL ON THIS MAC</p><h1>Models, ready to serve.</h1></div><button id="refresh" class="icon-button" aria-label="Refresh model status" title="Refresh model status">↻</button></div>
+ <div id="runtime-notice" class="runtime-notice" role="status" hidden></div>
+ <div class="stats-row" id="stats-row">
+ <div class="stat"><span>Models on disk</span><strong id="model-count">—</strong></div>
+ <div class="stat"><span>In memory</span><strong id="running-count">—</strong></div>
+ <div class="stat"><span>Last request</span><strong id="last-speed">— <small>tok/s</small></strong></div>
+ <div class="stat"><span>Server RSS</span><strong id="model-memory">— <small>GiB</small></strong></div>
+ </div>
+ <section class="map-panel" aria-labelledby="map-heading">
+ <div class="panel-heading"><h2 id="map-heading">vLLM · Metal backend</h2><span class="mono live-label" id="map-status">DISCOVERING</span></div>
+ <div id="runtime-map" class="runtime-map"><div class="map-empty">Discovering your vLLM models…</div></div>
+ <div class="map-footer"><span><i class="legend-dot loaded"></i> In memory <i class="legend-dot"></i> On disk</span><span id="machine-info">Apple Silicon</span></div>
+ </section>
+ <section class="serving-panel" aria-labelledby="serving-heading"><div class="panel-heading"><h2 id="serving-heading">Serving</h2><span class="secondary" id="serving-state">Idle</span></div><div id="serving-content"></div></section>
+ <section class="models-panel" aria-labelledby="models-heading">
+ <div class="panel-heading"><h2 id="models-heading">vLLM models <span id="models-total" class="count-pill">0</span></h2><span class="secondary">One server · continuous batching</span></div>
+ <div id="models-list" class="models-list"></div>
+ </section>
+ <section class="concurrency-panel" aria-labelledby="concurrency-heading"><div class="panel-heading"><h2 id="concurrency-heading">Concurrent performance</h2><span class="secondary">Throughput &amp; latency</span></div><div id="concurrency-content"></div></section>
+ <section class="performance-panel" aria-labelledby="performance-heading">
+ <div class="panel-heading"><h2 id="performance-heading">Single-request baseline</h2><a id="export-benchmarks" class="secondary" href="/api/export" download hidden>Export results ↗</a></div>
+ <div id="performance-content" class="performance-content"><div class="empty-small"><span class="empty-icon" aria-hidden="true">▥</span><p>Your first run sets the baseline.</p><span>Benchmark a model to compare speed and response time.</span></div></div>
+ </section>
+ <section class="activity-panel" aria-labelledby="activity-heading"><div class="panel-heading"><h2 id="activity-heading">Activity</h2><span class="secondary">Live from vLLM</span></div><div id="activity-content"></div></section>
+ </section>
+ <aside class="assistant-panel" aria-label="Natural language model controls">
+ <div class="assistant-header"><img class="assistant-avatar" src="/assets/vllm-icon.png" width="36" height="36" alt=""><div><h2>vLLM operator</h2><span>Tell it what you want to run.</span></div><span class="private-badge">LOCAL</span></div>
+ <div class="conversation" id="conversation" role="log" aria-live="polite" aria-relevant="additions">
+ <div class="assistant-message"><span class="message-label">vLLM</span><p>Your vLLM models.<br>At your command.</p><p class="secondary">I can find a model, deploy it with vLLM, and measure how it performs here.</p></div>
+ <div class="suggestions" id="suggestions"><button data-prompt="Run a lightweight model">Run a lightweight model <span>↗</span></button><button data-prompt="Find a small model for coding">Find a model for coding <span>↗</span></button><button data-prompt="Show my models">Show my models <span>↗</span></button></div>
+ </div>
+ <div id="job-panel" class="job-panel" hidden></div>
+ <label class="model-picker" for="selected-model">Chat / follow-up model<select id="selected-model"><option value="">Auto · running model</option></select></label>
+ <form id="command-form" class="composer"><label class="sr-only" for="command-input">Ask the vLLM operator</label><textarea id="command-input" rows="3" placeholder="Run an vLLM model, compare speeds, free up memory…" maxlength="4000"></textarea><div class="composer-footer"><span id="operator-status">vLLM harness</span><button id="send-button" type="submit" aria-label="Send request">↑</button></div></form>
+ <div class="assistant-footer">vLLM inference stays on this Mac.<span>↵ Send · ⇧↵ New line</span></div>
+ </aside>
+ </main>
+ <dialog id="model-dialog" aria-labelledby="detail-title"><div id="model-detail"></div><button class="dialog-close" aria-label="Close model details">×</button></dialog>
+ <div id="toast" class="toast" role="status" hidden></div>
+</body>
+</html>
diff --git a/store/agents/vllm/dist/styles.css b/store/agents/vllm/dist/styles.css
new file mode 100644
index 000000000..86790335f
--- /dev/null
+++ b/store/agents/vllm/dist/styles.css
@@ -0,0 +1,25 @@
+:root{color-scheme:dark;--bg:#10151d;--surface:#171e29;--surface-2:#202a38;--line:#344052;--muted:#a2adbf;--text:#edf3ee;--accent:#ffc743;--cyan:#7bb1ff;--error:#ffafa4;--mono:ui-monospace,SFMono-Regular,Menlo,monospace;font-family:Inter,-apple-system,BlinkMacSystemFont,"Segoe UI",sans-serif;font-size:16px}
+*{box-sizing:border-box}body{margin:0;background:var(--bg);color:var(--text)}button,textarea,input,select{font:inherit}button{cursor:pointer}button:disabled{cursor:wait;opacity:.55}button,a,textarea,select{outline-offset:4px}button:focus-visible,a:focus-visible{outline:2px solid var(--accent)}button{color:inherit}a{color:inherit}button{border:1px solid var(--line);background:var(--surface);border-radius:8px}button:hover{border-color:#617166}h1,h2,p{margin:0}h2{font-size:1rem;font-weight:600;letter-spacing:-.02em}.mono{font-family:var(--mono)}.secondary{color:var(--muted);font-size:.875rem;line-height:1.6}.topbar{height:76px;border-bottom:1px solid var(--line);display:flex;justify-content:space-between;align-items:center;padding:0 32px}.brand{text-decoration:none;font-size:1rem;font-weight:750;letter-spacing:.15em;display:flex;align-items:center;gap:13px}.brand-mark{background:var(--accent);color:#111b0c;width:30px;height:32px;display:grid;place-items:center;clip-path:polygon(0 0,80% 0,100% 25%,100% 100%,20% 100%,0 75%);font-size:1.125rem;font-weight:900;letter-spacing:-.15em;padding-right:4px}.pilot{font-family:var(--mono);letter-spacing:.05em;font-size:.75rem;font-weight:400;color:var(--muted);padding-left:16px;margin-left:8px;border-left:1px solid var(--line)}.top-status{display:flex;gap:10px;align-items:center;font-size:.8125rem;color:var(--muted)}.local-symbol{color:var(--accent);font-size:1.1rem}.separator{height:16px;width:1px;background:var(--line);margin:0 6px}.workspace{display:grid;grid-template-columns:minmax(0,1fr) 360px;min-height:calc(100vh - 76px);max-width:1900px;margin:auto}.control-room{padding:34px 32px 40px;min-width:0}.page-heading{display:flex;justify-content:space-between;align-items:center;margin-bottom:30px}.eyebrow{font-family:var(--mono);font-size:.75rem;color:var(--accent);letter-spacing:.15em;margin-bottom:10px}h1{font-size:clamp(1.75rem,2.6vw,2.5rem);letter-spacing:-.06em;line-height:1.15;font-weight:550}.icon-button{border-radius:50%;width:38px;height:38px;font-size:1.4rem;background:transparent;color:var(--muted)}.stats-row{display:grid;grid-template-columns:repeat(4,1fr);margin-bottom:30px}.stat{padding:0 22px;border-right:1px solid var(--line);min-width:0}.stat:first-child{padding-left:0}.stat:last-child{border:0}.stat>span{font-size:.8125rem;color:var(--muted);display:block;margin-bottom:9px}.stat strong{font-family:var(--mono);font-size:1.75rem;font-weight:400;letter-spacing:-.07em}.stat small{font-family:inherit;color:var(--muted);font-size:.8rem;letter-spacing:-.02em}.panel-heading{display:flex;justify-content:space-between;align-items:center;gap:12px;margin-bottom:18px}.map-panel{border:1px solid var(--line);background:var(--surface);border-radius:12px;overflow:hidden}.map-panel .panel-heading{padding:20px 20px 0;margin-bottom:0}.live-label{font-size:.6875rem;letter-spacing:.09em;color:var(--muted)}.live-label.online{color:var(--accent)}.runtime-map{min-height:252px;position:relative;padding:25px 22px;background-image:radial-gradient(#3a454238 .8px,transparent .8px);background-size:16px 16px;display:flex;align-items:center;gap:36px}.runtime-node{border:1px solid #596a50;background:#202c21;border-radius:14px;padding:18px;min-width:150px;box-shadow:0 0 40px #ffc74307;z-index:1;position:relative}.runtime-node::after{content:'';width:37px;height:1px;background:#617751;position:absolute;right:-37px;top:50%}.runtime-node.offline{border-color:var(--line);background:var(--surface-2)}.runtime-node .node-icon{width:32px;height:32px;display:grid;place-items:center;background:var(--accent);color:#233014;border-radius:9px;font-weight:700;margin-bottom:13px}.runtime-node strong{font-size:1rem;display:block;margin-bottom:6px}.runtime-node small{font-family:var(--mono);color:var(--muted);font-size:.75rem}.model-nodes{display:flex;gap:10px;flex-wrap:wrap;align-content:center;flex:1;border-left:1px solid #3c4d3c;padding-left:22px;min-height:120px}.model-node{position:relative;text-align:left;padding:13px 15px;border-radius:8px;min-width:160px;max-width:240px;background:#1b2222}.model-node::before{content:'';position:absolute;left:-23px;top:50%;height:1px;width:22px;background:#3c4d3c}.model-node.active{border-color:#7a995a;background:#263324}.model-node strong{display:block;font-size:.875rem;overflow-wrap:anywhere;line-height:1.4}.model-node span{display:block;color:var(--muted);font-size:.75rem;margin-top:6px}.model-node.active span{color:var(--accent)}.map-empty{margin:auto;color:var(--muted);font-size:.875rem;max-width:330px;line-height:1.7;text-align:center}.empty-node{text-align:left;margin:0;padding:12px 0}.empty-node strong{display:block;color:var(--text);font-size:1rem;margin-bottom:6px}.map-footer{border-top:1px solid var(--line);display:flex;align-items:center;justify-content:space-between;gap:12px;padding:12px 20px;font-family:var(--mono);font-size:.6875rem;color:var(--muted)}.map-footer>span:first-child{display:flex;align-items:center;gap:6px}.legend-dot{width:5px;height:5px;display:inline-block;border-radius:50%;background:#6a7a74;margin-left:12px}.legend-dot:first-child{margin-left:0}.legend-dot.loaded{background:var(--accent)}.models-panel,.performance-panel{margin-top:30px}.count-pill{display:inline-grid;place-items:center;font-family:var(--mono);font-size:.75rem;min-width:23px;height:21px;border-radius:5px;border:1px solid var(--line);color:var(--muted);margin-left:6px}.model-row{width:100%;display:grid;grid-template-columns:minmax(0,1fr) 85px 80px 80px;gap:14px;align-items:center;text-align:left;border-width:0 0 1px;border-radius:0;background:transparent;padding:17px 4px;font-size:.875rem}.model-row:hover{background:var(--surface)}.model-title strong{font-weight:500;overflow-wrap:anywhere;display:block}.model-title small{font-size:.75rem;color:var(--muted);display:block;margin-top:6px}.model-status{font-size:.75rem;color:var(--muted)}.model-status.loaded{color:var(--accent)}.model-numeric{font-family:var(--mono);font-size:.75rem;color:var(--muted)}.empty-small{padding:30px 18px;border:1px dashed var(--line);border-radius:10px;color:var(--muted);text-align:center;font-size:.875rem}.empty-small p{color:var(--text);margin:8px 0;font-size:1rem}.empty-small span:not(.empty-icon){font-size:.8125rem;line-height:1.6}.empty-icon{font-size:1.5rem;color:#667973}.assistant-panel{border-left:1px solid var(--line);background:#131a1b;display:flex;flex-direction:column;min-width:0;position:sticky;top:0;height:calc(100vh - 76px);max-height:1200px}.assistant-header{display:flex;gap:12px;align-items:center;padding:26px 22px;border-bottom:1px solid var(--line)}.assistant-avatar{height:35px;width:35px;display:grid;place-items:center;color:var(--accent);background:#293526;font-size:1.5rem;border-radius:10px}.assistant-header h2{font-size:.9375rem}.assistant-header div>span{font-size:.75rem;color:var(--muted);display:block;margin-top:4px}.private-badge{font-size:.625rem;letter-spacing:.08em;color:var(--muted);border:1px solid var(--line);border-radius:4px;padding:4px 5px;margin-left:auto;font-family:var(--mono)}.conversation{padding:26px 22px;overflow:auto;flex:1;min-height:200px;scroll-behavior:smooth}.message-label{font-family:var(--mono);font-size:.625rem;letter-spacing:.1em;color:var(--accent);display:block;margin-bottom:13px}.assistant-message{font-size:.9375rem;line-height:1.65;margin-bottom:25px}.assistant-message p+p{margin-top:12px}.assistant-message .secondary{font-size:.875rem}.suggestions{display:flex;flex-direction:column;gap:8px;margin-top:25px}.suggestions button{display:flex;justify-content:space-between;align-items:center;text-align:left;font-size:.8125rem;padding:13px 12px;background:transparent;color:#c4d0cb}.suggestions button span{color:var(--accent)}.composer{margin:0 18px;border:1px solid #485441;background:#202829;border-radius:11px;overflow:hidden}.composer:focus-within{border-color:var(--accent)}textarea{color:var(--text);display:block;resize:none;background:transparent;width:100%;border:0;padding:15px 15px 5px;line-height:1.6;outline:none;font-size:.875rem}textarea::placeholder{color:#91a29b}.composer-footer{padding:8px 11px 10px 15px;display:flex;justify-content:space-between;align-items:center}.composer-footer>span{font-size:.6875rem;font-family:var(--mono);color:var(--muted)}.composer-footer button{background:var(--accent);color:#243315;border:0;border-radius:7px;font-size:1.4rem;width:31px;height:31px}.assistant-footer{display:flex;flex-wrap:wrap;gap:8px;justify-content:space-between;padding:14px 20px 20px;font-size:.625rem;color:#98a79f;line-height:1.5}.assistant-footer span{font-size:.625rem}.user-message{padding:12px 14px;background:#263127;border:1px solid #3b4e35;border-radius:9px 9px 2px 9px;margin:15px 0 25px;white-space:pre-wrap;overflow-wrap:anywhere;font-size:.875rem;line-height:1.6}.message-text{white-space:pre-wrap;overflow-wrap:anywhere}.job-panel{padding:15px;margin:0 18px 15px;background:var(--surface-2);border:1px solid var(--line);border-radius:10px;font-size:.8125rem}.job-panel strong{font-size:.8125rem;font-weight:500}.job-panel p{color:var(--muted);font-size:.75rem;margin-top:7px}.job-top{display:flex;justify-content:space-between;align-items:center}.job-top button{font-size:.6875rem;padding:4px 7px}.progress-track{height:4px;background:var(--line);border-radius:4px;margin:12px 0 2px;overflow:hidden}.progress-fill{height:100%;width:0;background:var(--accent);transition:width .3s}.toast{position:fixed;bottom:20px;left:50%;transform:translateX(-50%);padding:12px 18px;background:#33402f;color:var(--text);border:1px solid #60784d;border-radius:8px;max-width:90vw;z-index:10;font-size:.875rem}dialog{background:var(--surface);color:var(--text);border:1px solid var(--line);border-radius:16px;padding:30px;max-width:600px;width:calc(100% - 32px)}dialog::backdrop{background:#000a;backdrop-filter:blur(3px)}.dialog-close{position:absolute;right:12px;top:10px;border:0;background:transparent;font-size:1.5rem}.sr-only{position:absolute;width:1px;height:1px;overflow:hidden;clip:rect(0,0,0,0);white-space:nowrap}.detail-grid{display:grid;grid-template-columns:repeat(2,1fr);gap:16px;margin:22px 0}.detail-grid>div{border:1px solid var(--line);padding:14px;border-radius:8px}.detail-grid span{font-size:.75rem;color:var(--muted);display:block;margin-bottom:7px}.detail-grid strong{font-size:1rem;font-weight:500}.actions{display:flex;flex-wrap:wrap;gap:8px;margin-top:18px}.actions button{padding:9px 13px;font-size:.875rem}.actions button.primary{background:var(--accent);color:#233014;border-color:var(--accent)}.chart-bars{display:flex;flex-direction:column;gap:15px;padding:8px 0}.chart-row{display:grid;grid-template-columns:minmax(120px,160px) 1fr 65px;gap:16px;align-items:center;font-size:.8125rem}.chart-row>span:first-child{overflow-wrap:anywhere}.bar-track{height:22px;background:var(--surface-2);border-radius:4px;overflow:hidden}.bar-fill{height:100%;background:var(--accent);border-radius:3px;min-width:2px}.bar-value{font-family:var(--mono);font-size:.8125rem}.benchmark-meta{color:var(--muted);font-size:.75rem;margin-top:15px;line-height:1.8}.plan-card{border:1px solid #536846;border-radius:10px;background:#222e20;padding:15px;margin-top:12px}.plan-card p{font-size:.8125rem;line-height:1.6}.plan-card .actions{margin-top:10px}.error-text{color:var(--error)}.catalog-card{border:1px solid var(--line);border-radius:9px;padding:13px;margin:10px 0;font-size:.8125rem}.catalog-card strong{display:block;margin-bottom:5px}.catalog-card button{margin-top:10px;padding:6px 10px;background:var(--accent);color:#233014;font-size:.75rem;border-color:var(--accent)}.stream-output{max-height:340px;overflow:auto;border-left:2px solid #526c3f;padding-left:12px;font-size:.875rem}.warning-note{font-size:.75rem;color:#d4cca8;line-height:1.6;margin-top:10px}.mini-label{font-family:var(--mono);font-size:.6875rem;color:var(--muted);letter-spacing:.03em}
+@media(min-width:1450px){.workspace{grid-template-columns:minmax(0,1fr) 420px}.control-room{padding:40px 48px}.runtime-map{min-height:295px}.assistant-header,.conversation{padding:28px}.composer{margin:0 24px}.model-node{min-width:180px}}
+@media(max-width:1080px){.workspace{grid-template-columns:minmax(0,1fr) 320px}.control-room{padding:28px 22px}.topbar{padding:0 22px}.stat{padding:0 12px}.stat strong{font-size:1.5rem}.stat>span{font-size:.75rem}.runtime-map{gap:25px;padding:22px 16px}.runtime-node{min-width:118px;padding:14px}.runtime-node::after{width:26px;right:-26px}.model-node{min-width:130px;padding:12px;max-width:100%}.model-nodes{padding-left:14px}.model-node::before{left:-15px;width:14px}.model-row{grid-template-columns:minmax(0,1fr) 68px 65px;gap:7px}.model-row .model-numeric:last-child{display:none}.map-footer{font-size:.625rem;padding:12px}.top-status{font-size:.75rem}.assistant-header{padding:22px 16px}}
+@media(max-width:820px){.workspace{display:flex;flex-direction:column}.assistant-panel{position:relative;height:auto;max-height:none;border-left:0;border-top:1px solid var(--line);min-height:500px}.control-room{padding:26px 20px}.conversation{max-height:480px}.topbar{height:64px;padding:0 20px}.top-status .separator,.top-status #connection-status{display:none}.pilot{font-size:.625rem;padding-left:10px;margin-left:0}.brand{font-size:.875rem;gap:9px}.runtime-map{min-height:230px}.model-nodes{gap:8px}.model-node{max-width:220px}.stats-row{margin-bottom:24px}.stat{padding:0 10px}.stat>span{font-size:.6875rem}.map-footer{font-size:.625rem}.assistant-footer{font-size:.6875rem}.top-status{font-size:.6875rem;gap:5px}.assistant-header h2{font-size:1rem}.assistant-header div>span{font-size:.875rem}.private-badge{font-size:.75rem}.suggestions button{font-size:.875rem}textarea{font-size:1rem}.assistant-message{font-size:1rem}.model-row{grid-template-columns:minmax(0,1fr) 75px 80px}.model-node strong{font-size:.875rem}.model-node span{font-size:.75rem}}
+@media(max-width:430px){.top-status{display:none}.stat strong{font-size:1.35rem}.stat>span{min-height:30px;line-height:1.4}.stat small{font-size:.6875rem}.runtime-map{gap:20px;padding:18px 12px}.runtime-node{min-width:106px;padding:12px}.runtime-node::after{right:-21px;width:21px}.model-node{min-width:120px}.map-footer{flex-wrap:wrap}.panel-heading>.secondary{font-size:.75rem}.chart-row{grid-template-columns:110px 1fr 52px;gap:8px}}
+@media(prefers-reduced-motion:reduce){*{scroll-behavior:auto!important;transition:none!important;animation:none!important}}
+/* Live control and measurement surfaces. */
+[hidden]{display:none!important}.model-picker{display:flex;flex-direction:column;gap:7px;margin:0 18px 12px;color:var(--muted);font-size:.75rem}.model-picker select{width:100%;color:var(--text);background:var(--surface);border:1px solid var(--line);border-radius:7px;padding:9px;font-size:.875rem}.model-node.selected{box-shadow:0 0 0 2px var(--cyan)}.model-row.selected{background:#1a2826}.job-state{color:var(--muted);margin-left:8px;letter-spacing:0;font-family:inherit;text-transform:lowercase}.run-metrics{font-family:var(--mono);font-size:.75rem;color:var(--cyan);margin-top:12px!important}.stream-output{margin-top:14px;line-height:1.7}.chart-heading{display:flex;justify-content:space-between;gap:12px;align-items:center;flex-wrap:wrap;margin-bottom:10px}.chart-heading h3{font-size:.875rem;margin:0;font-weight:500}.chart-heading>span{font-size:.75rem;color:var(--muted)}.second-chart{margin-top:26px}.bar-fill.latency{background:var(--cyan)}.benchmark-details{margin-top:16px;border-top:1px solid var(--line);padding-top:14px;font-size:.875rem}.benchmark-details summary{cursor:pointer;color:var(--muted)}.sample-record{display:flex;flex-direction:column;gap:5px;padding:18px 0;border-bottom:1px solid var(--line);font-size:.875rem;overflow-wrap:anywhere}.sample-record span{color:var(--muted);font-size:.75rem}.chart-row{grid-template-columns:minmax(125px,170px) 1fr 64px}.bar-value{text-align:right}.bar-value small{font-size:.75rem}.detail-grid strong,#detail-title{overflow-wrap:anywhere}#operator-status{overflow:hidden;text-overflow:ellipsis;white-space:nowrap;max-width:calc(100% - 44px)}.job-top{gap:12px}.job-top button{flex-shrink:0}.assistant-message,.stream-output,.user-message{font-size:1rem}.live-label,.map-footer,.private-badge,.message-label,.composer-footer>span,.assistant-footer,.assistant-footer span,.job-top button,.mini-label,.pilot,.stat>span,.top-status,.stat small{font-size:.75rem}.stat>span,.suggestions button,.top-status{font-size:.875rem}.composer textarea{font-size:1rem}.model-node span,.model-title small,.model-status,.model-numeric{font-size:.8125rem}.map-footer{line-height:1.6;flex-wrap:wrap}.topbar{gap:14px}.assistant-panel{min-height:650px}.job-panel p{overflow-wrap:anywhere}.catalog-card p{line-height:1.6}.model-nodes{min-width:0}.model-node{min-width:0;flex:1 1 150px}button,a,select{-webkit-tap-highlight-color:transparent}
+@media(max-width:1080px){.stat strong{font-size:1.45rem}.stat>span{min-height:32px}.model-row{grid-template-columns:minmax(0,1fr) 76px 76px}.chart-row{grid-template-columns:130px 1fr 60px;gap:10px}}
+@media(max-width:820px){.assistant-panel{min-height:650px}.top-status{font-size:.75rem}.stat>span{font-size:.75rem}.stat small{font-size:.75rem}.model-picker select{font-size:1rem}.assistant-panel{height:auto}.model-nodes{flex-basis:140px}.model-node{min-width:0}.map-footer{font-size:.75rem}}
+@media(max-width:430px){.chart-row{grid-template-columns:100px 1fr 54px;gap:8px}.model-row{grid-template-columns:minmax(0,1fr) 72px 62px}.model-numeric,.model-status{font-size:.75rem}.stats-row{gap:0}.stat{padding:0 7px}.stat strong{font-size:1.3rem}.stat>span{min-height:36px}.runtime-node{min-width:100px}.control-room{padding:24px 16px}.runtime-map{gap:17px}.runtime-node::after{width:18px;right:-18px}.map-footer>span{flex-wrap:wrap}.assistant-header{padding:20px 16px}.conversation{padding:22px 18px}}
+@media(pointer:coarse){button{min-height:44px}.model-picker select{min-height:44px}.composer-footer button{min-width:44px}.icon-button{width:44px;height:44px}}
+.activity-panel{margin-top:32px}.activity-row{display:flex;align-items:flex-start;gap:16px;padding:17px 0;border-top:1px solid var(--line);font-size:.875rem}.activity-state{min-width:78px;font-family:var(--mono);font-size:.75rem;color:var(--muted);padding-top:3px}.activity-row>div{flex:1;min-width:0}.activity-row strong{font-weight:500;overflow-wrap:anywhere}.activity-row p{font-size:.8125rem;line-height:1.6;margin-top:5px;color:var(--muted);overflow-wrap:anywhere}.activity-time{font-size:.75rem;white-space:nowrap;color:var(--muted)}.activity-row button{padding:5px 9px;font-size:.75rem}.embedded-viewer .workspace{display:block;max-width:1440px}.embedded-viewer .assistant-panel{display:none}.embedded-viewer .control-room{padding:34px clamp(20px,4vw,60px)}.embedded-viewer .runtime-map{min-height:240px}.embedded-viewer .topbar{height:64px}.embedded-viewer .workspace{min-height:calc(100vh - 64px)}.embedded-viewer .stats-row{max-width:1050px}.embedded-viewer .model-nodes{flex:1}.embedded-viewer .model-node{flex:0 1 230px}.embedded-viewer .model-row{grid-template-columns:minmax(0,1fr) 100px 110px 100px}.embedded-viewer .runtime-node{min-width:160px}.embedded-viewer .model-row .model-numeric:last-child{display:block}
+@media(max-width:600px){.embedded-viewer .model-row{grid-template-columns:minmax(0,1fr) 76px 76px}.embedded-viewer .model-row .model-numeric:last-child{display:none}.embedded-viewer .runtime-node{min-width:100px}.activity-row{gap:10px}.activity-state{min-width:64px}.activity-time{display:none}.embedded-viewer .model-node{flex-basis:160px}.embedded-viewer .control-room{padding:24px 16px}}
+.embedded-viewer [data-chat]{display:none}
+
+/* vLLM identity: use the official mark at every logo size. */
+.brand{font-size:1.45rem;letter-spacing:-.04em;text-transform:none;gap:12px}.brand .brand-mark{display:block;width:40px;height:40px;padding:0;background:none;clip-path:none;border-radius:10px;object-fit:contain}.brand .pilot{font-size:.65rem;letter-spacing:.13em;margin-left:10px;padding-left:20px}.runtime-node .node-icon{display:block;width:42px;height:42px;padding:0;margin:0 auto 9px;background:none;border-radius:10px;object-fit:contain}.assistant-header .assistant-avatar{padding:0;object-fit:contain;background:none;border-radius:9px}
+
+.runtime-notice{margin:0 0 20px;padding:16px 18px;border:1px solid #614840;border-radius:12px;background:#241e1a}.runtime-notice[hidden]{display:none}.runtime-notice strong{color:var(--error);font-size:.9rem}.runtime-notice p{margin:8px 0;color:var(--text);font-size:.85rem;line-height:1.5}.runtime-notice summary{cursor:pointer;color:var(--muted);font-size:.8rem}.runtime-notice pre{white-space:pre-wrap;overflow-wrap:anywhere;font: .75rem/1.5 var(--mono);color:var(--error)}
+
+/* vLLM's blue / gold identity and serving-specific measurements. */
+.runtime-node{border-color:#405e84;background:#1b2b41}.runtime-node::after{background:#527bab}.model-nodes{border-left-color:#3a526d}.model-node{background:#192230}.model-node::before{background:#3a526d}.model-node.active{border-color:#b69643;background:#302b1e}.model-row.selected{background:#1d2b3d}.runtime-map{background-image:radial-gradient(#58749a35 .8px,transparent .8px)}.assistant-panel{background:#131a25}.brand .brand-mark,.runtime-node .node-icon,.assistant-avatar{background:white}.model-title strong,.sample-record strong{overflow-wrap:anywhere}.chart-row>span:first-child{font-size:.75rem}.serving-panel,.concurrency-panel{margin-top:30px}.serving-grid{display:grid;grid-template-columns:repeat(3,minmax(0,1fr));border:1px solid var(--line);border-radius:12px;overflow:hidden;background:var(--surface)}.serving-stat{padding:18px 20px;min-width:0;border-bottom:1px solid var(--line);border-right:1px solid var(--line)}.serving-stat:nth-child(3n){border-right:0}.serving-stat:nth-last-child(-n+3){border-bottom:0}.serving-stat span{display:block;font-size:.75rem;color:var(--muted);line-height:1.5;margin-bottom:10px}.serving-stat strong{font:500 1.1rem var(--mono);overflow-wrap:anywhere}.load-result{padding:20px;border:1px solid var(--line);border-radius:12px;background:var(--surface);margin-top:12px}.load-title{display:flex;justify-content:space-between;gap:12px;align-items:center}.load-title>div{display:flex;align-items:center;gap:12px;min-width:0}.load-title strong{font-weight:500;font-size:.875rem;overflow-wrap:anywhere}.concurrency-badge{flex-shrink:0;color:var(--cyan);background:#213851;border:1px solid #3a5d85;border-radius:7px;padding:7px 10px;font:500 .875rem var(--mono)}.load-bar{display:flex;gap:18px;align-items:center;margin:20px 0 15px}.load-bar .bar-track{flex:1;height:12px}.load-bar>strong{font:400 1.3rem var(--mono);white-space:nowrap}.load-bar small{font-size:.75rem;color:var(--muted)}.load-metrics{display:flex;gap:18px;flex-wrap:wrap;font-size:.75rem;color:var(--muted);line-height:1.8}.load-metrics b{font-family:var(--mono);color:var(--text);font-weight:400}.load-details{margin-top:14px;font-size:.75rem;color:var(--muted);line-height:1.8}.load-details summary{cursor:pointer}.load-details p{margin-top:10px;overflow-wrap:anywhere}.load-title .secondary{white-space:nowrap;font-size:.75rem}
+@media(max-width:600px){.serving-stat{padding:14px 12px}.serving-stat strong{font-size:.95rem}.load-result{padding:15px}.load-title{align-items:flex-start}.load-title>div{gap:8px}.load-title .secondary{display:none}.load-metrics{gap:10px 15px}.serving-panel .actions button{flex:1;font-size:.75rem}.chart-heading>span{font-size:.7rem}}
diff --git a/store/agents/vllm/guard.py b/store/agents/vllm/guard.py
new file mode 100644
index 000000000..b0e658f53
--- /dev/null
+++ b/store/agents/vllm/guard.py
@@ -0,0 +1,27 @@
+"""Authenticate every native-server route, including administrative endpoints."""
+import hmac
+import os
+
+
+class LocalOnlyMiddleware:
+ def __init__(self, app):
+ self.app = app
+ self.key = os.environ["VLLM_API_KEY"]
+
+ async def __call__(self, scope, receive, send):
+ if scope["type"] == "lifespan":
+ return await self.app(scope, receive, send)
+ if scope["type"] != "http":
+ return await send({"type": "websocket.close", "code": 1008})
+ headers = {k.lower(): v.decode("latin1") for k, v in scope.get("headers", [])}
+ port = os.environ["HARNESS_VLLM_PORT"]
+ host = headers.get(b"host", "")
+ origin = headers.get(b"origin")
+ allowed = host in {f"127.0.0.1:{port}", f"localhost:{port}"}
+ allowed = allowed and (not origin or origin == f"http://{host}")
+ allowed = allowed and headers.get(b"sec-fetch-site") != "cross-site"
+ allowed = allowed and hmac.compare_digest(headers.get(b"authorization", ""), "Bearer " + self.key)
+ if not allowed:
+ await send({"type": "http.response.start", "status": 403, "headers": [(b"content-type", b"application/json")]})
+ return await send({"type": "http.response.body", "body": b'{"error":"Private workspace server"}'})
+ return await self.app(scope, receive, send)
diff --git a/store/agents/vllm/harness.json b/store/agents/vllm/harness.json
new file mode 100644
index 000000000..deddd9582
--- /dev/null
+++ b/store/agents/vllm/harness.json
@@ -0,0 +1,29 @@
+{
+ "spec": 1,
+ "id": "autonomous/vllm",
+ "name": "vLLM",
+ "category": "Local AI",
+ "author": "OpenHarness contributors",
+ "description": "Serve language models on this Mac with vLLM Metal. Ask the agent to configure context, parallel sequences and prefix caching, then measure concurrent request throughput in the live viewer.",
+ "engine": "codex",
+ "workspace": {
+ "template": "template",
+ "marker": "local-ai.json"
+ },
+ "agent": {
+ "instructions": "agent/AGENTS.md",
+ "env": {
+ "LOCAL_AI": "${dsh}/toolchain/local-ai",
+ "VLLM": "${dsh}/toolchain/vllm"
+ }
+ },
+ "toolchain": {
+ "setup": "toolchain/setup.sh",
+ "doctor": "toolchain/doctor.sh"
+ },
+ "viewer": {
+ "command": "./viewer.sh",
+ "url": "http://127.0.0.1:${port}/?view=grid"
+ },
+ "verdict": ".harness/verdict.json"
+}
diff --git a/store/agents/vllm/licenses/vllm-metal.txt b/store/agents/vllm/licenses/vllm-metal.txt
new file mode 100644
index 000000000..261eeb9e9
--- /dev/null
+++ b/store/agents/vllm/licenses/vllm-metal.txt
@@ -0,0 +1,201 @@
+ Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
+
+ APPENDIX: How to apply the Apache License to your work.
+
+ To apply the Apache License to your work, attach the following
+ boilerplate notice, with the fields enclosed by brackets "[]"
+ replaced with your own identifying information. (Don't include
+ the brackets!) The text should be enclosed in the appropriate
+ comment syntax for the file format. We also recommend that a
+ file or class name and description of purpose be included on the
+ same "printed page" as the copyright notice for easier
+ identification within third-party archives.
+
+ Copyright [yyyy] [name of copyright owner]
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
diff --git a/store/agents/vllm/licenses/vllm.txt b/store/agents/vllm/licenses/vllm.txt
new file mode 100644
index 000000000..261eeb9e9
--- /dev/null
+++ b/store/agents/vllm/licenses/vllm.txt
@@ -0,0 +1,201 @@
+ Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
+
+ APPENDIX: How to apply the Apache License to your work.
+
+ To apply the Apache License to your work, attach the following
+ boilerplate notice, with the fields enclosed by brackets "[]"
+ replaced with your own identifying information. (Don't include
+ the brackets!) The text should be enclosed in the appropriate
+ comment syntax for the file format. We also recommend that a
+ file or class name and description of purpose be included on the
+ same "printed page" as the copyright notice for easier
+ identification within third-party archives.
+
+ Copyright [yyyy] [name of copyright owner]
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
diff --git a/store/agents/vllm/main.mjs b/store/agents/vllm/main.mjs
new file mode 100644
index 000000000..3f850e4b1
--- /dev/null
+++ b/store/agents/vllm/main.mjs
@@ -0,0 +1,8 @@
+import { dirname, join, resolve } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { start } from './src/server.mjs';
+import { VllmAdapter, VLLM_PROFILE } from './src/vllm.mjs';
+const packageRoot = dirname(fileURLToPath(import.meta.url));
+const workspace = resolve(process.env.HARNESS_WORKSPACE || packageRoot);
+const adapter = new VllmAdapter({ packageRoot, dataDir: join(workspace, '.harness') });
+await start(Number(process.env.HARNESS_VIEWER_PORT || process.env.HARNESS_PORT || 4312), { workspaceRoot: workspace, assetRoot: join(packageRoot, 'dist'), profile: VLLM_PROFILE, adapter });
diff --git a/store/agents/vllm/requirements.lock b/store/agents/vllm/requirements.lock
new file mode 100644
index 000000000..283192f44
--- /dev/null
+++ b/store/agents/vllm/requirements.lock
@@ -0,0 +1,159 @@
+accelerate==1.15.0
+agent-detector==2.0.0
+aiohappyeyeballs==2.7.1
+aiohttp==3.14.3
+aiosignal==1.4.0
+annotated-doc==0.0.5
+annotated-types==0.8.0
+anthropic==1.7.0
+anyio==4.15.1
+apache-tvm-ffi==0.1.12
+astor==0.8.1
+attrs==26.1.0
+blake3==1.0.9
+cachetools==7.2.0
+cbor2==6.1.4
+certifi==2026.7.22
+cffi==2.1.1
+charset-normalizer==3.5.1
+click==8.5.0
+cloudpickle==3.1.2
+compressed-tensors==0.17.0
+cryptography==50.0.1
+depyf==0.20.0
+detect-installer==0.2.1
+dill==0.4.1
+dnspython==2.8.0
+docstring-parser==0.18.0
+einops==0.8.2
+email-validator==2.3.0
+fastapi==0.136.3
+fastapi-cli==0.0.32
+fastapi-cloud-cli==0.26.0
+fastar==0.12.0
+filelock==4.0.1
+frozenlist==1.8.0
+fsspec==2026.9.0
+googleapis-common-protos==1.75.3
+grpcio==1.84.0
+h11==0.16.0
+hf-xet==1.6.0
+httpcore==1.0.9
+httpcore2==2.13.0
+httptools==0.8.0
+httpx==0.28.1
+httpx2==2.13.0
+huggingface-hub==1.32.0
+idna==3.20
+ijson==3.5.1
+interegular==0.3.3
+jinja2==3.1.6
+jiter==0.17.0
+jmespath==1.1.0
+jsonschema==4.26.0
+jsonschema-specifications==2025.9.1
+lark==1.2.2
+llguidance==1.7.6
+llvmlite==0.47.0
+lm-format-enforcer==0.11.3
+loguru==0.7.3
+markdown-it-py==4.2.0
+markupsafe==3.0.3
+mcp==2.2.0
+mcp-types==2.2.0
+mdurl==0.1.2
+miniaudio==1.71
+mistral-common==1.11.7
+mlx==0.32.1
+mlx-audio==0.5.4
+mlx-lm @ git+https://github.com/ml-explore/mlx-lm@9e6acca691e64d6d8bb808c328fcdea459099cca
+mlx-metal==0.32.1
+mlx-vlm==0.6.17
+model-hosting-container-standards==0.1.16
+mpmath==1.3.0
+msgspec==0.21.1
+multidict==6.9.0
+nanobind==2.10.2
+networkx==3.6.1
+ninja==1.13.2
+numba==0.65.0
+numpy==2.3.5
+openai==3.16.2
+openai-harmony==0.0.8
+opencv-python==5.0.0.93
+opencv-python-headless==5.0.0.93
+opentelemetry-api==1.44.0
+opentelemetry-exporter-otlp==1.44.0
+opentelemetry-exporter-otlp-proto-common==1.44.0
+opentelemetry-exporter-otlp-proto-grpc==1.44.0
+opentelemetry-exporter-otlp-proto-http==1.44.0
+opentelemetry-proto==1.44.0
+opentelemetry-sdk==1.44.0
+opentelemetry-semantic-conventions==0.65b0
+opentelemetry-semantic-conventions-ai==0.5.1
+outlines-core==0.2.14
+packaging==26.3
+partial-json-parser==0.2.1.1.post7
+pillow==12.3.0
+prometheus-client==0.26.0
+prometheus-fastapi-instrumentator==8.1.0
+propcache==0.5.4
+protobuf==7.36.2
+psutil==7.2.2
+py-cpuinfo==9.0.0
+pybase64==1.5.0
+pycountry==26.2.16
+pycparser==3.0
+pydantic==2.13.5
+pydantic-core==2.46.5
+pydantic-extra-types==2.11.1
+pydantic-settings==2.15.0
+pygments==2.21.0
+pyjwt==2.14.0
+python-dotenv==1.2.3
+python-json-logger==4.2.0
+python-multipart==0.0.32
+pyyaml==6.0.3
+pyzmq==27.2.0
+referencing==0.37.0
+regex==2026.9.10
+requests==2.34.2
+rich==15.0.0
+rich-toolkit==0.20.5
+rignore==0.8.1
+rpds-py==2026.6.3
+safetensors==0.8.0
+scipy==1.18.1
+sentencepiece==0.2.2
+sentry-sdk==2.69.2
+setproctitle==1.3.7
+setuptools==77.0.3
+shellingham==1.5.4
+six==1.17.0
+sniffio==1.3.1
+sounddevice==0.5.6
+sse-starlette==3.4.11
+starlette==1.6.0
+supervisor==4.3.0
+sympy==1.14.0
+tiktoken==0.14.0
+tokenizers==0.23.2
+torch==2.13.0
+torchaudio==2.11.0
+torchcodec==0.16.0
+torchvision==0.28.0
+tqdm==4.70.1
+transformers==5.17.0
+truststore==0.10.4
+typer==0.27.2
+typing-extensions==4.16.0
+typing-inspection==0.4.4
+urllib3==2.8.0
+uvicorn==0.53.0
+uvloop==0.22.1
+vllm @ https://github.com/vllm-project/vllm/releases/download/v0.29.0/vllm-0.29.0%2Bcpu-cp312-cp312-macosx_11_0_arm64.whl#sha256=7133cb494664c502b07b114fe915847f0e71296d502f67f6fa76172ba46978df
+vllm-metal @ https://github.com/vllm-project/vllm-metal/releases/download/v0.29.0/vllm_metal-0.29.0-cp312-cp312-macosx_15_0_arm64.whl#sha256=0d03dcc2be9a4286a19c5e53e1355d2e47acdbf2be6f9de9cbb48cc4b880f393
+watchfiles==1.2.0
+websockets==17.1
+xgrammar==0.2.7
+yarl==1.25.1
diff --git a/store/agents/vllm/requirements.txt b/store/agents/vllm/requirements.txt
new file mode 100644
index 000000000..b86099f15
--- /dev/null
+++ b/store/agents/vllm/requirements.txt
@@ -0,0 +1,3 @@
+# Matched macOS wheels from the official vLLM Metal v0.29.0 Homebrew formula.
+vllm @ https://github.com/vllm-project/vllm/releases/download/v0.29.0/vllm-0.29.0%2Bcpu-cp312-cp312-macosx_11_0_arm64.whl#sha256=7133cb494664c502b07b114fe915847f0e71296d502f67f6fa76172ba46978df
+vllm-metal @ https://github.com/vllm-project/vllm-metal/releases/download/v0.29.0/vllm_metal-0.29.0-cp312-cp312-macosx_15_0_arm64.whl#sha256=0d03dcc2be9a4286a19c5e53e1355d2e47acdbf2be6f9de9cbb48cc4b880f393
diff --git a/store/agents/vllm/server.py b/store/agents/vllm/server.py
new file mode 100644
index 000000000..1f3fe7dff
--- /dev/null
+++ b/store/agents/vllm/server.py
@@ -0,0 +1,19 @@
+"""Run the official CLI with a lease tied to the owning viewer process."""
+import os
+import signal
+import threading
+import time
+
+
+def watch_parent(expected):
+ while True:
+ time.sleep(2)
+ if os.getppid() != expected:
+ os.killpg(os.getpgrp(), signal.SIGTERM)
+ return
+
+
+if __name__ == "__main__":
+ threading.Thread(target=watch_parent, args=(int(os.environ["HARNESS_PARENT_PID"]),), daemon=True).start()
+ from vllm.entrypoints.cli.main import main
+ main()
diff --git a/store/agents/vllm/src/catalog.mjs b/store/agents/vllm/src/catalog.mjs
new file mode 100644
index 000000000..616380e9d
--- /dev/null
+++ b/store/agents/vllm/src/catalog.mjs
@@ -0,0 +1,6 @@
+export const CATALOG = [
+ { id: 'qwen3:0.6b', name: 'Qwen 3 · 0.6B', purpose: 'A lightweight first model', description: 'Quick to download. Good for trying the harness, short answers, and simple tasks.', downloadGB: 0.523, use: 'general', small: true, source: 'https://ollama.com/library/qwen3:0.6b' },
+ { id: 'qwen2.5-coder:1.5b', name: 'Qwen 2.5 Coder · 1.5B', purpose: 'Small coding assistant', description: 'A compact model for code snippets, explanations, and simple programming tasks.', downloadGB: 0.986, use: 'coding', small: true, source: 'https://ollama.com/library/qwen2.5-coder:1.5b' },
+ { id: 'gemma3:1b', name: 'Gemma 3 · 1B', purpose: 'Everyday text tasks', description: 'A small text-only model for summarizing, rewriting, and everyday questions.', downloadGB: 0.815, use: 'writing', small: true, source: 'https://ollama.com/library/gemma3:1b' },
+];
+export const DEFAULT_MODEL = CATALOG[0].id;
diff --git a/store/agents/vllm/src/controller.mjs b/store/agents/vllm/src/controller.mjs
new file mode 100644
index 000000000..79613900b
--- /dev/null
+++ b/store/agents/vllm/src/controller.mjs
@@ -0,0 +1,191 @@
+import { EventEmitter } from 'node:events';
+import { randomUUID, createHash } from 'node:crypto';
+import os from 'node:os';
+import { setTimeout as delay } from 'node:timers/promises';
+import * as ollama from './ollama.mjs';
+import { resolveIntent, validateAction } from './intents.mjs';
+import { CATALOG } from './catalog.mjs';
+
+export const BENCHMARK_PROMPT = 'Explain how a bicycle works to a curious beginner. Discuss the pedals, chain, gears, wheels, brakes, and balance. Write a clear explanation of at least 250 words without a heading.';
+export const BENCHMARK_SETTINGS = { context: 2048, maxTokens: 128, seed: 42, temperature: 0, thinking: false };
+export const median = values => { const v = values.filter(Number.isFinite).sort((a, b) => a - b); return v.length ? (v[Math.floor((v.length - 1) / 2)] + v[Math.ceil((v.length - 1) / 2)]) / 2 : null; };
+const terminal = job => !['queued', 'running'].includes(job.status);
+export const safeError = error => /TLS handshake timeout|fetch failed/.test(error.message) ? 'The model download or runtime connection timed out. Check your connection and retry; partial downloads can resume.' : String(error.message || error).replace(/https?:\/\/[^\s"<>]+/g, url => url.split('?')[0]).slice(0, 700);
+
+export class Controller extends EventEmitter {
+ constructor(store, adapter = ollama, profile = {}) {
+ super();
+ this.profile = { id: 'ollama', name: 'Ollama', catalog: CATALOG, ...profile };
+ this.store = store; this.adapter = adapter; this.queue = []; this.aborts = new Map(); this.actions = new Map(); this.processing = false; this.closing = false;
+ this.system = { platform: os.platform(), arch: os.arch(), chip: os.cpus()[0]?.model || 'Local machine', totalMemory: os.totalmem(), cores: os.cpus().length };
+ this.runtime = { id: this.profile.id, name: this.profile.name, online: false, models: [] };
+ this.updatedAt = null;
+ }
+ snapshot() { return { runtime: this.runtime, system: { ...this.system, freeMemory: os.freemem() }, benchmarks: this.store.data.benchmarks, loadTests: this.store.data.loadTests, runs: this.store.data.runs, jobs: this.store.data.jobs, catalog: this.profile.catalog, updatedAt: this.updatedAt }; }
+ async refresh() {
+ if (this.refreshing) return this.refreshing;
+ this.refreshing = this.adapter.discover().then(runtime => { this.runtime = runtime; this.updatedAt = new Date().toISOString(); this.emit('state', this.snapshot()); }).finally(() => { this.refreshing = null; });
+ return this.refreshing;
+ }
+ async command(text, { selectedModel, history } = {}) {
+ await this.refresh();
+ const intent = (this.profile.resolveIntent || resolveIntent)(text, { models: this.runtime.models, selectedModel, profile: this.profile });
+ if (intent.kind === 'action') {
+ if (intent.action.type === 'chat') intent.action.history = history;
+ return { kind: 'job', job: await this.submit(intent.action) };
+ }
+ if (intent.kind === 'inventory') {
+ const models = this.runtime.models;
+ return { kind: 'reply', message: !this.runtime.online ? `${this.profile.name} is not running. Say “start ${this.profile.name}” or “run a lightweight model”.` : models.length ? `${models.length} model${models.length === 1 ? '' : 's'} on disk, ${models.filter(m => m.running).length} in memory.\n${models.map(m => `${m.name} — ${m.running ? 'in memory' : 'on disk'}`).join('\n')}` : `${this.profile.name} is ready. Say “run a lightweight model” to download your first model.` };
+ }
+ return intent;
+ }
+ async submit(input) {
+ if (this.closing) throw new Error('Harness is shutting down. Try again after it restarts.');
+ if (this.queue.length >= 8) throw new Error('There are already eight jobs waiting. Let them finish or cancel one.');
+ const action = this.profile.validateAction ? this.profile.validateAction(input) : validateAction(input, this.profile.validateModel);
+ const job = { id: randomUUID(), type: action.type, model: action.model || null, models: action.models || [], status: 'queued', progress: 0, message: 'Waiting for the model operator', createdAt: new Date().toISOString(), output: '', result: null };
+ this.store.data.jobs.unshift(job);
+ this.store.data.jobs = this.store.data.jobs.filter((j, i) => i < 40 || !terminal(j));
+ this.actions.set(job.id, action); this.queue.push(job.id);
+ await this.store.save(); this.emit('job', job);
+ void this.drain();
+ return job;
+ }
+ async cancel(id) {
+ const job = this.store.data.jobs.find(j => j.id === id);
+ if (!job) throw new Error('Job not found.');
+ if (terminal(job)) return job;
+ this.aborts.get(id)?.abort(new Error('Cancelled by you.'));
+ if (job.status === 'queued') { job.status = 'cancelled'; job.message = 'Cancelled before starting.'; job.completedAt = new Date().toISOString(); this.queue = this.queue.filter(queued => queued !== id); this.actions.delete(id); await this.store.save(); }
+ else job.message = 'Cancelling…';
+ this.emit('job', job); return job;
+ }
+ async drain() {
+ if (this.processing) return;
+ this.processing = true;
+ try {
+ while (this.queue.length && !this.closing) {
+ const id = this.queue.shift(); const job = this.store.data.jobs.find(j => j.id === id); const action = this.actions.get(id);
+ if (!job || job.status !== 'queued') continue;
+ const abort = new AbortController(); this.aborts.set(id, abort);
+ job.status = 'running'; job.startedAt = new Date().toISOString();
+ const progress = (message, value, extra) => { job.message = message; if (Number.isFinite(value)) job.progress = Math.max(0, Math.min(100, value)); if (extra) job.download = extra; this.emit('job', job); };
+ try {
+ await this.store.save();
+ progress(`Connecting to ${this.profile.name}`, 1);
+ await this.adapter.ensureRunning(this.store.directory, { signal: abort.signal, onProgress: progress });
+ await this.refresh();
+ job.result = await this.execute(action, { signal: abort.signal, progress, job });
+ abort.signal.throwIfAborted();
+ job.status = 'succeeded'; job.progress = 100; job.message = job.result.message;
+ } catch (error) {
+ job.status = abort.signal.aborted ? 'cancelled' : 'failed';
+ job.message = abort.signal.aborted ? 'Cancelled. Any downloaded files stay cached; retrying can resume the download.' : safeError(error);
+ } finally {
+ job.completedAt = new Date().toISOString(); this.aborts.delete(id); this.actions.delete(id);
+ try { await this.refresh(); await this.store.save(); } catch (error) { job.persistenceError = error.message; }
+ this.emit('job', job);
+ }
+ }
+ } finally { this.processing = false; }
+ }
+ async deploy(model, { signal, progress }, load = true, context) {
+ await this.refresh();
+ if (!this.runtime.models.some(m => m.id === model)) {
+ progress(`Inspecting ${model} before downloading`, 2);
+ await this.adapter.pullModel(model, { signal, onProgress: (message, value, extra) => progress(message, 3 + (value || 0) * 0.75, extra) });
+ }
+ await this.adapter.localModelDetails(model, { signal });
+ if (load) {
+ progress(`Loading ${model} into memory`, 82);
+ await this.adapter.loadModel(model, { signal, context: context ?? (this.profile.preserveServingConfig ? undefined : 4096) });
+ for (let i = 0; i < 15; i++) { await this.refresh(); if (this.runtime.models.some(m => m.id === model && m.running)) return; await delay(200, undefined, { signal }); }
+ throw new Error(`${this.profile.name} accepted the load request, but the model did not appear in memory. Check the runtime log and try again.`);
+ }
+ await this.refresh();
+ if (!this.runtime.models.some(m => m.id === model)) throw new Error(`${this.profile.name} did not report the model as installed after the download.`);
+ }
+ async execute(action, ctx) {
+ const { signal, progress, job } = ctx;
+ if (action.model) job.activeModel = action.model;
+ if (action.type === 'start') return { message: `${this.profile.name} ${this.runtime.version} is ready. ${this.runtime.models.length} models found on disk.` };
+ if (action.type === 'serve') {
+ await this.deploy(action.model, ctx, false);
+ progress(`Starting ${action.model} with the requested serving settings`, 80);
+ await this.adapter.loadModel(action.model, { ...action, signal });
+ await this.refresh();
+ return { model: action.model, serving: this.runtime.serving, message: `${action.model} is serving · ${this.runtime.serving.context.toLocaleString()} context · ${this.runtime.serving.maxSequences} parallel sequences · prefix cache ${this.runtime.serving.prefixCaching ? 'on' : 'off'}.` };
+ }
+ if (action.type === 'load_test') {
+ await this.deploy(action.model, ctx);
+ progress('Warming the server before the concurrent test', 5);
+ await this.adapter.generate(action.model, [{ role: 'user', content: BENCHMARK_PROMPT }], { ...BENCHMARK_SETTINGS, maxTokens: 32, signal });
+ const serving = { ...this.runtime.serving }; const samples = []; let next = 0; let complete = 0;
+ const began = performance.now();
+ const worker = async () => {
+ while (next < action.requests) {
+ signal.throwIfAborted(); const index = next++;
+ try {
+ const { metrics } = await this.adapter.generate(action.model, [{ role: 'user', content: `${BENCHMARK_PROMPT} Reference number: ${index + 1}.` }], { ...BENCHMARK_SETTINGS, signal });
+ samples.push({ index, status: 'succeeded', ...metrics });
+ } catch (error) { signal.throwIfAborted(); samples.push({ index, status: 'failed', error: safeError(error) }); }
+ complete++; progress(`${complete} of ${action.requests} requests completed · concurrency ${action.concurrency}`, 10 + complete / action.requests * 85);
+ }
+ };
+ const outcomes = await Promise.allSettled(Array.from({ length: action.concurrency }, worker)); signal.throwIfAborted();
+ const rejected = outcomes.find(outcome => outcome.status === 'rejected'); if (rejected) throw rejected.reason;
+ const totalMs = performance.now() - began;
+ const passed = samples.filter(s => s.status === 'succeeded');
+ const latency = passed.map(s => s.totalMs).sort((a, b) => a - b);
+ const result = { id: randomUUID(), at: new Date().toISOString(), model: action.model, digest: this.runtime.models.find(m => m.id === action.model)?.digest, runtime: { name: this.profile.name, version: this.runtime.version, backend: this.runtime.backend, backendVersion: this.runtime.backendVersion }, hardware: this.system, concurrency: action.concurrency, requests: action.requests, succeeded: passed.length, failed: samples.length - passed.length, totalMs, aggregateTokensPerSecond: passed.reduce((sum, s) => sum + s.tokens, 0) / (totalMs / 1000), medianLatencyMs: median(latency), p95LatencyMs: latency.length ? latency[Math.ceil(latency.length * 0.95) - 1] : null, medianFirstTokenMs: median(passed.map(s => s.firstTokenMs)), settings: { ...BENCHMARK_SETTINGS, context: serving.context }, serving, prompt: BENCHMARK_PROMPT, promptVariation: 'Reference number: {1-based request index}.', mode: 'one discarded warmup; repeated prompt prefixes may be cached; client end-to-end measurements', samples: samples.sort((a, b) => a.index - b.index) };
+ this.store.data.loadTests.unshift(result); this.store.data.loadTests = this.store.data.loadTests.slice(0, 200); await this.store.save();
+ if (!passed.length) throw new Error(`All ${action.requests} requests failed. The load-test record contains each error.`);
+ return { model: action.model, loadTestId: result.id, message: `${passed.length}/${action.requests} requests succeeded at concurrency ${action.concurrency} · ${result.aggregateTokensPerSecond.toFixed(1)} aggregate tokens/s · ${Math.round(result.p95LatencyMs)} ms p95 request latency · ${result.failed} errors.` };
+ }
+ if (['deploy', 'download'].includes(action.type)) {
+ await this.deploy(action.model, ctx, action.type === 'deploy');
+ return { model: action.model, message: action.type === 'deploy' ? `${action.model} is ready on this Mac. Ask a question, or say “benchmark it”. ${this.profile.readyMessage || 'Idle models unload after 15 minutes; their files stay on disk.'}` : `${action.model} is downloaded. Say “run ${action.model}” to load it into memory.` };
+ }
+ if (['unload', 'unload_all'].includes(action.type)) {
+ const targets = action.type === 'unload_all' ? this.runtime.models.filter(m => m.running).map(m => m.id) : [action.model];
+ for (const model of targets) { progress(`Releasing ${model} from memory`, 30); await this.adapter.unloadModel(model, { signal }); }
+ await this.refresh();
+ if (targets.some(model => this.runtime.models.some(m => m.id === model && m.running))) throw new Error('The runtime still reports a requested model in memory. It may be in use by another client.');
+ return { message: targets.length ? `${targets.length === 1 ? targets[0] : `${targets.length} models`} unloaded. Downloaded files are still on disk.` : 'No models were in memory.' };
+ }
+ if (action.type === 'chat') {
+ await this.deploy(action.model, ctx);
+ progress(`Generating with ${action.model}`, 90);
+ let lastEmit = 0;
+ const generated = await this.adapter.generate(action.model, [...action.history, { role: 'user', content: action.prompt }], { signal, onToken: text => { job.output += text; if (Date.now() - lastEmit > 80) { this.emit('job', job); lastEmit = Date.now(); } } });
+ const run = { id: randomUUID(), at: new Date().toISOString(), ...generated.metrics };
+ this.store.data.runs.unshift(run); this.store.data.runs = this.store.data.runs.slice(0, 200);
+ return { model: action.model, metrics: generated.metrics, message: `Response from ${action.model}.` };
+ }
+ if (action.type === 'benchmark') {
+ const results = [];
+ for (const [modelIndex, model] of action.models.entries()) {
+ job.activeModel = model;
+ await this.deploy(model, ctx, true, BENCHMARK_SETTINGS.context);
+ const metadata = this.runtime.models.find(m => m.id === model);
+ progress(`Warming up ${model} (not included in results)`, 5 + modelIndex / action.models.length * 90);
+ await this.adapter.generate(model, [{ role: 'user', content: BENCHMARK_PROMPT }], { ...BENCHMARK_SETTINGS, maxTokens: 32, signal });
+ const samples = [];
+ for (let i = 0; i < 3; i++) {
+ progress(`${model} · measured run ${i + 1} of 3`, 10 + (modelIndex + i / 3) / action.models.length * 85);
+ const { metrics } = await this.adapter.generate(model, [{ role: 'user', content: BENCHMARK_PROMPT }], { ...BENCHMARK_SETTINGS, signal });
+ if (!Number.isFinite(metrics.tokensPerSecond) || !Number.isFinite(metrics.firstTokenMs)) throw new Error(`${this.profile.name} did not return usable timing data; no benchmark was recorded.`);
+ samples.push(metrics);
+ }
+ await this.refresh();
+ const result = { id: randomUUID(), model, digest: metadata?.digest || null, at: new Date().toISOString(), runtime: { name: this.profile.name, version: this.runtime.version, mlxVersion: this.runtime.mlxVersion, backend: this.runtime.backend, backendVersion: this.runtime.backendVersion }, hardware: this.system, settings: BENCHMARK_SETTINGS, prompt: BENCHMARK_PROMPT, promptHash: createHash('sha256').update(BENCHMARK_PROMPT).digest('hex'), mode: this.profile.benchmarkMode || 'warm; repeated prompt may use prefix cache', samples, medianTokensPerSecond: median(samples.map(s => s.tokensPerSecond)), medianFirstTokenMs: median(samples.map(s => s.firstTokenMs)), medianFirstOutputMs: median(samples.map(s => s.firstOutputMs)), residentBytes: this.runtime.models.find(m => m.id === model)?.resident?.size || null };
+ this.store.data.benchmarks.unshift(result); this.store.data.benchmarks = this.store.data.benchmarks.slice(0, 200); results.push(result);
+ await this.store.save();
+ }
+ return { model: action.models.at(-1), benchmarkIds: results.map(r => r.id), message: results.map(r => `${r.model}: ${r.medianTokensPerSecond.toFixed(1)} tokens/s · ${Math.round(r.medianFirstTokenMs)} ms to first token (median of 3 warm runs).`).join('\n') };
+ }
+ throw new Error('Unknown action.');
+ }
+ async close() { this.closing = true; for (const abort of this.aborts.values()) abort.abort(new Error('Harness shutting down.')); await this.store.save(); await this.adapter.close?.(); }
+}
diff --git a/store/agents/vllm/src/intents.mjs b/store/agents/vllm/src/intents.mjs
new file mode 100644
index 000000000..7aa35e8df
--- /dev/null
+++ b/store/agents/vllm/src/intents.mjs
@@ -0,0 +1,91 @@
+import { CATALOG, DEFAULT_MODEL } from './catalog.mjs';
+import { validateModel } from './ollama.mjs';
+
+export const HELP = 'Try “run a lightweight model”, “find a model for coding”, “benchmark it”, “ask it: explain unified memory”, or “free up memory”. You can also name any local Ollama model, such as qwen3:0.6b.';
+
+// Only explicit operational requests enter the executor. Model replies are text,
+// never instructions. There is no shell-command generation or eval here.
+export function resolveIntent(input, { models = [], selectedModel, profile = {} } = {}) {
+ const catalog = profile.catalog || CATALOG;
+ const defaultModel = profile.defaultModel || DEFAULT_MODEL;
+ const normalize = profile.validateModel || validateModel;
+ const help = profile.help || HELP;
+ if (typeof input !== 'string' || !input.trim() || input.length > 4000) throw new Error('Enter a request between 1 and 4,000 characters.');
+ const text = input.trim();
+ const lower = text.toLowerCase();
+ const selected = selectedModel ? normalize(selectedModel) : null;
+ const preferred = selected || models.find(m => m.running)?.id || models[0]?.id;
+ const action = (type, props = {}) => ({ kind: 'action', action: { type, ...props } });
+ const reply = message => ({ kind: 'reply', message });
+ const choose = () => reply('Which model? Select one in the model map, or include its name in your request.');
+ // Prompt contents must never be parsed as commands or model identifiers.
+ const ask = text.match(/^(?:(?:please |can you )?)(?:ask|tell|chat with)\s+(.+?)\s*:\s*([\s\S]+)$/i);
+ if (ask) {
+ const target = /^(it|me|us|the model|my model)$/i.test(ask[1]) ? preferred : normalize(ask[1].trim());
+ return target ? action('chat', { model: target, prompt: ask[2] }) : choose();
+ }
+ const names = [...new Set([...models.map(m => m.id), ...catalog.map(m => m.id)])];
+ const mentioned = names.filter(name => lower.includes(name.toLowerCase()));
+ for (const match of text.matchAll(profile.modelPattern || /\b(?:[a-z0-9][a-z0-9._-]*\/)?[a-z0-9][a-z0-9._-]*:[a-z0-9][a-z0-9._-]*\b/g)) {
+ if (!mentioned.includes(match[0]) && !/^\d+:\d+$/.test(match[0])) mentioned.push(normalize(match[0]));
+ }
+ const modelForTask = /cod(?:e|ing)|programming/.test(lower) ? catalog[1].id : /writing|summari[sz]|rewrite/.test(lower) ? catalog[2].id : defaultModel;
+ const lead = lower.replace(/^(?:please\s+|(?:can|could|would) you\s+|i(?:'d| would) like (?:you )?to\s+|i want (?:you )?to\s+)+/, '');
+ if (/\b(don['’]?t|do not|never|not yet|cancel|abort)\b/.test(lower) && /\b(run|load|download|deploy|start|stop|unload|benchmark|compare|cancel|abort)\b/.test(lower)) return reply('I haven’t started an action. Use the Cancel button on an active job to stop it.');
+ if (/^(how (?:do|would|can|should)|what (?:if|happens)|why|explain|help)\b/.test(lead)) return reply(help);
+ if (/^(show|list|what|which|check|status|refresh)\b/.test(lead) && /models?|running|loaded|status|memory|installed/.test(lead) || /^(status|models|refresh)[.!?]?$/.test(lead)) return { kind: 'inventory' };
+ if (/^(find|recommend|suggest|pick|choose|what['’]?s|what is)\b/.test(lead) && /model|coding|writing/.test(lead)) return { kind: 'catalog', message: 'Start small, then choose a larger model once you have a baseline. These downloads run locally on this Mac.', models: /coding|code|programming/.test(lower) ? [catalog[1]] : /writing|summari[sz]/.test(lower) ? [catalog[2]] : catalog };
+ if (/^(delete|remove|erase)\b/.test(lead)) return reply('This pilot keeps downloaded models on disk. To release their memory, say “unload it” or “free up memory”.');
+ if (/^(free (?:up )?(?:my |the )?memory|unload all(?: models)?|stop all(?: models)?|release (?:all )?memory)[.!?]?$/.test(lead)) return action('unload_all');
+ if (/^(start|launch|open) (?:the )?(?:ollama|mlx(?:-lm)?|runtime|server)[.!?]?$/.test(lead)) return action('start');
+
+ const operation = lead.match(/^(run|deploy|load|start|download|pull|install|unload|stop|benchmark|test|measure|compare)\b/);
+ if (operation) {
+ if (/\b(?:and then|then|after that)\b/.test(lead) || /\band\s+(run|deploy|download|stop|unload|benchmark|ask)\b/.test(lead)) return reply('Let’s do one operation at a time. Start with the first request, then ask for the next step when it finishes.');
+ const verb = operation[1];
+ const rest = lead.slice(verb.length).trim().replace(/[.!?]+$/, '');
+ let model = mentioned[0];
+ if (!model && /^[a-z0-9][a-z0-9._-]*(?:\/[a-z0-9][a-z0-9._-]*)?$/.test(rest) && !/^(it|this|that|model|ollama|speed|performance|all)$/.test(rest)) {
+ model = names.find(name => name.split(':')[0] === rest) || normalize(rest);
+ }
+ if (!model && /\b(it|this model|that model|my model|the model|the running model)\b/.test(rest)) model = preferred;
+ if (['benchmark', 'test', 'measure', 'compare'].includes(verb)) {
+ if (verb === 'compare' && /\ball\b/.test(rest)) return models.length > 4 ? reply('This pilot compares up to four models at a time. Name the models to compare.') : models.length ? action('benchmark', { models: models.map(m => m.id) }) : choose();
+ const targets = mentioned.length ? mentioned : model ? [model] : preferred ? [preferred] : [];
+ if (verb === 'compare' && targets.length < 2) return reply('Name two models from this framework to compare. Each gets the same prompt and settings.');
+ return targets.length ? action('benchmark', { models: targets }) : choose();
+ }
+ if (['unload', 'stop'].includes(verb)) return model ? action('unload', { model }) : choose();
+ if (!model && /\b(lightweight|small|tiny|fast|coding|writing|starter|first)\b/.test(rest)) model = modelForTask;
+ if (!model) return reply('Name the model you want, or say “run a lightweight model” and I’ll use Qwen 3 0.6B.');
+ return action(['download', 'pull', 'install'].includes(verb) ? 'download' : 'deploy', { model });
+ }
+ if (/\b(benchmark|speed|fast|tokens per second|performance)\b/.test(lower) && /^(how|test|measure)\b/.test(lead)) return preferred ? action('benchmark', { models: [preferred] }) : choose();
+ if (/^(chat|talk)(?: with| to)?(?: it| the model)?[.!?]?$/.test(lead)) return preferred ? reply(`${preferred} is selected. Ask a question below, or say “ask it: …”.`) : choose();
+ if (/^(run|deploy|download|load|start|stop|unload|benchmark|compare|install|delete|remove)\b/.test(lead) || /\b(shell|terminal|sudo|rm -rf)\b/.test(lower)) return reply(help);
+ return preferred ? action('chat', { model: preferred, prompt: text }) : reply(help);
+}
+
+export function validateAction(input, normalize = validateModel) {
+ if (!input || typeof input !== 'object' || Array.isArray(input)) throw new Error('Provide a model action.');
+ const { type } = input;
+ if (!['start', 'download', 'deploy', 'unload', 'unload_all', 'benchmark', 'chat'].includes(type)) throw new Error('That action is not supported by this pilot.');
+ const action = { type };
+ if (['download', 'deploy', 'unload', 'chat'].includes(type)) action.model = normalize(input.model);
+ if (type === 'benchmark') {
+ if (!Array.isArray(input.models) || !input.models.length || input.models.length > 4) throw new Error('Benchmark between one and four models at a time.');
+ action.models = [...new Set(input.models.map(normalize))];
+ }
+ if (type === 'chat') {
+ if (typeof input.prompt !== 'string' || !input.prompt.trim() || input.prompt.length > 4000) throw new Error('Enter a prompt between 1 and 4,000 characters.');
+ action.prompt = input.prompt.trim();
+ action.history = [];
+ if (Array.isArray(input.history)) {
+ action.history = input.history.slice(-6).map(m => {
+ if (!['user', 'assistant'].includes(m.role) || typeof m.content !== 'string' || m.content.length > 6000) throw new Error('Invalid chat history.');
+ return { role: m.role, content: m.content };
+ });
+ }
+ }
+ return action;
+}
diff --git a/store/agents/vllm/src/json-worker.mjs b/store/agents/vllm/src/json-worker.mjs
new file mode 100644
index 000000000..7129ce6a1
--- /dev/null
+++ b/store/agents/vllm/src/json-worker.mjs
@@ -0,0 +1,80 @@
+import { spawn } from 'node:child_process';
+import { access, mkdir, open } from 'node:fs/promises';
+import { constants } from 'node:fs';
+import { createInterface } from 'node:readline';
+import { randomUUID } from 'node:crypto';
+import { join } from 'node:path';
+
+// Private, request-correlated JSON-lines transport. It never invokes a shell.
+export class JsonWorker {
+ constructor({ python, script, directory, env = {}, name }) {
+ Object.assign(this, { python, script, directory, env, name });
+ this.child = null; this.ready = false; this.pending = new Map(); this.state = {}; this.closed = false;
+ }
+ async start() {
+ if (this.closed) throw new Error(`${this.name} is closing.`);
+ if (this.stopping) await this.stopping;
+ if (this.child && this.ready) return;
+ if (this.starting) return this.starting;
+ this.starting = this.launch().finally(() => { this.starting = null; });
+ return this.starting;
+ }
+ async launch() {
+ await access(this.python, constants.X_OK).catch(() => { throw new Error(`${this.name} needs setup. Run this harness's setup command.`); });
+ await mkdir(this.directory, { recursive: true });
+ const log = await open(join(this.directory, 'model-cache.log'), 'a', 0o600);
+ const child = spawn(this.python, ['-u', this.script], { stdio: ['pipe', 'pipe', log.fd], env: { ...process.env, ...this.env } });
+ await log.close(); this.child = child; this.ready = false;
+ child.stdin.on('error', () => {});
+ await new Promise((resolve, reject) => {
+ const timer = setTimeout(() => { this.stop(child); reject(new Error(`${this.name} startup timed out.`)); }, 45000);
+ const fail = error => { clearTimeout(timer); reject(error); };
+ child.once('error', error => { if (this.child === child) this.child = null; fail(error); });
+ const lines = createInterface({ input: child.stdout });
+ lines.on('line', line => {
+ if (this.child !== child) return;
+ try {
+ if (line.length > 8 * 1024 * 1024) throw new Error('Oversized worker event.');
+ const event = JSON.parse(line);
+ if (event.event === 'ready') { clearTimeout(timer); this.ready = true; this.state = event; resolve(); return; }
+ if (event.state) this.state = { ...this.state, ...event.state };
+ const pending = this.pending.get(event.id); if (!pending) return;
+ if (event.event === 'done') pending.resolve(event.result);
+ else if (event.event === 'error') pending.reject(new Error(event.message));
+ else pending.onEvent(event);
+ } catch (error) { this.stop(child); fail(error); }
+ });
+ child.once('close', (code, signal) => {
+ clearTimeout(timer); lines.close();
+ if (this.child === child) { this.child = null; this.ready = false; }
+ const error = new Error(`${this.name} stopped (${signal || code}).`);
+ for (const pending of this.pending.values()) if (pending.child === child) pending.reject(error);
+ fail(error);
+ });
+ });
+ }
+ stop(child = this.child) {
+ if (!child || this.stopping) return this.stopping;
+ this.ready = false;
+ this.stopping = new Promise(resolve => {
+ const timer = setTimeout(() => child.kill('SIGKILL'), 1500);
+ child.once('close', () => { clearTimeout(timer); resolve(); });
+ child.kill('SIGTERM');
+ }).finally(() => { this.stopping = null; });
+ return this.stopping;
+ }
+ async request(action, body = {}, { signal, onEvent = () => {}, timeout = 180000 } = {}) {
+ signal?.throwIfAborted(); await this.start(); signal?.throwIfAborted();
+ const child = this.child; const id = randomUUID();
+ return new Promise((resolve, reject) => {
+ let settled = false;
+ const finish = (fn, value) => { if (settled) return; settled = true; clearTimeout(timer); signal?.removeEventListener('abort', abort); this.pending.delete(id); fn(value); };
+ const abort = () => { this.stop(child); finish(reject, signal.reason || new Error('Cancelled.')); };
+ const timer = setTimeout(() => { this.stop(child); finish(reject, new Error(`${this.name} ${action} timed out.`)); }, timeout);
+ this.pending.set(id, { child, resolve: value => finish(resolve, value), reject: error => finish(reject, error), onEvent });
+ signal?.addEventListener('abort', abort, { once: true });
+ child.stdin.write(JSON.stringify({ id, action, ...body }) + '\n', error => { if (error) finish(reject, error); });
+ });
+ }
+ async close() { this.closed = true; await this.starting?.catch(() => {}); await this.stop(); }
+}
diff --git a/store/agents/vllm/src/mlx.mjs b/store/agents/vllm/src/mlx.mjs
new file mode 100644
index 000000000..a347b2ea1
--- /dev/null
+++ b/store/agents/vllm/src/mlx.mjs
@@ -0,0 +1,156 @@
+import { spawn } from 'node:child_process';
+import { access, mkdir, open } from 'node:fs/promises';
+import { constants } from 'node:fs';
+import { createInterface } from 'node:readline';
+import { join } from 'node:path';
+import { randomUUID } from 'node:crypto';
+import { totalmem } from 'node:os';
+import { performance } from 'node:perf_hooks';
+
+export function validateMlxModel(model) {
+ if (typeof model !== 'string' || model.length > 180 || !/^[A-Za-z0-9][A-Za-z0-9_.-]*\/[A-Za-z0-9][A-Za-z0-9_.-]*$/.test(model) || model.includes('..')) throw new Error('Use a Hugging Face model ID, such as mlx-community/Qwen3-0.6B-4bit.');
+ return model;
+}
+
+export const MLX_CATALOG = [
+ { id: 'mlx-community/Qwen3-0.6B-4bit', name: 'Qwen 3 · 0.6B', purpose: 'A lightweight first model', description: 'A small MLX model for trying local chat and measuring this Mac.', downloadGB: 0.4, use: 'general', small: true },
+ { id: 'mlx-community/Qwen2.5-Coder-1.5B-Instruct-4bit', name: 'Qwen 2.5 Coder · 1.5B', purpose: 'Small coding assistant', description: 'A compact MLX coding model for explanations and short snippets.', downloadGB: 1.0, use: 'coding', small: true },
+ { id: 'mlx-community/Qwen2.5-0.5B-Instruct-4bit', name: 'Qwen 2.5 · 0.5B', purpose: 'Small baseline', description: 'A compact instruction model for short answers and a second performance baseline.', downloadGB: 0.3, use: 'writing', small: true },
+].map(model => ({ ...model, source: `https://huggingface.co/${model.id}` }));
+
+export const MLX_PROFILE = {
+ id: 'mlx-lm', name: 'MLX-LM', catalog: MLX_CATALOG, defaultModel: MLX_CATALOG[0].id,
+ validateModel: validateMlxModel, modelPattern: /\b[A-Za-z0-9][A-Za-z0-9_.-]*\/[A-Za-z0-9][A-Za-z0-9_.-]*\b/g,
+ help: 'Try “run a lightweight model”, “benchmark it”, “ask it: explain unified memory”, or “free up memory”. Use Hugging Face MLX model IDs, such as mlx-community/Qwen3-0.6B-4bit.',
+ benchmarkMode: 'warm model; fresh prompt cache for each sample',
+ readyMessage: 'MLX-LM keeps one model loaded per workspace. Switching models releases the previous model; idle models unload after 15 minutes.',
+};
+
+export class MlxAdapter {
+ constructor({ packageRoot, dataDir, python, interpreterArgs = ['-u'] } = {}) {
+ this.python = python || join(packageRoot, '.venv/bin/python');
+ this.interpreterArgs = interpreterArgs;
+ this.workerPath = join(packageRoot, 'worker.py');
+ this.dataDir = dataDir;
+ this.child = null; this.starting = null; this.pending = new Map(); this.generation = 0;
+ this.current = { models: [] }; this.version = null; this.mlxVersion = null; this.expiresAt = null;
+ this.installed = false; this.closed = false;
+ }
+ async ensureRunning(_directory, { signal } = {}) {
+ signal?.throwIfAborted();
+ if (this.closed) throw new Error('The MLX-LM workspace is closing.');
+ if (this.stopping) await this.stopping;
+ signal?.throwIfAborted();
+ if (this.closed) throw new Error('The MLX-LM workspace is closing.');
+ if (this.child && this.ready) return;
+ if (this.starting) return this.starting;
+ this.starting = this.startWorker().finally(() => { this.starting = null; });
+ return this.starting;
+ }
+ async startWorker() {
+ try { await access(this.python, constants.X_OK); this.installed = true; }
+ catch { throw new Error('MLX-LM needs setup. Run this harness’s setup command to install its isolated Python environment.'); }
+ await mkdir(this.dataDir, { recursive: true });
+ const log = await open(join(this.dataDir, 'mlx-lm.log'), 'a', 0o600);
+ const child = spawn(this.python, [...this.interpreterArgs, this.workerPath], { stdio: ['pipe', 'pipe', log.fd], env: { ...process.env, HF_HUB_DISABLE_TELEMETRY: '1', TOKENIZERS_PARALLELISM: 'false', HARNESS_MLX_REGISTRY: join(this.dataDir, 'mlx-models.json') } });
+ await log.close();
+ this.child = child; this.ready = false;
+ child.stdin.on('error', () => {}); // The write callback rejects the corresponding request.
+ const generation = ++this.generation;
+ await new Promise((resolve, reject) => {
+ const timer = setTimeout(() => { child.kill('SIGKILL'); reject(new Error('MLX-LM did not start within 45 seconds. Check .harness/mlx-lm.log.')); }, 45000);
+ const startupFailure = error => { clearTimeout(timer); reject(error); };
+ child.once('error', startupFailure);
+ const lines = createInterface({ input: child.stdout });
+ lines.on('line', line => {
+ try {
+ if (line.length > 8 * 1024 * 1024) throw new Error('MLX-LM returned an oversized event.');
+ const event = JSON.parse(line);
+ if (this.generation !== generation) return;
+ if (event.event === 'ready') {
+ clearTimeout(timer); this.ready = true; this.current = event;
+ this.version = event.version; this.mlxVersion = event.mlxVersion;
+ resolve(); return;
+ }
+ if (event.state) this.current = { ...this.current, ...event.state };
+ const pending = this.pending.get(event.id); if (!pending) return;
+ if (event.event === 'error') pending.reject(new Error(event.message));
+ else if (event.event === 'done') pending.resolve(event.result);
+ else pending.onEvent(event);
+ } catch (error) { child.kill('SIGKILL'); startupFailure(error); }
+ });
+ child.once('exit', (code, signal) => {
+ clearTimeout(timer); lines.close();
+ const error = new Error(`MLX-LM worker stopped (${signal || code}). Its loaded model was released. Check .harness/mlx-lm.log if unexpected.`);
+ if (this.generation === generation) { this.child = null; this.ready = false; this.expiresAt = null; this.current = { ...this.current, models: this.current.models.map(m => ({ ...m, running: false, resident: null })) }; }
+ for (const request of this.pending.values()) if (request.child === child) request.reject(error);
+ startupFailure(error);
+ });
+ });
+ }
+ stopWorker(child, signal = 'SIGTERM') {
+ if (child !== this.child) return;
+ this.ready = false;
+ if (this.stopping) return;
+ this.stopping = new Promise(resolve => {
+ const timer = setTimeout(() => child.kill('SIGKILL'), 1500);
+ child.once('close', () => { clearTimeout(timer); resolve(); });
+ child.kill(signal);
+ }).finally(() => { this.stopping = null; });
+ }
+ async request(action, body = {}, { signal, onEvent = () => {}, timeout = 180000 } = {}) {
+ await this.ensureRunning(undefined, { signal }); signal?.throwIfAborted();
+ const child = this.child; const id = randomUUID();
+ return new Promise((resolve, reject) => {
+ let settled = false;
+ const finish = (fn, value) => { if (settled) return; settled = true; clearTimeout(timer); signal?.removeEventListener('abort', abort); this.pending.delete(id); fn(value); };
+ const abort = () => { this.stopWorker(child); finish(reject, signal.reason || new Error('Cancelled.')); };
+ const timer = setTimeout(() => { this.stopWorker(child, 'SIGKILL'); finish(reject, new Error(`MLX-LM ${action} timed out. Its worker was stopped to release memory.`)); }, timeout);
+ this.pending.set(id, { child, resolve: value => finish(resolve, value), reject: error => finish(reject, error), onEvent });
+ signal?.addEventListener('abort', abort, { once: true });
+ child.stdin.write(JSON.stringify({ id, action, ...body }) + '\n', error => { if (error) finish(reject, error); });
+ });
+ }
+ async discover() {
+ let error;
+ try {
+ await this.ensureRunning();
+ if (!this.pending.size) {
+ const loaded = this.current.models.find(m => m.running);
+ if (loaded && this.expiresAt && Date.now() > this.expiresAt) await this.unloadModel(loaded.id);
+ await this.request('inventory');
+ }
+ } catch (e) { error = e.message; }
+ return { id: 'mlx-lm', name: 'MLX-LM', online: Boolean(this.child && this.ready), installed: this.installed, version: this.version, mlxVersion: this.mlxVersion, error,
+ endpoint: 'Private local worker', cacheRoot: this.current.cacheRoot, memoryScope: 'MLX active allocations; one model per workspace',
+ models: this.current.models.map(m => ({ ...m, resident: m.resident ? { ...m.resident, expires_at: this.expiresAt ? new Date(this.expiresAt).toISOString() : null } : null })) };
+ }
+ async pullModel(model, { signal, onProgress = () => {} } = {}) {
+ return this.request('download', { model: validateMlxModel(model), memoryBudget: Math.floor(totalmem() * 0.8) }, { signal, timeout: 30 * 60 * 1000, onEvent: event => { if (event.event === 'progress') onProgress(event.message, event.percent, { completed: event.completed, total: event.total }); } });
+ }
+ localModelDetails(model, options) { return this.request('details', { model: validateMlxModel(model) }, options); }
+ async loadModel(model, { signal, context = 4096 } = {}) {
+ const result = await this.request('load', { model: validateMlxModel(model), context }, { signal });
+ this.expiresAt = Date.now() + 15 * 60 * 1000; return result;
+ }
+ async unloadModel(model, { signal } = {}) {
+ const result = await this.request('unload', { model: validateMlxModel(model) }, { signal });
+ this.expiresAt = null; return result;
+ }
+ async generate(model, messages, { signal, onToken = () => {}, ...options } = {}) {
+ const started = performance.now(); let firstTokenMs = null; let firstOutputMs = null;
+ const result = await this.request('generate', { model: validateMlxModel(model), messages, options }, { signal, onEvent: event => {
+ if (event.event === 'first_token' && firstTokenMs === null) firstTokenMs = performance.now() - started;
+ if (event.event === 'token') { if (firstOutputMs === null && /\S/.test(event.text)) firstOutputMs = performance.now() - started; onToken(event.text); }
+ } });
+ this.expiresAt = Date.now() + 15 * 60 * 1000;
+ result.metrics.firstTokenMs = firstTokenMs; result.metrics.firstOutputMs = firstOutputMs;
+ return result;
+ }
+ async close() {
+ this.closed = true;
+ if (this.starting) await this.starting.catch(() => {});
+ if (this.child) this.stopWorker(this.child);
+ await this.stopping;
+ }
+}
diff --git a/store/agents/vllm/src/ollama.mjs b/store/agents/vllm/src/ollama.mjs
new file mode 100644
index 000000000..17063efec
--- /dev/null
+++ b/store/agents/vllm/src/ollama.mjs
@@ -0,0 +1,187 @@
+import { access, mkdir, open, statfs } from 'node:fs/promises';
+import { constants } from 'node:fs';
+import { homedir, totalmem } from 'node:os';
+import { join } from 'node:path';
+import { spawn } from 'node:child_process';
+import { setTimeout as delay } from 'node:timers/promises';
+import { performance } from 'node:perf_hooks';
+
+export const OLLAMA_URL = 'http://127.0.0.1:11434';
+export const DEFAULT_CONTEXT = 4096;
+let starting;
+
+export function validateModel(model) {
+ if (typeof model !== 'string' || model.length > 180 || !/^[a-zA-Z0-9][a-zA-Z0-9._-]*(?:\/[a-zA-Z0-9][a-zA-Z0-9._-]*)?(?::[a-zA-Z0-9][a-zA-Z0-9._-]*)?$/.test(model) || model.includes('..')) throw new Error('Use an Ollama model name, such as qwen3:0.6b.');
+ if (/(?:^|[-:])cloud(?:$|[-:])/i.test(model)) throw new Error('This harness runs local models. Choose a model without a cloud tag.');
+ return model.includes(':') ? model : `${model}:latest`;
+}
+
+export async function* readNDJSON(body) {
+ const decoder = new TextDecoder();
+ let buffer = '';
+ for await (const chunk of body) {
+ buffer += typeof chunk === 'string' ? chunk : decoder.decode(chunk, { stream: true });
+ if (buffer.length > 8 * 1024 * 1024) throw new Error('The runtime returned an oversized event.');
+ let newline;
+ while ((newline = buffer.indexOf('\n')) !== -1) {
+ const line = buffer.slice(0, newline).trim(); buffer = buffer.slice(newline + 1);
+ if (line) { const event = JSON.parse(line); if (event.error) throw new Error(event.error); yield event; }
+ }
+ }
+ buffer += decoder.decode();
+ if (buffer.trim()) { const event = JSON.parse(buffer); if (event.error) throw new Error(event.error); yield event; }
+}
+
+export async function streamRequest(path, body, { signal, timeout = 30 * 60 * 1000, onEvent = () => {} } = {}) {
+ const response = await fetch(`${OLLAMA_URL}${path}`, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ ...body, stream: true }), signal: signal ? AbortSignal.any([signal, AbortSignal.timeout(timeout)]) : AbortSignal.timeout(timeout) });
+ if (!response.ok) { const message = await response.text(); throw new Error(message.slice(0, 600) || `Ollama returned ${response.status}`); }
+ let last;
+ for await (const event of readNDJSON(response.body)) { last = event; await onEvent(event); }
+ return last;
+}
+
+export async function ensureRunning(dataDir, { signal, onProgress = () => {} } = {}) {
+ try { return await ollamaJSON('/api/version', { timeout: 1500, signal }); } catch {}
+ signal?.throwIfAborted();
+ if (starting) return starting;
+ starting = (async () => {
+ const binary = await findOllama();
+ if (!binary) throw new Error('Ollama is not installed. Install the official Ollama app from ollama.com, then try again.');
+ onProgress('Starting Ollama on this Mac');
+ await mkdir(dataDir, { recursive: true });
+ const log = await open(join(dataDir, 'ollama.log'), 'a', 0o600);
+ let failure;
+ const child = spawn(binary, ['serve'], { env: { ...process.env, OLLAMA_HOST: '127.0.0.1:11434', OLLAMA_NO_CLOUD: '1' }, detached: true, stdio: ['ignore', log.fd, log.fd] });
+ child.on('error', error => { failure = error; });
+ child.unref();
+ await log.close();
+ const deadline = Date.now() + 30000;
+ while (Date.now() < deadline) {
+ if (failure) throw failure;
+ signal?.throwIfAborted();
+ try { return await ollamaJSON('/api/version', { timeout: 1000, signal }); } catch {}
+ if (child.exitCode !== null) throw new Error(`Ollama exited during startup. Details are in ${join(dataDir, 'ollama.log')}.`);
+ await delay(300, undefined, { signal });
+ }
+ throw new Error('Ollama did not become ready within 30 seconds. Check the runtime log and try again.');
+ })().finally(() => { starting = null; });
+ return starting;
+}
+
+export async function modelManifest(model, { signal } = {}) {
+ const normalized = validateModel(model);
+ const [path, tag] = normalized.split(':');
+ const repository = path.includes('/') ? path : `library/${path}`;
+ const url = `https://registry.ollama.ai/v2/${repository}/manifests/${tag}`;
+ const response = await fetch(url, { signal: signal ? AbortSignal.any([signal, AbortSignal.timeout(12000)]) : AbortSignal.timeout(12000), headers: { Accept: 'application/vnd.docker.distribution.manifest.v2+json' } });
+ if (!response.ok) throw new Error(response.status === 404 ? `The model ${model} was not found in Ollama's registry. Check its name and tag.` : `Could not inspect the download size (${response.status}). Try again when the registry is reachable.`);
+ const manifest = await response.json();
+ const layers = [...(manifest.layers || []), ...(manifest.config ? [manifest.config] : [])];
+ if (!layers.length || layers.some(layer => !Number.isSafeInteger(layer.size) || layer.size < 0)) throw new Error('The model registry returned an invalid manifest.');
+ return { bytes: layers.reduce((total, layer) => total + layer.size, 0), repository, tag };
+}
+
+export async function pullModel(model, options = {}) {
+ model = validateModel(model);
+ const manifest = await modelManifest(model, options);
+ if (manifest.bytes > totalmem() * 0.8) throw new Error(`This model download is ${(manifest.bytes / 1024 ** 3).toFixed(1)} GiB, too large for the pilot's memory budget on this Mac. Choose a smaller quantization or model.`);
+ try {
+ const disk = await statfs(process.env.OLLAMA_MODELS || join(homedir(), '.ollama'));
+ if (disk.bavail * disk.bsize < manifest.bytes + 2 * 1024 ** 3) throw new Error('There is not enough free disk space for this download plus 2 GiB of headroom.');
+ } catch (error) { if (!['ENOENT', 'EPERM', 'EACCES'].includes(error.code)) throw error; }
+ const layers = new Map();
+ const last = await streamRequest('/api/pull', { model }, { ...options, onEvent: event => {
+ if (event.digest && event.total) layers.set(event.digest, { total: event.total, completed: event.completed || 0 });
+ const known = [...layers.values()];
+ const completed = known.reduce((sum, layer) => sum + layer.completed, 0);
+ options.onProgress?.(event.status || 'Downloading model', Math.min(99, Math.floor(completed / manifest.bytes * 100)), { completed, total: manifest.bytes });
+ } });
+ if (last?.status !== 'success') throw new Error('The model download ended before Ollama confirmed success. You can retry to resume it.');
+ return manifest;
+}
+
+export async function localModelDetails(model, options = {}) {
+ const details = await ollamaJSON('/api/show', { method: 'POST', body: { model: validateModel(model) }, timeout: 15000, ...options });
+ if (details.remote_host || details.remote_model) throw new Error('This model forwards inference to a remote service. Choose a local model for this harness.');
+ if (Array.isArray(details.capabilities) && !details.capabilities.includes('completion')) throw new Error('This pilot supports language generation models. The selected model does not expose completion capability.');
+ return details;
+}
+
+export async function loadModel(model, { signal, context = DEFAULT_CONTEXT, keepAlive = '15m' } = {}) {
+ await localModelDetails(model, { signal });
+ return ollamaJSON('/api/generate', { method: 'POST', body: { model, prompt: '', stream: false, keep_alive: keepAlive, options: { num_ctx: context } }, timeout: 180000, signal });
+}
+
+export async function unloadModel(model, { signal } = {}) {
+ return ollamaJSON('/api/generate', { method: 'POST', body: { model: validateModel(model), stream: false, keep_alive: 0 }, timeout: 30000, signal });
+}
+
+export async function generate(model, messages, { signal, maxTokens = 512, context = DEFAULT_CONTEXT, keepAlive = '15m', seed = 42, temperature = 0.3, thinking = true, onToken = () => {} } = {}) {
+ const details = await localModelDetails(model, { signal });
+ const start = performance.now();
+ let firstTokenMs = null;
+ let firstOutputMs = null;
+ let text = '';
+ const body = { model, messages, options: { num_predict: maxTokens, num_ctx: context, temperature, seed }, keep_alive: keepAlive };
+ if (details.capabilities?.includes('thinking')) body.think = /^gpt-oss(?::|$)/.test(model) ? 'low' : Boolean(thinking);
+ const last = await streamRequest('/api/chat', body, { signal, timeout: 180000, onEvent: event => {
+ const piece = event.message?.content || '';
+ if (firstTokenMs === null && (piece || event.message?.thinking)) firstTokenMs = performance.now() - start;
+ if (piece) { if (firstOutputMs === null) firstOutputMs = performance.now() - start; text += piece; onToken(piece); }
+ } });
+ if (!last?.done) throw new Error('The model response ended unexpectedly. Partial output was not counted as a successful run.');
+ const elapsedMs = performance.now() - start;
+ return { text, metrics: {
+ model, ...responseMetrics(last), firstTokenMs, firstOutputMs, elapsedMs,
+ context, maxTokens, temperature, seed, thinking: body.think ?? null,
+ } };
+}
+
+export function responseMetrics(last) {
+ return {
+ tokens: last.eval_count ?? null,
+ promptTokens: last.prompt_eval_count ?? null,
+ tokensPerSecond: last.eval_duration > 0 && Number.isFinite(last.eval_count) ? last.eval_count / (last.eval_duration / 1e9) : null,
+ promptTokensPerSecond: last.prompt_eval_duration > 0 && Number.isFinite(last.prompt_eval_count) ? last.prompt_eval_count / (last.prompt_eval_duration / 1e9) : null,
+ loadMs: Number.isFinite(last.load_duration) ? last.load_duration / 1e6 : null,
+ evalMs: Number.isFinite(last.eval_duration) ? last.eval_duration / 1e6 : null,
+ doneReason: last.done_reason || null,
+ };
+}
+
+export async function findOllama() {
+ const candidates = [process.env.OLLAMA_BIN, '/usr/local/bin/ollama', '/opt/homebrew/bin/ollama', '/Applications/Ollama.app/Contents/Resources/ollama', ...String(process.env.PATH || '').split(':').map(p => join(p, 'ollama'))].filter(Boolean);
+ for (const candidate of candidates) {
+ try { await access(candidate, constants.X_OK); return candidate; } catch {}
+ }
+ return null;
+}
+
+export async function ollamaJSON(path, { method = 'GET', body, timeout = 5000, signal } = {}) {
+ const response = await fetch(`${OLLAMA_URL}${path}`, {
+ method, headers: { 'Content-Type': 'application/json' },
+ body: body === undefined ? undefined : JSON.stringify(body),
+ signal: signal ? AbortSignal.any([signal, AbortSignal.timeout(timeout)]) : AbortSignal.timeout(timeout),
+ });
+ const result = await response.json();
+ if (!response.ok || result.error) throw new Error(result.error || `Ollama returned ${response.status}`);
+ return result;
+}
+
+export async function discover() {
+ const binary = await findOllama();
+ try {
+ const [version, tags, running] = await Promise.all([ollamaJSON('/api/version'), ollamaJSON('/api/tags'), ollamaJSON('/api/ps')]);
+ return {
+ id: 'ollama', name: 'Ollama', online: true, installed: Boolean(binary), version: version.version, endpoint: OLLAMA_URL,
+ models: (tags.models || []).map(m => ({
+ id: m.name, name: m.name, size: m.size, digest: m.digest, modifiedAt: m.modified_at,
+ family: m.details?.family, parameters: m.details?.parameter_size, quantization: m.details?.quantization_level,
+ running: Boolean((running.models || []).find(r => r.name === m.name || r.model === m.name)),
+ resident: (running.models || []).find(r => r.name === m.name || r.model === m.name) || null,
+ })).filter(m => !/(?:^|[-:])cloud(?:$|[-:])/i.test(m.name)),
+ };
+ } catch (error) {
+ return { id: 'ollama', name: 'Ollama', online: false, installed: Boolean(binary), version: null, endpoint: OLLAMA_URL, models: [], error: error.message };
+ }
+}
diff --git a/store/agents/vllm/src/server.mjs b/store/agents/vllm/src/server.mjs
new file mode 100644
index 000000000..06984c602
--- /dev/null
+++ b/store/agents/vllm/src/server.mjs
@@ -0,0 +1,143 @@
+import http from 'node:http';
+import { readFile, writeFile, rename } from 'node:fs/promises';
+import { fileURLToPath, pathToFileURL } from 'node:url';
+import { join, dirname, resolve } from 'node:path';
+import { randomBytes, timingSafeEqual } from 'node:crypto';
+import { Store } from './store.mjs';
+import { Controller } from './controller.mjs';
+import { createViewerRelay } from './viewer-relay.mjs';
+
+export const ROOT = dirname(dirname(fileURLToPath(import.meta.url)));
+const assets = { '/': ['index.html', 'text/html'], '/app.js': ['app.js', 'text/javascript'], '/styles.css': ['styles.css', 'text/css'], '/assets/ollama-icon.svg': ['assets/ollama-icon.svg', 'image/svg+xml'], '/assets/ollama-icon.png': ['assets/ollama-icon.png', 'image/png'], '/favicon.ico': ['assets/ollama-icon.png', 'image/png'], '/assets/mlx-icon.svg': ['assets/mlx-icon.svg', 'image/svg+xml'], '/assets/mlx-icon.png': ['assets/mlx-icon.png', 'image/png'] };
+assets['/assets/vllm-icon.png'] = ['assets/vllm-icon.png', 'image/png'];
+const json = (res, status, value) => { res.writeHead(status, { 'Content-Type': 'application/json; charset=utf-8' }); res.end(JSON.stringify(value)); };
+
+async function readBody(req) {
+ if (!/^application\/json(?:\s*;|$)/i.test(req.headers['content-type'] || '')) throw Object.assign(new Error('Use application/json for control requests.'), { status: 415 });
+ if (Number(req.headers['content-length']) > 65536) throw Object.assign(new Error('Request too large.'), { status: 413 });
+ const chunks = []; let size = 0;
+ for await (const chunk of req) { size += chunk.length; if (size > 65536) throw Object.assign(new Error('Request too large.'), { status: 413 }); chunks.push(chunk); }
+ try { return JSON.parse(Buffer.concat(chunks).toString('utf8')); } catch { throw Object.assign(new Error('Invalid JSON request.'), { status: 400 }); }
+}
+
+export function createServer(controller, { assetRoot = join(ROOT, 'dist') } = {}) {
+ const token = randomBytes(32).toString('hex');
+ const events = new Set();
+ const sendEvent = (res, type, data) => { if (res.writableLength > 1024 * 1024) { res.destroy(); events.delete(res); } else if (!res.destroyed) res.write(`event: ${type}\ndata: ${JSON.stringify(data)}\n\n`); };
+ const onState = state => { for (const res of events) sendEvent(res, 'state', state); };
+ const onJob = job => { for (const res of events) sendEvent(res, 'job', job); };
+ controller.on('state', onState); controller.on('job', onJob);
+ const server = http.createServer(async (req, res) => {
+ try {
+ const port = server.address()?.port;
+ const hosts = [`127.0.0.1:${port}`, `localhost:${port}`];
+ if (!hosts.includes(req.headers.host)) return json(res, 403, { error: 'Local requests only.' });
+ const origin = `http://${req.headers.host}`;
+ if (req.headers.origin && req.headers.origin !== origin || req.headers['sec-fetch-site'] === 'cross-site') return json(res, 403, { error: 'Cross-origin control is disabled.' });
+ const url = new URL(req.url, origin);
+ res.setHeader('X-Content-Type-Options', 'nosniff');
+ res.setHeader('Referrer-Policy', 'no-referrer');
+ res.setHeader('Cache-Control', 'no-store');
+ res.setHeader('Content-Security-Policy', "default-src 'self'; script-src 'self'; style-src 'self' 'unsafe-inline'; connect-src 'self'; img-src 'self' data:; frame-ancestors 'self'; base-uri 'none'; form-action 'self'");
+ if (req.method === 'GET' && url.pathname === '/api/health') return json(res, 200, { application: 'local-ai-harnesses', protocol: 1, workspace: controller.workspace, runtime: controller.profile?.id || 'ollama', pid: process.pid });
+ if (req.method === 'POST') {
+ const submitted = Buffer.from(String(req.headers['x-harness-token'] || ''));
+ const expected = Buffer.from(token);
+ if (submitted.length !== expected.length || !timingSafeEqual(submitted, expected)) return json(res, 403, { error: 'Refresh the viewer before sending a control request.' });
+ const body = await readBody(req);
+ if (!body || typeof body !== 'object' || Array.isArray(body)) return json(res, 400, { error: 'Expected a JSON object.' });
+ if (url.pathname === '/api/command') return json(res, 200, await controller.command(body.text, { selectedModel: body.selectedModel, history: body.history }));
+ if (url.pathname === '/api/action') return json(res, 202, { kind: 'job', job: await controller.submit(body) });
+ if (url.pathname === '/api/cancel') return json(res, 200, { job: await controller.cancel(body.id) });
+ }
+ if (req.method === 'GET' && url.pathname === '/api/state') { await controller.refresh(); return json(res, 200, { ...controller.snapshot(), token }); }
+ if (req.method === 'GET' && url.pathname === '/api/jobs') return json(res, 200, { jobs: controller.snapshot().jobs });
+ if (req.method === 'GET' && url.pathname === '/api/events') {
+ if (events.size >= 32) return json(res, 503, { error: 'Too many viewer connections.' });
+ res.writeHead(200, { 'Content-Type': 'text/event-stream', Connection: 'keep-alive', 'X-Accel-Buffering': 'no' });
+ res.write('retry: 2000\n\n'); events.add(res); sendEvent(res, 'state', controller.snapshot());
+ req.on('close', () => events.delete(res)); return;
+ }
+ if (req.method === 'GET' && url.pathname === '/api/export') {
+ res.setHeader('Content-Disposition', `attachment; filename="${controller.profile?.id || 'ollama'}-benchmarks.json"`);
+ return json(res, 200, { exportedAt: new Date().toISOString(), system: controller.system, benchmarks: controller.snapshot().benchmarks, loadTests: controller.snapshot().loadTests });
+ }
+ if (req.method === 'GET' && Object.hasOwn(assets, url.pathname)) {
+ let [file, type] = assets[url.pathname];
+ if (url.pathname === '/favicon.ico' && controller.profile?.id === 'mlx-lm') file = 'assets/mlx-icon.png';
+ if (url.pathname === '/favicon.ico' && controller.profile?.id === 'vllm') file = 'assets/vllm-icon.png';
+ const data = await readFile(join(assetRoot, file));
+ res.writeHead(200, { 'Content-Type': `${type}; charset=utf-8` }); return res.end(data);
+ }
+ json(res, 404, { error: 'Not found.' });
+ } catch (error) { if (!res.headersSent) json(res, error.status || 400, { error: error.message }); else res.end(); }
+ });
+ const heartbeat = setInterval(() => { for (const res of events) res.write(': heartbeat\n\n'); }, 15000); heartbeat.unref();
+ server.on('close', () => { clearInterval(heartbeat); controller.off('state', onState); controller.off('job', onJob); });
+ server.closeEvents = () => { for (const res of events) res.end(); events.clear(); };
+ server.requestTimeout = 30000;
+ return server;
+}
+
+export async function start(port = Number(process.env.HARNESS_VIEWER_PORT || process.env.HARNESS_PORT || 4310), options = {}) {
+ if (!Number.isInteger(port) || port < 0 || port > 65535) throw new Error('The viewer port must be between 0 and 65535.');
+ const workspace = resolve(process.env.HARNESS_WORKSPACE || options.workspaceRoot || ROOT);
+ const runtime = options.profile?.id || 'ollama';
+ let marker;
+ try { marker = JSON.parse(await readFile(join(workspace, 'local-ai.json'), 'utf8')); }
+ catch (error) { if (error.code !== 'ENOENT') throw error; }
+ if (marker?.runtime && marker.runtime !== runtime) throw new Error(`This workspace belongs to ${marker.runtime}. Open a separate ${runtime} workspace.`);
+ let existing;
+ let conflictingRuntime;
+ try {
+ const endpoint = JSON.parse(await readFile(join(workspace, '.harness/endpoint.json'), 'utf8'));
+ if (endpoint.workspace === workspace && Number.isInteger(endpoint.port) && endpoint.port > 0 && endpoint.port <= 65535) {
+ const health = await (await fetch(`http://127.0.0.1:${endpoint.port}/api/health`, { signal: AbortSignal.timeout(1200) })).json();
+ if (health.application === 'local-ai-harnesses' && health.workspace === workspace && health.pid === endpoint.pid) {
+ if ((health.runtime || 'ollama') === runtime) existing = endpoint;
+ else conflictingRuntime = health.runtime || 'ollama';
+ }
+ }
+ } catch {}
+ if (conflictingRuntime) throw new Error(`This workspace is already serving ${conflictingRuntime}. Open a separate ${runtime} workspace.`);
+ if (existing) {
+ if (existing.port === port) throw new Error(`This workspace is already running at http://127.0.0.1:${port}. Open that viewer instead of starting a second copy.`);
+ const relay = createViewerRelay(existing.port);
+ await new Promise((resolve, reject) => { relay.once('error', reject); relay.listen(port, '127.0.0.1', resolve); });
+ console.log(`Local: http://127.0.0.1:${relay.address().port} (shared workspace service)`);
+ const stopRelay = () => { relay.closeAllConnections(); relay.close(); };
+ process.once('SIGINT', stopRelay); process.once('SIGTERM', stopRelay);
+ return { server: relay, controller: null };
+ }
+ const store = await new Store(join(workspace, '.harness')).init();
+ const controller = new Controller(store, options.adapter, options.profile);
+ controller.workspace = workspace;
+ const server = createServer(controller, options);
+ await new Promise((resolve, reject) => { server.once('error', reject); server.listen(port, '127.0.0.1', resolve); });
+ const boundPort = server.address().port;
+ await writeFile(join(store.directory, 'endpoint.json'), JSON.stringify({ port: boundPort, pid: process.pid, workspace, runtime, startedAt: new Date().toISOString() }), { mode: 0o600 });
+ console.log(`Local: http://127.0.0.1:${boundPort}`);
+ let verdictWriting = Promise.resolve();
+ let lastVerdict;
+ const verdict = async snapshot => {
+ const last = snapshot.jobs[0];
+ const failed = last?.status === 'failed';
+ const affected = last?.activeModel || last?.model || last?.models?.join(', ') || controller.profile.name;
+ const failure = failed ? `${affected} · ${last.type} failed: ${last.message}` : undefined;
+ const data = { spec: 1, ready: snapshot.runtime.online && !failed, summary: failed ? failure : snapshot.runtime.online ? `${snapshot.runtime.models.length} local models · ${controller.profile.name} connected` : `${controller.profile.name} is ready to start`, findings: failed ? [{ severity: 'error', kind: 'runtime', message: failure }] : [], updatedAt: new Date().toISOString() };
+ const key = JSON.stringify({ ...data, updatedAt: undefined });
+ if (key === lastVerdict) return;
+ lastVerdict = key;
+ verdictWriting = verdictWriting.catch(() => {}).then(async () => { const temp = join(store.directory, 'verdict.json.tmp'); await writeFile(temp, JSON.stringify(data, null, 2), { mode: 0o600 }); await rename(temp, join(store.directory, 'verdict.json')); });
+ await verdictWriting;
+ };
+ controller.on('state', snapshot => { void verdict(snapshot).catch(() => {}); });
+ controller.on('job', () => { void verdict(controller.snapshot()).catch(() => {}); });
+ await controller.refresh();
+ const poll = setInterval(() => { void controller.refresh().catch(error => console.error('Discovery:', error.message)); }, 5000); poll.unref();
+ let stopping = false;
+ const stop = async () => { if (stopping) return; stopping = true; clearInterval(poll); server.closeEvents(); server.close(); await controller.close(); };
+ process.once('SIGINT', stop); process.once('SIGTERM', stop);
+ return { server, controller };
+}
+if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) await start();
diff --git a/store/agents/vllm/src/store.mjs b/store/agents/vllm/src/store.mjs
new file mode 100644
index 000000000..59957fa1b
--- /dev/null
+++ b/store/agents/vllm/src/store.mjs
@@ -0,0 +1,21 @@
+import { mkdir, readFile, writeFile, rename } from 'node:fs/promises';
+import { join } from 'node:path';
+
+export class Store {
+ constructor(directory) { this.directory = directory; this.file = join(directory, 'state.json'); this.data = { benchmarks: [], loadTests: [], runs: [], jobs: [] }; this.writing = Promise.resolve(); }
+ async init() {
+ await mkdir(this.directory, { recursive: true });
+ try {
+ const parsed = JSON.parse(await readFile(this.file, 'utf8'));
+ for (const key of ['benchmarks', 'loadTests', 'runs', 'jobs']) if (Array.isArray(parsed[key])) this.data[key] = parsed[key];
+ this.data.jobs = this.data.jobs.map(job => ['queued', 'running'].includes(job.status) ? { ...job, status: 'interrupted', message: 'Harness restarted before this job completed. Retry the action to continue.', completedAt: new Date().toISOString() } : { ...job, message: String(job.message || '').replace(/https?:\/\/[^\s"<>]+/g, url => url.split('?')[0]).slice(0, 700) });
+ } catch (error) { if (error.code !== 'ENOENT') throw new Error(`Could not read saved Harness data: ${error.message}`); }
+ await this.save();
+ return this;
+ }
+ save() {
+ const content = JSON.stringify(this.data, null, 2);
+ this.writing = this.writing.catch(() => {}).then(async () => { const temp = `${this.file}.tmp`; await writeFile(temp, content, { mode: 0o600 }); await rename(temp, this.file); });
+ return this.writing;
+ }
+}
diff --git a/store/agents/vllm/src/viewer-relay.mjs b/store/agents/vllm/src/viewer-relay.mjs
new file mode 100644
index 000000000..7aa643ddd
--- /dev/null
+++ b/store/agents/vllm/src/viewer-relay.mjs
@@ -0,0 +1,26 @@
+import http from 'node:http';
+
+// Multiple views of one workspace share one controller and job queue. The native
+// host assigns a fresh viewer port; this loopback relay serves that port without
+// opening a second writer against the workspace's history.
+export function createViewerRelay(targetPort) {
+ if (!Number.isInteger(targetPort) || targetPort < 1 || targetPort > 65535) throw new Error('Invalid local service port.');
+ const server = http.createServer((req, res) => {
+ const port = server.address().port;
+ const host = req.headers.host;
+ const reject = (status, message) => { res.writeHead(status, { 'Content-Type': 'application/json' }); res.end(JSON.stringify({ error: message })); };
+ if (![`localhost:${port}`, `127.0.0.1:${port}`].includes(host)) return reject(403, 'Local requests only.');
+ if (req.headers.origin && req.headers.origin !== `http://${host}` || req.headers['sec-fetch-site'] === 'cross-site') return reject(403, 'Cross-origin control is disabled.');
+ if (!req.url.startsWith('/') || req.url.startsWith('//')) return reject(400, 'Invalid local path.');
+ const headers = { ...req.headers, host: `127.0.0.1:${targetPort}` };
+ if (headers.origin) headers.origin = `http://127.0.0.1:${targetPort}`;
+ const upstream = http.request({ hostname: '127.0.0.1', port: targetPort, path: req.url, method: req.method, headers }, response => {
+ res.writeHead(response.statusCode || 502, response.headers); response.pipe(res);
+ });
+ upstream.on('error', () => { if (!res.headersSent) reject(502, 'The workspace service stopped. Reopen this viewer to reconnect.'); else res.end(); });
+ req.on('aborted', () => upstream.destroy()); res.on('close', () => upstream.destroy());
+ req.pipe(upstream);
+ });
+ server.requestTimeout = 30000;
+ return server;
+}
diff --git a/store/agents/vllm/src/vllm.mjs b/store/agents/vllm/src/vllm.mjs
new file mode 100644
index 000000000..8603b27f4
--- /dev/null
+++ b/store/agents/vllm/src/vllm.mjs
@@ -0,0 +1,218 @@
+import { spawn } from 'node:child_process';
+import { mkdir, open, readFile } from 'node:fs/promises';
+import net from 'node:net';
+import { join } from 'node:path';
+import { randomBytes } from 'node:crypto';
+import { totalmem } from 'node:os';
+import { setTimeout as delay } from 'node:timers/promises';
+import { JsonWorker } from './json-worker.mjs';
+import { MLX_CATALOG, validateMlxModel } from './mlx.mjs';
+import { resolveIntent, validateAction } from './intents.mjs';
+
+export const DEFAULT_SERVING = Object.freeze({ context: 4096, maxSequences: 4, memoryFraction: 0.1, prefixCaching: true });
+const bounded = (value, name, min, max, integer = true) => {
+ if (!Number.isFinite(value) || (integer && !Number.isInteger(value)) || value < min || value > max) throw new Error(`${name} must be ${integer ? 'an integer ' : ''}between ${min} and ${max}.`);
+ return value;
+};
+export function validateServing(input = {}, base = DEFAULT_SERVING) {
+ const config = { ...base };
+ if (input.context !== undefined) config.context = bounded(input.context, 'Context', 512, 16384);
+ if (input.maxSequences !== undefined) config.maxSequences = bounded(input.maxSequences, 'Parallel sequences', 1, 8);
+ if (input.memoryFraction !== undefined) config.memoryFraction = bounded(input.memoryFraction, 'Metal memory fraction', 0.05, 0.8, false);
+ if (input.prefixCaching !== undefined) {
+ if (typeof input.prefixCaching !== 'boolean') throw new Error('Prefix caching must be true or false.');
+ config.prefixCaching = input.prefixCaching;
+ }
+ return config;
+}
+export function validateVllmAction(input) {
+ if (!input || !['serve', 'load_test'].includes(input.type)) return validateAction(input, validateMlxModel);
+ const action = { type: input.type, model: validateMlxModel(input.model) };
+ if (input.type === 'serve') {
+ const allowed = ['type', 'model', ...Object.keys(DEFAULT_SERVING)];
+ if (Object.keys(input).some(key => !allowed.includes(key))) throw new Error('Unknown serving setting.');
+ validateServing(input);
+ for (const key of Object.keys(DEFAULT_SERVING)) if (input[key] !== undefined) action[key] = input[key];
+ } else {
+ if (Object.keys(input).some(key => !['type', 'model', 'concurrency', 'requests'].includes(key))) throw new Error('Unknown load-test setting.');
+ action.concurrency = bounded(input.concurrency ?? 4, 'Concurrency', 1, 8);
+ action.requests = bounded(input.requests ?? 8, 'Requests', action.concurrency, 32);
+ }
+ return action;
+}
+export const VLLM_PROFILE = {
+ id: 'vllm', name: 'vLLM', catalog: MLX_CATALOG, defaultModel: MLX_CATALOG[0].id,
+ preserveServingConfig: true,
+ validateModel: validateMlxModel, validateAction: validateVllmAction,
+ modelPattern: /\b[A-Za-z0-9][A-Za-z0-9_.-]*\/[A-Za-z0-9][A-Za-z0-9_.-]*\b/g,
+ help: 'Try “run a lightweight model”, “benchmark it”, “test 4 concurrent requests”, or “free up memory”. Ask the agent to change context, parallel sequences, Metal memory budget or prefix caching.',
+ benchmarkMode: 'warm server; repeated prompt may use prefix cache; client end-to-end token throughput, including prompt processing and HTTP',
+ readyMessage: 'This workspace owns one vLLM server using the Metal backend. Idle servers stop after 15 minutes.',
+ resolveIntent(text, options) {
+ const match = typeof text === 'string' && text.trim().match(/^(?:please )?(?:test|benchmark) (\d+) concurrent requests[.!?]?$/i);
+ if (match) {
+ const model = options.selectedModel || options.models.find(m => m.running)?.id;
+ return model ? { kind: 'action', action: { type: 'load_test', model, concurrency: Number(match[1]), requests: Math.max(8, Number(match[1])) } } : { kind: 'reply', message: 'Load or select a model first.' };
+ }
+ if (typeof text === 'string' && /^(?:please )?start vllm[.!?]?$/i.test(text.trim())) return { kind: 'action', action: { type: 'start' } };
+ return resolveIntent(text, options);
+ },
+};
+
+// Decode streaming UTF-8 before splitting frames. Chunk boundaries are not tokens.
+export async function readCompletion(body, onToken = () => {}, now = () => performance.now()) {
+ const start = now(); const decoder = new TextDecoder(); let buffer = ''; let text = ''; let usage; let finishReason; let firstTokenMs = null; let firstOutputMs = null; let done = false;
+ const frame = data => {
+ if (data === '[DONE]') { done = true; return; }
+ const event = JSON.parse(data);
+ if (event.error) throw new Error(event.error.message || 'vLLM generation failed.');
+ if (event.usage) usage = event.usage;
+ for (const choice of event.choices || []) {
+ const delta = choice.delta || {};
+ const content = delta.content || '';
+ const reasoning = delta.reasoning_content || delta.reasoning || '';
+ if ((content.length || reasoning.length) && firstTokenMs === null) firstTokenMs = now() - start;
+ if (/\S/.test(content) && firstOutputMs === null) firstOutputMs = now() - start;
+ if (content) { text += content; onToken(content); }
+ if (choice.finish_reason) finishReason = choice.finish_reason;
+ }
+ };
+ const consume = () => {
+ let boundary;
+ while ((boundary = buffer.indexOf('\n\n')) >= 0) {
+ const raw = buffer.slice(0, boundary); buffer = buffer.slice(boundary + 2);
+ const data = raw.split('\n').filter(line => line.startsWith('data:')).map(line => line.slice(5).trimStart()).join('\n');
+ if (data) frame(data);
+ }
+ if (buffer.length > 4 * 1024 * 1024) throw new Error('Oversized vLLM stream frame.');
+ };
+ for await (const bytes of body) { buffer += decoder.decode(bytes, { stream: true }).replace(/\r/g, ''); consume(); }
+ buffer += decoder.decode(); consume();
+ if (!done || !Number.isInteger(usage?.completion_tokens) || !Number.isInteger(usage?.prompt_tokens)) throw new Error('vLLM returned an incomplete stream or missing token usage; no measurement was recorded.');
+ return { text, usage, firstTokenMs, firstOutputMs, finishReason };
+}
+
+async function freePort() {
+ const server = net.createServer();
+ await new Promise((resolve, reject) => { server.once('error', reject); server.listen(0, '127.0.0.1', resolve); });
+ const port = server.address().port;
+ await new Promise((resolve, reject) => server.close(error => error ? reject(error) : resolve()));
+ return port;
+}
+export function parsePrometheus(text) {
+ const sums = {};
+ for (const line of text.split('\n')) {
+ const match = line.match(/^(vllm:[a-z_]+)(?:\{.*\})?\s+([\d.eE+-]+)$/);
+ if (match && Number.isFinite(Number(match[2]))) sums[match[1]] = (sums[match[1]] || 0) + Number(match[2]);
+ }
+ return { runningRequests: sums['vllm:num_requests_running'] ?? null, waitingRequests: sums['vllm:num_requests_waiting'] ?? null, cacheUsage: sums['vllm:kv_cache_usage_perc'] ?? null, prefixQueries: sums['vllm:prefix_cache_queries_total'] ?? null, prefixHits: sums['vllm:prefix_cache_hits_total'] ?? null };
+}
+export class VllmAdapter {
+ constructor({ packageRoot, dataDir, python } = {}) {
+ Object.assign(this, { packageRoot, dataDir }); this.python = python || join(packageRoot, '.venv/bin/python');
+ this.worker = new JsonWorker({ python: this.python, script: join(packageRoot, 'cache.py'), directory: dataDir, name: 'vLLM model cache', env: { HARNESS_MODEL_REGISTRY: join(dataDir, 'vllm-models.json'), HF_HUB_DISABLE_TELEMETRY: '1' } });
+ this.config = { ...DEFAULT_SERVING }; this.server = null; this.loaded = null; this.inflight = 0; this.closed = false; this.expiresAt = null;
+ }
+ async ensureRunning(_directory, { signal } = {}) { signal?.throwIfAborted(); if (this.closed) throw new Error('vLLM is closing.'); await this.worker.start(); signal?.throwIfAborted(); }
+ async discover() {
+ try {
+ await this.ensureRunning();
+ if (this.loaded && !this.inflight && Date.now() > this.expiresAt) await this.stopServer();
+ // Avoid putting inventory requests behind a long model download.
+ if (!this.worker.pending.size) await this.worker.request('inventory', {}, { timeout: 10000 });
+ let rss = null; let metrics = {};
+ if (this.loaded && this.server) {
+ const record = this.server;
+ try {
+ metrics = parsePrometheus(await (await this.fetchNative('/metrics', { signal: AbortSignal.timeout(1500) }, record)).text());
+ if (!this.worker.pending.size) rss = (await this.worker.request('memory', { pid: record.child.pid }, { timeout: 5000 })).rssBytes;
+ } catch {} // Scrape delay does not change server health or invent measurements.
+ }
+ const versions = this.worker.state.versions;
+ const models = (this.worker.state.models || []).map(model => ({ ...model, running: model.id === this.loaded, resident: model.id === this.loaded ? { size: rss, context_length: this.config.context, expires_at: new Date(this.expiresAt).toISOString() } : null }));
+ return { id: 'vllm', name: 'vLLM', online: true, version: versions?.vllm, backend: 'Metal', backendVersion: versions?.['vllm-metal'], mlxVersion: versions?.mlx, models, serving: { ...this.config, status: this.loaded ? 'serving' : this.server ? 'starting' : 'idle', model: this.loaded, processRssBytes: rss, ...metrics } };
+ } catch (error) { return { id: 'vllm', name: 'vLLM', online: false, models: [], error: error.message, backend: 'Metal', serving: { ...this.config, status: 'offline' } }; }
+ }
+ async localModelDetails(model, { signal } = {}) { return this.worker.request('details', { model: validateMlxModel(model) }, { signal }); }
+ async pullModel(model, { signal, onProgress = () => {} } = {}) {
+ return this.worker.request('download', { model: validateMlxModel(model), memoryBudget: Math.floor(totalmem() * 0.8) }, { signal, timeout: 2 * 60 * 60 * 1000, onEvent: event => { if (event.event === 'progress') onProgress(event.message, event.percent, { completed: event.completed, total: event.total }); } });
+ }
+ async fetchNative(path, options = {}, record = this.server) {
+ if (!record) throw new Error('The vLLM server is not running.');
+ const response = await fetch(`http://127.0.0.1:${record.port}${path}`, { ...options, headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${record.key}`, ...options.headers } });
+ if (!response.ok) throw new Error(`vLLM HTTP ${response.status}: ${(await response.text()).slice(0, 700)}`);
+ return response;
+ }
+ async loadModel(model, options = {}) {
+ const config = validateServing(options, this.config); validateMlxModel(model); options.signal?.throwIfAborted();
+ if (this.loaded === model && JSON.stringify(config) === JSON.stringify(this.config)) { this.touch(); return; }
+ if (this.inflight) throw new Error('Wait for current requests before changing the server.');
+ await this.stopServer();
+ if (this.closed) throw new Error('vLLM is closing.');
+ const details = await this.localModelDetails(model, options);
+ const port = await freePort(); const key = randomBytes(32).toString('hex');
+ await mkdir(this.dataDir, { recursive: true });
+ const log = await open(join(this.dataDir, 'vllm.log'), 'a', 0o600);
+ const args = ['-u', join(this.packageRoot, 'server.py'), 'serve', details.path, '--host', '127.0.0.1', '--port', String(port), '--served-model-name', model, '--tokenizer', details.path, '--max-model-len', String(config.context), '--max-num-seqs', String(config.maxSequences), '--max-num-batched-tokens', String(config.context), '--gpu-memory-utilization', String(config.memoryFraction), config.prefixCaching ? '--enable-prefix-caching' : '--no-enable-prefix-caching', '--middleware', 'guard.LocalOnlyMiddleware'];
+ if (details.family === 'qwen3') args.push('--reasoning-parser', 'qwen3');
+ options.signal?.throwIfAborted();
+ const child = spawn(this.python, args, { detached: true, stdio: ['ignore', log.fd, log.fd], cwd: this.packageRoot, env: { ...process.env, PYTHONPATH: this.packageRoot, VLLM_API_KEY: key, HARNESS_VLLM_PORT: String(port), HARNESS_PARENT_PID: String(process.pid), HF_HUB_OFFLINE: '1', HF_HUB_DISABLE_TELEMETRY: '1', TOKENIZERS_PARALLELISM: 'false', VLLM_NO_USAGE_STATS: '1', DO_NOT_TRACK: '1', VLLM_METAL_MEMORY_FRACTION: 'auto', VLLM_MLX_DEVICE: 'gpu' } });
+ const record = { child, port, key, exited: false, error: null };
+ record.exit = new Promise(resolve => { child.once('error', error => { record.error = error; }); child.once('close', () => { record.exited = true; if (child.pid) { try { process.kill(-child.pid, 'SIGKILL'); } catch {} } if (this.server === record) { this.server = null; this.loaded = null; } resolve(); }); });
+ this.server = record; this.config = config; await log.close();
+ const abort = () => { void this.stopServer(record); };
+ options.signal?.addEventListener('abort', abort, { once: true });
+ try {
+ const deadline = Date.now() + 180000;
+ while (Date.now() < deadline) {
+ options.signal?.throwIfAborted();
+ if (record.exited) {
+ const tail = (await readFile(join(this.dataDir, 'vllm.log'), 'utf8')).slice(-2500);
+ throw new Error(`vLLM did not start. ${record.error?.message || tail}`);
+ }
+ try {
+ const response = await this.fetchNative('/v1/models', { signal: AbortSignal.timeout(1000) }, record);
+ const inventory = await response.json();
+ if (inventory.data?.some(item => item.id === model)) { options.signal?.throwIfAborted(); this.loaded = model; this.touch(); return; }
+ } catch (error) { if (options.signal?.aborted) throw error; }
+ await delay(300, undefined, { signal: options.signal });
+ }
+ throw new Error('vLLM startup exceeded three minutes. Check .harness/vllm.log.');
+ } catch (error) { await this.stopServer(record); throw error; }
+ finally { options.signal?.removeEventListener('abort', abort); }
+ }
+ touch() { this.expiresAt = Date.now() + 15 * 60 * 1000; }
+ async stopServer(record = this.server) {
+ if (!record) return;
+ if (record.stopping) return record.stopping;
+ if (this.server === record) this.loaded = null;
+ record.stopping = (async () => {
+ // Only this detached, harness-owned process group is eligible for termination.
+ const kill = signal => { if (!record.child.pid) return; try { process.kill(-record.child.pid, signal); } catch (error) { if (error.code !== 'ESRCH') throw error; } };
+ if (!record.exited) kill('SIGTERM');
+ const timer = setTimeout(() => { try { kill('SIGKILL'); } catch {} }, 4000);
+ try { await record.exit; } finally { clearTimeout(timer); }
+ // The close handler reaps remaining group members once, including after crashes.
+ if (this.server === record) { this.server = null; this.loaded = null; }
+ })();
+ return record.stopping;
+ }
+ async unloadModel(model) { if (this.loaded === model || !this.loaded) await this.stopServer(); }
+ async generate(model, messages, { signal, onToken, maxTokens = 512, temperature = 0.7, seed = 42, thinking = false } = {}) {
+ signal?.throwIfAborted();
+ if (this.loaded !== model) await this.loadModel(model, { signal });
+ const record = this.server; this.inflight++; this.touch();
+ const requestSignal = AbortSignal.any([...(signal ? [signal] : []), AbortSignal.timeout(180000)]);
+ const start = performance.now(); const abort = () => { void this.stopServer(record); };
+ signal?.addEventListener('abort', abort, { once: true });
+ try {
+ const response = await this.fetchNative('/v1/chat/completions', { method: 'POST', signal: requestSignal, body: JSON.stringify({ model, messages, max_tokens: maxTokens, temperature, seed, stream: true, stream_options: { include_usage: true }, chat_template_kwargs: { enable_thinking: thinking } }) }, record);
+ const headersMs = performance.now() - start;
+ const completion = await readCompletion(response.body, onToken);
+ signal?.throwIfAborted();
+ const totalMs = performance.now() - start;
+ return { text: completion.text, metrics: { model, tokens: completion.usage.completion_tokens, promptTokens: completion.usage.prompt_tokens, tokensPerSecond: completion.usage.completion_tokens / (totalMs / 1000), firstTokenMs: completion.firstTokenMs === null ? null : headersMs + completion.firstTokenMs, firstOutputMs: completion.firstOutputMs === null ? null : headersMs + completion.firstOutputMs, totalMs, thinking, doneReason: completion.finishReason, metric: 'client_end_to_end', serving: { ...this.config } } };
+ } finally { signal?.removeEventListener('abort', abort); if (requestSignal.aborted) await this.stopServer(record); this.inflight--; this.touch(); }
+ }
+ async close() { this.closed = true; await this.stopServer(); await this.worker.close(); }
+}
diff --git a/store/agents/vllm/store.json b/store/agents/vllm/store.json
new file mode 100644
index 000000000..0c9dc5d8f
--- /dev/null
+++ b/store/agents/vllm/store.json
@@ -0,0 +1,7 @@
+{
+ "homepage": "https://docs.vllm.ai/projects/vllm-metal/en/latest/",
+ "upstream": "https://github.com/vllm-project/vllm-metal",
+ "license": "MIT",
+ "tagline": "High-performance LLM inference on Apple Silicon using MLX and vLLM",
+ "screenshots": []
+}
diff --git a/store/agents/vllm/template/local-ai.json b/store/agents/vllm/template/local-ai.json
new file mode 100644
index 000000000..5cd705c3f
--- /dev/null
+++ b/store/agents/vllm/template/local-ai.json
@@ -0,0 +1 @@
+{ "spec": 1, "runtime": "vllm", "backend": "metal", "scope": "local-language-models", "notes": "One vLLM harness, with Metal as its Mac backend. This workspace owns one private server. Model weights use the Hugging Face cache; measurements belong to this workspace." }
diff --git a/store/agents/vllm/toolchain/doctor.sh b/store/agents/vllm/toolchain/doctor.sh
new file mode 100755
index 000000000..30b2b6ee2
--- /dev/null
+++ b/store/agents/vllm/toolchain/doctor.sh
@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+set -euo pipefail
+cd "$(dirname "$0")/.."
+. toolchain/node.sh
+"$LOCAL_AI_NODE" -e 'console.log(`ok Node ${process.versions.node}`)'
+if [ ! -x .venv/bin/python ]; then echo 'miss vLLM environment — run toolchain/setup.sh' >&2; exit 1; fi
+VLLM_NO_USAGE_STATS=1 .venv/bin/python - <<'PY'
+import importlib.metadata, platform, sys
+import mlx.core as mx
+from vllm.platforms import current_platform
+from vllm_metal.platform import MetalPlatform
+from vllm_metal.metal import get_ops
+assert platform.machine() == 'arm64' and sys.version_info[:2] == (3, 12), 'Native arm64 Python 3.12 required'
+assert isinstance(current_platform, MetalPlatform), 'vLLM did not select Metal'
+assert mx.metal.is_available(), 'Metal is unavailable'
+key = mx.ones((1, 1, 64), dtype=mx.float16)
+value = key * 2
+kc = mx.zeros((1, 16, 1, 64), dtype=mx.float16)
+vc = mx.zeros_like(kc)
+kc, vc = get_ops().reshape_and_cache(key, value, kc, vc, mx.array([3], dtype=mx.int64))
+mx.eval(kc, vc)
+assert mx.sum(kc).item() == 64 and mx.sum(vc).item() == 128, 'Native Metal cache kernel failed'
+for name in ['vllm', 'vllm-metal', 'mlx', 'mlx-lm']:
+ print('ok ', name, importlib.metadata.version(name))
+print('ok Metal selected · native paged-attention cache kernel passed')
+PY
diff --git a/store/agents/vllm/toolchain/local-ai b/store/agents/vllm/toolchain/local-ai
new file mode 100755
index 000000000..3275734df
--- /dev/null
+++ b/store/agents/vllm/toolchain/local-ai
@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+package_dir="$(cd "$(dirname "$0")/.." && pwd)"
+export HARNESS_WORKSPACE="${HARNESS_WORKSPACE:-$PWD}"
+export LOCAL_AI_RUNTIME=vllm
+export LOCAL_AI_NAME=vLLM
+. "$package_dir/toolchain/node.sh"
+exec "$LOCAL_AI_NODE" "$package_dir/bin/harness.mjs" "$@"
diff --git a/store/agents/vllm/toolchain/node.sh b/store/agents/vllm/toolchain/node.sh
new file mode 100644
index 000000000..df3e4ee05
--- /dev/null
+++ b/store/agents/vllm/toolchain/node.sh
@@ -0,0 +1,15 @@
+# Use this Mac's Node or the interpreter managed by Harness, including GUI launches.
+local_ai_node_candidate="$(command -v node 2>/dev/null || true)"
+if [ -n "$local_ai_node_candidate" ] && "$local_ai_node_candidate" -e 'process.exit(Number(process.versions.node.split(".")[0]) >= 22 ? 0 : 1)' >/dev/null 2>&1; then
+ LOCAL_AI_NODE="$local_ai_node_candidate"
+else
+ local_ai_node_record="${ADAPTER_RUNTIME_DIR:-${HOME}/.harness/runtime}/current-node"
+ local_ai_node_candidate="$(cat "$local_ai_node_record" 2>/dev/null || true)"
+ if [ -n "$local_ai_node_candidate" ] && [ -x "${local_ai_node_candidate%/*}/node" ] && "${local_ai_node_candidate%/*}/node" -e 'process.exit(Number(process.versions.node.split(".")[0]) >= 22 ? 0 : 1)' >/dev/null 2>&1; then
+ LOCAL_AI_NODE="${local_ai_node_candidate%/*}/node"
+ else
+ echo 'miss Node.js 22 or newer — update Harness or install Node.js 22' >&2
+ return 1
+ fi
+fi
+export LOCAL_AI_NODE
diff --git a/store/agents/vllm/toolchain/setup.sh b/store/agents/vllm/toolchain/setup.sh
new file mode 100755
index 000000000..9ed827bf3
--- /dev/null
+++ b/store/agents/vllm/toolchain/setup.sh
@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+set -euo pipefail
+cd "$(dirname "$0")/.."
+if [ "$(uname -s)" != Darwin ] || [ "$(uname -m)" != arm64 ] || [ "$(sw_vers -productVersion | cut -d. -f1)" -lt 15 ]; then
+ echo 'miss vLLM Metal requires Apple Silicon and macOS 15+.' >&2
+ exit 1
+fi
+. toolchain/node.sh
+if [ ! -x .venv/bin/python ]; then
+ vllm_python=/opt/homebrew/opt/[email protected]/bin/python3.12
+ if [ ! -x "$vllm_python" ]; then echo 'miss Native arm64 Python 3.12 is required for these pinned wheels.' >&2; exit 1; fi
+ "$vllm_python" -c 'import platform,sys; assert platform.machine()=="arm64" and sys.version_info[:2]==(3,12)'
+ "$vllm_python" -m venv .venv
+fi
+if [ -x /opt/homebrew/bin/uv ]; then
+ /opt/homebrew/bin/uv pip sync --python .venv/bin/python requirements.lock
+else
+ .venv/bin/python -m pip install -r requirements.lock
+fi
+exec ./toolchain/doctor.sh
diff --git a/store/agents/vllm/toolchain/vllm b/store/agents/vllm/toolchain/vllm
new file mode 100755
index 000000000..880941c3d
--- /dev/null
+++ b/store/agents/vllm/toolchain/vllm
@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+set -euo pipefail
+package_dir="$(cd "$(dirname "$0")/.." && pwd)"
+exec "$package_dir/.venv/bin/vllm" "$@"
diff --git a/store/agents/vllm/viewer-extra.css b/store/agents/vllm/viewer-extra.css
new file mode 100644
index 000000000..285e6baed
--- /dev/null
+++ b/store/agents/vllm/viewer-extra.css
@@ -0,0 +1,4 @@
+
+/* vLLM's blue / gold identity and serving-specific measurements. */
+.runtime-node{border-color:#405e84;background:#1b2b41}.runtime-node::after{background:#527bab}.model-nodes{border-left-color:#3a526d}.model-node{background:#192230}.model-node::before{background:#3a526d}.model-node.active{border-color:#b69643;background:#302b1e}.model-row.selected{background:#1d2b3d}.runtime-map{background-image:radial-gradient(#58749a35 .8px,transparent .8px)}.assistant-panel{background:#131a25}.brand .brand-mark,.runtime-node .node-icon,.assistant-avatar{background:white}.model-title strong,.sample-record strong{overflow-wrap:anywhere}.chart-row>span:first-child{font-size:.75rem}.serving-panel,.concurrency-panel{margin-top:30px}.serving-grid{display:grid;grid-template-columns:repeat(3,minmax(0,1fr));border:1px solid var(--line);border-radius:12px;overflow:hidden;background:var(--surface)}.serving-stat{padding:18px 20px;min-width:0;border-bottom:1px solid var(--line);border-right:1px solid var(--line)}.serving-stat:nth-child(3n){border-right:0}.serving-stat:nth-last-child(-n+3){border-bottom:0}.serving-stat span{display:block;font-size:.75rem;color:var(--muted);line-height:1.5;margin-bottom:10px}.serving-stat strong{font:500 1.1rem var(--mono);overflow-wrap:anywhere}.load-result{padding:20px;border:1px solid var(--line);border-radius:12px;background:var(--surface);margin-top:12px}.load-title{display:flex;justify-content:space-between;gap:12px;align-items:center}.load-title>div{display:flex;align-items:center;gap:12px;min-width:0}.load-title strong{font-weight:500;font-size:.875rem;overflow-wrap:anywhere}.concurrency-badge{flex-shrink:0;color:var(--cyan);background:#213851;border:1px solid #3a5d85;border-radius:7px;padding:7px 10px;font:500 .875rem var(--mono)}.load-bar{display:flex;gap:18px;align-items:center;margin:20px 0 15px}.load-bar .bar-track{flex:1;height:12px}.load-bar>strong{font:400 1.3rem var(--mono);white-space:nowrap}.load-bar small{font-size:.75rem;color:var(--muted)}.load-metrics{display:flex;gap:18px;flex-wrap:wrap;font-size:.75rem;color:var(--muted);line-height:1.8}.load-metrics b{font-family:var(--mono);color:var(--text);font-weight:400}.load-details{margin-top:14px;font-size:.75rem;color:var(--muted);line-height:1.8}.load-details summary{cursor:pointer}.load-details p{margin-top:10px;overflow-wrap:anywhere}.load-title .secondary{white-space:nowrap;font-size:.75rem}
+@media(max-width:600px){.serving-stat{padding:14px 12px}.serving-stat strong{font-size:.95rem}.load-result{padding:15px}.load-title{align-items:flex-start}.load-title>div{gap:8px}.load-title .secondary{display:none}.load-metrics{gap:10px 15px}.serving-panel .actions button{flex:1;font-size:.75rem}.chart-heading>span{font-size:.7rem}}
diff --git a/store/agents/vllm/viewer-extra.js b/store/agents/vllm/viewer-extra.js
new file mode 100644
index 000000000..f607f4f17
--- /dev/null
+++ b/store/agents/vllm/viewer-extra.js
@@ -0,0 +1,32 @@
+function renderServing() {
+ const serving = state.runtime.serving || {};
+ $('#serving-state').textContent = `${serving.status || 'idle'} · Metal ${state.runtime.backendVersion || ''}`;
+ const item = (label, value) => `<div class="serving-stat"><span>${escapeHTML(label)}</span><strong>${escapeHTML(value)}</strong></div>`;
+ $('#serving-content').innerHTML = `<div class="serving-grid">${item('Context', serving.context?.toLocaleString() || '—')}${item('Parallel sequences', serving.maxSequences ?? '—')}${item('Metal memory budget', Number.isFinite(serving.memoryFraction) ? `${Math.round(serving.memoryFraction * 100)}%` : '—')}${item('Prefix cache', serving.prefixCaching === undefined ? '—' : serving.prefixCaching ? 'On' : 'Off')}${item('Requests running / waiting', `${serving.runningRequests ?? '—'} / ${serving.waitingRequests ?? '—'}`)}${item('KV cache used', Number.isFinite(serving.cacheUsage) ? `${(serving.cacheUsage * 100).toFixed(1)}%` : '—')}</div><p class="benchmark-meta">Ask the agent to change these settings. The memory budget uses Metal’s recommended working-set limit. Server telemetry can lag active requests.</p>${serving.model ? `<div class="actions"><button data-load-concurrency="1" data-target="${escapeHTML(serving.model)}">Measure 1 request at a time</button><button data-load-concurrency="4" data-target="${escapeHTML(serving.model)}">Measure 4 concurrent requests</button></div>` : ''}`;
+ $('#model-memory').innerHTML = `${Number.isFinite(serving.processRssBytes) ? gib(serving.processRssBytes) : '—'} <small>GiB</small>`;
+ $('#model-memory').title = 'Summed server process RSS. Shared pages may be counted more than once; this is not GPU allocation.';
+ $('#last-speed').title += ' · client end-to-end throughput';
+}
+let loadTestSignature = '';
+function renderLoadTests() {
+ const results = (state.loadTests || []).slice(0, 6);
+ const signature = JSON.stringify(results.map(test => test.id));
+ if (signature === loadTestSignature) return;
+ loadTestSignature = signature;
+ if (!results.length) {
+ $('#concurrency-content').innerHTML = '<div class="empty-small"><span class="empty-icon" aria-hidden="true">⇉</span><p>See what happens when requests overlap.</p><span>Run a model, then ask “test 4 concurrent requests”. Real throughput, latency and errors appear here.</span></div>';
+ return;
+ }
+ const max = Math.max(1, ...results.map(test => test.aggregateTokensPerSecond));
+ $('#concurrency-content').innerHTML = results.map(test => `<article class="load-result"><div class="load-title"><div><span class="concurrency-badge">${test.concurrency}×</span><strong>${escapeHTML(test.model.split('/').at(-1))}</strong></div><span class="secondary">${new Date(test.at).toLocaleTimeString([], { hour: '2-digit', minute: '2-digit' })}</span></div><div class="load-bar"><div class="bar-track"><div class="bar-fill" style="width:${Math.max(0, test.aggregateTokensPerSecond / max * 100)}%"></div></div><strong>${number(test.aggregateTokensPerSecond)} <small>tok/s</small></strong></div><div class="load-metrics"><span><b>${number(test.medianLatencyMs, 0)}</b> ms median</span><span><b>${number(test.p95LatencyMs, 0)}</b> ms p95</span><span><b>${test.succeeded}/${test.requests}</b> completed</span><span class="${test.failed ? 'error-text' : ''}"><b>${test.failed}</b> errors</span></div><details class="load-details"><summary>Settings &amp; samples</summary><p>${test.serving.context.toLocaleString()} context · ${test.serving.maxSequences} parallel sequences · ${Math.round(test.serving.memoryFraction * 100)}% Metal budget · prefix cache ${test.serving.prefixCaching ? 'on' : 'off'}<br>${escapeHTML(test.mode)}<br>${new Date(test.at).toLocaleString()} · vLLM ${escapeHTML(test.runtime.version)} · Metal ${escapeHTML(test.runtime.backendVersion)}</p>${test.samples.map(sample => `<p>Request ${sample.index + 1}: ${sample.status === 'succeeded' ? `${sample.tokens} tokens · ${number(sample.totalMs, 0)} ms · ${number(sample.firstTokenMs, 0)} ms first token` : escapeHTML(sample.error)}</p>`).join('')}</details></article>`).join('') + '<p class="benchmark-meta">Aggregate output tokens ÷ measured batch time. Latency includes queueing, prompt processing and HTTP. Compare identical model and serving settings; shared prefixes may be cached. These small samples are exploratory. Raw samples are included in Export results.</p>';
+}
+document.addEventListener('click', event => {
+ const button = event.target.closest('button[data-load-concurrency]');
+ if (!button) return;
+ void api('/api/action', { type: 'load_test', model: button.dataset.target, concurrency: Number(button.dataset.loadConcurrency), requests: 8 }).then(result => receiveJob(result.job)).catch(error => toast(error.message));
+});
+if (document.modelContext?.registerTool) {
+ try {
+ await document.modelContext.registerTool({ name: 'vllm_serving_action', title: 'Configure or measure vLLM serving', description: 'Apply the serving settings shown in this viewer, or run a bounded concurrent request test. Returns a job ID.', inputSchema: { type: 'object', properties: { type: { enum: ['serve', 'load_test'] }, model: { type: 'string' }, context: { type: 'integer', minimum: 512, maximum: 16384 }, maxSequences: { type: 'integer', minimum: 1, maximum: 8 }, memoryFraction: { type: 'number', minimum: 0.05, maximum: 0.8 }, prefixCaching: { type: 'boolean' }, concurrency: { type: 'integer', minimum: 1, maximum: 8 }, requests: { type: 'integer', minimum: 1, maximum: 32 } }, required: ['type', 'model'], additionalProperties: false }, annotations: { readOnlyHint: false, untrustedContentHint: true }, async execute(input) { if (!['serve', 'load_test'].includes(input?.type)) throw new Error('Unsupported serving action.'); const result = await api('/api/action', input); receiveJob(result.job); return { jobId: result.job.id, status: result.job.status }; } });
+ } catch (error) { console.info('vLLM WebMCP unavailable:', error.message); }
+}
diff --git a/store/agents/vllm/viewer.sh b/store/agents/vllm/viewer.sh
new file mode 100755
index 000000000..ff6ca35df
--- /dev/null
+++ b/store/agents/vllm/viewer.sh
@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+set -euo pipefail
+cd "$(dirname "$0")"
+. toolchain/node.sh
+exec "$LOCAL_AI_NODE" main.mjs
diff --git a/store/tools/local-ai/.gitignore b/store/tools/local-ai/.gitignore
new file mode 100644
index 000000000..62ed112c9
--- /dev/null
+++ b/store/tools/local-ai/.gitignore
@@ -0,0 +1,11 @@
+.harness/
+.venv/
+.runtime/
+__pycache__/
+*.pyc
+*.log
+node_modules/
+.DS_Store
+!/dist/
+!/dist/**
+evidence/
diff --git a/store/tools/local-ai/README.md b/store/tools/local-ai/README.md
new file mode 100644
index 000000000..6ddd4472a
--- /dev/null
+++ b/store/tools/local-ai/README.md
@@ -0,0 +1,33 @@
+# Local AI harness development
+
+Shared source for the independently installable [Ollama](../../agents/ollama),
+[MLX-LM](../../agents/mlx-lm), and [vLLM](../../agents/vllm) packages. Each package
+contains its own complete runtime and branded viewer. Installing one folder does
+not require this development folder or either of the other harnesses.
+
+`src/` owns the queue, runtime adapters, measurements, persistence and loopback
+control API; `bin/` owns the agent's typed command runner; `dist/` is the source
+HTML/CSS/JavaScript for the shared viewer. MLX-LM and vLLM build scripts specialize
+its framework identity and telemetry. Their workers, dependency locks, agent
+instructions and setup scripts live in their respective package directories.
+
+From this directory, with Node 22+:
+
+```sh
+npm run build # regenerate all three packages before committing
+npm run check
+npm test # fixture and standalone-package checks; no model downloads
+```
+
+Generated `src`, `bin`, `dist`, and `toolchain/node.sh` files are committed inside
+the packages because the Store fetches only the selected package folder. Edit the
+shared originals here, then rebuild; edit framework workers and instructions in
+the package itself. No npm dependencies or global Python changes are required.
+
+The opt-in `test:live`, `test:mlx-live`, and `test:vllm-live` commands use real
+local models and GPU memory. Read each script before running it: they load,
+benchmark, cancel and unload the small demo models. Start the corresponding
+viewer first for Ollama/MLX-LM. Fresh evidence goes in ignored `evidence/`.
+The vLLM script starts and stops its own workspace server.
+
+See the [dated research and qualification evidence](../../../docs/research/local-ai-harnesses/README.md).
diff --git a/store/tools/local-ai/bin/harness.mjs b/store/tools/local-ai/bin/harness.mjs
new file mode 100755
index 000000000..16b63570d
--- /dev/null
+++ b/store/tools/local-ai/bin/harness.mjs
@@ -0,0 +1,54 @@
+#!/usr/bin/env node
+import { readFile } from 'node:fs/promises';
+import { resolve, join, dirname } from 'node:path';
+import { fileURLToPath } from 'node:url';
+const args = process.argv.slice(2);
+const runtimeName = process.env.LOCAL_AI_NAME || 'Ollama';
+const runtime = process.env.LOCAL_AI_RUNTIME || 'ollama';
+if (!args.length) { console.log(`Talk to the ${runtimeName} agent, or use:\n local-ai "run a lightweight model"\n local-ai status\n local-ai action deploy MODEL\n local-ai action benchmark MODEL [MODEL…]\n local-ai action chat MODEL "PROMPT"\n local-ai action unload MODEL\n local-ai action-json '{"type":"serve","model":"OWNER/REPO","maxSequences":4}' (vLLM)\n local-ai cancel JOB_ID`); process.exit(0); }
+const packageRoot = dirname(dirname(fileURLToPath(import.meta.url)));
+const workspace = resolve(process.env.HARNESS_WORKSPACE || packageRoot);
+let port = Number(process.env.HARNESS_PORT || ({ 'mlx-lm': 4311, vllm: 4312 }[runtime] || 4310));
+try { const endpoint = JSON.parse(await readFile(join(workspace, '.harness', 'endpoint.json'), 'utf8')); if (Number.isInteger(endpoint.port) && endpoint.port > 0 && endpoint.port <= 65535 && endpoint.workspace === workspace) port = endpoint.port; } catch {}
+const base = `http://127.0.0.1:${port}`;
+const get = async path => { const response = await fetch(`${base}${path}`, { signal: AbortSignal.timeout(15000) }); const value = await response.json(); if (!response.ok) throw new Error(value.error || `HTTP ${response.status}`); return value; };
+let currentJob;
+try {
+ const health = await get('/api/health');
+ if (health.application !== 'local-ai-harnesses' || health.workspace !== workspace || (health.runtime || 'ollama') !== runtime) throw new Error(`This endpoint belongs to another workspace or framework. Open the ${runtimeName} viewer for this workspace.`);
+ const state = await get('/api/state');
+ const post = async (path, body) => {
+ const response = await fetch(`${base}${path}`, { method: 'POST', headers: { 'Content-Type': 'application/json', 'X-Harness-Token': state.token }, body: JSON.stringify(body), signal: AbortSignal.timeout(15000) });
+ const value = await response.json(); if (!response.ok) throw new Error(value.error || `HTTP ${response.status}`); return value;
+ };
+ if (args[0] === 'status' && args.length === 1) {
+ const { token, ...publicState } = state; console.log(JSON.stringify(publicState, null, 2)); process.exit(0);
+ }
+ if (args[0] === 'cancel') { const { job } = await post('/api/cancel', { id: args[1] }); console.log(JSON.stringify({ id: job.id, status: job.status, message: job.message }, null, 2)); process.exit(0); }
+ let result;
+ if (args[0] === 'action-json') {
+ if (args.length !== 2) throw new Error('Pass one JSON action object.');
+ result = await post('/api/action', JSON.parse(args[1]));
+ } else if (args[0] === 'action') {
+ const type = args[1];
+ const body = type === 'benchmark' ? { type, models: args.slice(2) } : type === 'chat' ? { type, model: args[2], prompt: args.slice(3).join(' ') } : { type, model: args[2] };
+ result = await post('/api/action', body);
+ } else result = await post('/api/command', { text: args.join(' ') });
+ if (!result.job) { console.log(result.message || JSON.stringify(result, null, 2)); if (result.models) console.log(result.models.map(m => `${m.id}: ${m.description}`).join('\n')); process.exit(0); }
+ currentJob = result.job.id;
+ console.log(`Job ${currentJob}`);
+ process.once('SIGINT', async () => { try { await post('/api/cancel', { id: currentJob }); console.log('Cancellation requested.'); } finally { process.exit(130); } });
+ let last;
+ while (true) {
+ const { jobs } = await get('/api/jobs');
+ const job = jobs.find(j => j.id === currentJob);
+ if (!job) throw new Error('The job could not be found. Check the viewer.');
+ if (job.message !== last) { console.log(job.message); last = job.message; }
+ if (!['queued', 'running'].includes(job.status)) {
+ if (job.output) console.log(job.output);
+ if (job.result?.metrics) console.log(JSON.stringify(job.result.metrics, null, 2));
+ process.exit(job.status === 'succeeded' ? 0 : 1);
+ }
+ await new Promise(resolve => setTimeout(resolve, 1000));
+ }
+} catch (error) { console.error(error.cause?.code === 'ECONNREFUSED' ? `The viewer is not running. Open the ${runtimeName} harness viewer.` : `${error.message}${currentJob ? ` (job ${currentJob}; check its status before retrying)` : ''}`); process.exitCode = 1; }
diff --git a/store/tools/local-ai/dist/app.js b/store/tools/local-ai/dist/app.js
new file mode 100644
index 000000000..d1b2e6e45
--- /dev/null
+++ b/store/tools/local-ai/dist/app.js
@@ -0,0 +1,209 @@
+const $ = selector => document.querySelector(selector);
+const escapeHTML = value => String(value ?? '').replace(/[&<>"']/g, c => ({ '&': '&amp;', '<': '&lt;', '>': '&gt;', '"': '&quot;', "'": '&#39;' }[c]));
+const gib = bytes => ((bytes || 0) / 1024 ** 3).toFixed(1);
+const number = (value, digits = 1) => Number.isFinite(value) ? value.toFixed(digits) : '—';
+const active = job => ['queued', 'running'].includes(job.status);
+let state;
+let selectedModel = sessionStorage.getItem('harness-model') || '';
+let pending = false;
+let modelSignature = '';
+let benchmarkSignature = '';
+let initialized = false;
+let lastReceived = Date.now();
+const jobElements = new Map();
+const jobSignatures = new Map();
+const chatHistory = new Map();
+const chatRequests = new Map();
+const completed = new Set();
+const embedded = new URLSearchParams(location.search).get('view') === 'grid';
+document.body.classList.toggle('embedded-viewer', embedded);
+
+function toast(text) { const el = $('#toast'); el.textContent = text; el.hidden = false; clearTimeout(toast.timer); toast.timer = setTimeout(() => { el.hidden = true; }, 4500); }
+function scrollConversation() { const el = $('#conversation'); el.scrollTop = el.scrollHeight; }
+function appendMessage(text, user = false) {
+ const el = document.createElement('div'); el.className = user ? 'user-message' : 'assistant-message';
+ if (user) el.textContent = text;
+ else el.innerHTML = `<span class="message-label">OLLAMA</span><p class="message-text">${escapeHTML(text)}</p>`;
+ $('#conversation').append(el); scrollConversation(); return el;
+}
+async function api(path, body) {
+ if (!state?.token) await refresh();
+ const response = await fetch(path, { method: 'POST', headers: { 'Content-Type': 'application/json', 'X-Harness-Token': state?.token || '' }, body: JSON.stringify(body) });
+ const result = await response.json();
+ if (!response.ok) { if (response.status === 403) void refresh(); throw new Error(result.error || 'The request failed.'); }
+ return result;
+}
+async function refresh() {
+ try {
+ const response = await fetch('/api/state'); if (!response.ok) throw new Error('Could not reach the Ollama viewer');
+ applyState(await response.json());
+ } catch (error) { $('#connection-status').textContent = 'Ollama viewer disconnected'; $('#operator-status').textContent = 'Reconnect to send a request'; return false; }
+ return true;
+}
+function applyState(next) {
+ lastReceived = Date.now(); state = { ...state, ...next };
+ if (selectedModel && !state.runtime.models.some(m => m.id === selectedModel)) selectedModel = '';
+ if (!selectedModel) selectedModel = state.runtime.models.find(m => m.running)?.id || '';
+ render();
+ if (!initialized) {
+ [...state.jobs].slice(0, 4).reverse().forEach(job => showJob(job, true)); initialized = true;
+ }
+ for (const job of state.jobs) if (jobElements.has(job.id) || active(job)) showJob(job);
+ renderActiveJob();
+ renderActivity();
+}
+function selectModel(model) {
+ selectedModel = model; sessionStorage.setItem('harness-model', model);
+ $('#selected-model').value = model;
+ $('#operator-status').textContent = model || 'Ollama harness';
+ document.querySelectorAll('[data-model]').forEach(el => el.classList.toggle('selected', el.dataset.model === model));
+}
+function render() {
+ const { runtime, system, benchmarks, runs } = state;
+ const models = runtime.models || [];
+ const loaded = models.filter(m => m.running);
+ const lastJob = state.jobs[0];
+ const notice = $('#runtime-notice');
+ notice.hidden = lastJob?.status !== 'failed';
+ if (!notice.hidden) {
+ const affected = lastJob.activeModel || lastJob.model || lastJob.models?.join(', ') || 'Ollama';
+ const explanation = /tensor.*size overflow/i.test(lastJob.message)
+ ? "Ollama could not read this model’s weights. Other models can still run."
+ : 'The last operation failed. Check the details before retrying.';
+ notice.innerHTML = `<strong>${escapeHTML(affected)} · ${escapeHTML(lastJob.type)} failed</strong><p>${escapeHTML(explanation)}</p><details><summary>Error details</summary><pre>${escapeHTML(lastJob.message)}</pre></details>`;
+ }
+ $('#connection-status').textContent = runtime.online ? 'Ollama connected' : 'Ollama offline';
+ $('#model-count').textContent = runtime.online ? models.length : '—';
+ $('#models-total').textContent = models.length;
+ $('#running-count').textContent = runtime.online ? loaded.length : '—';
+ $('#model-memory').innerHTML = `${runtime.online ? gib(loaded.reduce((sum, m) => sum + (m.resident?.size || 0), 0)) : '—'} <small>GiB</small>`;
+ const recent = runs[0];
+ $('#last-speed').innerHTML = `${number(recent?.tokensPerSecond)} <small>tok/s</small>`;
+ $('#last-speed').title = recent ? `${recent.model} · ${new Date(recent.at).toLocaleString()}` : 'A chat response will appear here';
+ $('#map-status').textContent = runtime.online ? 'LIVE CONNECTION' : 'READY TO START';
+ $('#map-status').classList.toggle('online', runtime.online);
+ $('#machine-info').textContent = `${system.chip} · ${Math.round(system.totalMemory / 1024 ** 3)} GiB`;
+ const signature = JSON.stringify({ online: runtime.online, version: runtime.version, models, selectedModel });
+ if (signature !== modelSignature) {
+ modelSignature = signature;
+ $('#runtime-map').innerHTML = `<div class="runtime-node ${runtime.online ? '' : 'offline'}"><img class="node-icon" src="/assets/ollama-icon.svg" width="42" height="42" alt=""><strong>Ollama</strong><small>${runtime.online ? `v${escapeHTML(runtime.version)}` : 'Not running'}</small></div><div class="model-nodes">${models.length ? models.map(m => `<button class="model-node ${m.running ? 'active' : ''} ${selectedModel === m.id ? 'selected' : ''}" data-model="${escapeHTML(m.id)}"><strong>${escapeHTML(m.name)}</strong><span>${m.running ? 'In memory' : 'On disk'} · ${gib(m.size)} GiB</span></button>`).join('') : `<div class="map-empty empty-node"><strong>Your first Ollama model starts here.</strong>Ask the operator to run your first model.<br>Its connection will appear here.</div>`}</div>`;
+ $('#models-list').innerHTML = models.length ? models.map(m => `<button class="model-row ${selectedModel === m.id ? 'selected' : ''}" data-model="${escapeHTML(m.id)}"><span class="model-title"><strong>${escapeHTML(m.name)}</strong><small>${escapeHTML([m.family, m.parameters, m.quantization].filter(Boolean).join(' · '))}</small></span><span class="model-status ${m.running ? 'loaded' : ''}">${m.running ? 'In memory' : 'On disk'}</span><span class="model-numeric">${gib(m.size)} GiB</span><span class="model-numeric">Ollama</span></button>`).join('') : `<div class="empty-small"><p>${runtime.online ? 'No models downloaded yet.' : 'Start Ollama to discover your models.'}</p><span>Say “run a lightweight model” to get started.</span></div>`;
+ $('#selected-model').innerHTML = `<option value="">Auto · running model</option>${models.map(m => `<option value="${escapeHTML(m.id)}">${escapeHTML(m.name)}</option>`).join('')}`;
+ $('#selected-model').value = selectedModel;
+ $('#operator-status').textContent = selectedModel || 'Ollama harness';
+ }
+ const benchSignature = JSON.stringify(benchmarks.map(b => b.id));
+ if (benchSignature !== benchmarkSignature) { benchmarkSignature = benchSignature; renderBenchmarks(); }
+ $('#export-benchmarks').hidden = benchmarks.length === 0;
+}
+function renderBenchmarks() {
+ const latest = [...new Map([...state.benchmarks].reverse().map(b => [b.model, b])).values()].sort((a, b) => b.medianTokensPerSecond - a.medianTokensPerSecond);
+ if (!latest.length) return;
+ const maxSpeed = Math.max(...latest.map(b => b.medianTokensPerSecond), 1);
+ const maxLatency = Math.max(...latest.map(b => b.medianFirstTokenMs), 1);
+ const bars = (key, max, unit) => latest.map(b => `<div class="chart-row"><span>${escapeHTML(b.model)}</span><div class="bar-track"><div class="bar-fill ${key === 'medianFirstTokenMs' ? 'latency' : ''}" style="width:${Math.max(0, b[key] / max * 100)}%"></div></div><span class="bar-value">${number(b[key], key === 'medianFirstTokenMs' ? 0 : 1)} <small>${unit}</small></span></div>`).join('');
+ $('#performance-content').innerHTML = `<div class="chart-heading"><h3>Generation speed</h3><span>tokens/s · higher is faster</span></div><div class="chart-bars" role="group" aria-label="Measured median generation speed">${bars('medianTokensPerSecond', maxSpeed, '')}</div><div class="chart-heading second-chart"><h3>Time to first token</h3><span>milliseconds · lower is faster</span></div><div class="chart-bars" role="group" aria-label="Measured median time to first token">${bars('medianFirstTokenMs', maxLatency, '')}</div><p class="benchmark-meta">Latest benchmark per model · median of 3 warm runs<br>Same prompt · 2,048-token context · up to 128 generated tokens · seed 42<br>Thinking off when supported; actual mode is recorded per run.<br>Repeated prompts can use the prefix cache. Speed measures inference, not answer quality.</p><details class="benchmark-details"><summary>Samples and measurement details</summary>${latest.map(b => `<div class="sample-record"><strong>${escapeHTML(b.model)}</strong><span>${new Date(b.at).toLocaleString()} · Ollama ${escapeHTML(b.runtime.version)}</span><span>Speed: ${b.samples.map(s => number(s.tokensPerSecond)).join(' / ')} tokens/s</span><span>First token: ${b.samples.map(s => number(s.firstTokenMs, 0)).join(' / ')} ms</span><span>Tokens generated: ${b.samples.map(s => s.tokens).join(' / ')} · thinking: ${escapeHTML(String(b.samples[0].thinking ?? 'not reported'))} · allocation: ${gib(b.residentBytes)} GiB</span></div>`).join('')}</details>`;
+}
+function showJob(job, historical = false) {
+ let el = jobElements.get(job.id);
+ if (!el) {
+ el = document.createElement('div'); el.className = 'assistant-message job-message';
+ jobElements.set(job.id, el); $('#conversation').append(el);
+ }
+ const signature = JSON.stringify(job);
+ if (jobSignatures.get(job.id) === signature) return;
+ jobSignatures.set(job.id, signature);
+ const failed = ['failed', 'cancelled', 'interrupted'].includes(job.status);
+ const metrics = job.result?.metrics;
+ const label = job.type === 'chat' ? job.model : 'OLLAMA';
+ el.innerHTML = `<span class="message-label">${escapeHTML(label)} <span class="job-state ${failed ? 'error-text' : ''}">${escapeHTML(job.status)}</span></span><p class="message-text ${failed ? 'error-text' : ''}">${escapeHTML(job.message)}</p>${job.output ? `<div class="stream-output message-text">${escapeHTML(job.output)}</div>` : ''}${metrics ? `<p class="run-metrics">${number(metrics.tokensPerSecond)} tok/s · ${number(metrics.firstTokenMs, 0)} ms first token · ${metrics.tokens ?? '—'} tokens${metrics.thinking ? ` · ${number(metrics.firstOutputMs, 0)} ms to visible output · reasoning on` : ''}${metrics.peakMemoryBytes ? ` · ${gib(metrics.peakMemoryBytes)} GiB peak allocation` : ''}${metrics.doneReason === 'length' ? ' · output limit reached' : ''}</p>` : ''}${job.persistenceError ? '<p class="error-text">Could not save this result to disk.</p>' : ''}${job.status === 'succeeded' && job.type === 'deploy' ? `<div class="actions"><button data-prompt="Benchmark ${escapeHTML(job.model)}">Benchmark it</button><button data-chat="${escapeHTML(job.model)}">Ask a question</button></div>` : ''}${job.status === 'failed' ? '<p class="secondary">You can retry the request after addressing the error.</p>' : ''}`;
+ if (job.status === 'succeeded' && !completed.has(job.id)) {
+ completed.add(job.id);
+ if (!historical && ['deploy', 'chat', 'benchmark'].includes(job.type) && job.result?.model) selectModel(job.result.model);
+ const request = chatRequests.get(job.id);
+ if (request) {
+ const history = chatHistory.get(job.model) || [];
+ history.push({ role: 'user', content: request }, { role: 'assistant', content: job.output });
+ chatHistory.set(job.model, history.slice(-6).map(m => ({ ...m, content: m.content.slice(0, 6000) }))); chatRequests.delete(job.id);
+ }
+ }
+ if (!historical) scrollConversation();
+}
+function renderActiveJob() {
+ const running = state.jobs.find(j => j.status === 'running') || state.jobs.find(j => j.status === 'queued');
+ const panel = $('#job-panel'); panel.hidden = !running;
+ if (!running) return;
+ const queued = state.jobs.filter(j => j.status === 'queued').length;
+ panel.innerHTML = `<div class="job-top"><strong>${escapeHTML(running.type === 'chat' ? 'Generating' : running.type.replaceAll('_', ' '))}</strong><button data-cancel="${escapeHTML(running.id)}">Cancel</button></div><div class="progress-track" role="progressbar" aria-label="Job progress" aria-valuemin="0" aria-valuemax="100" aria-valuenow="${Math.round(running.progress)}"><div class="progress-fill" style="width:${running.progress}%"></div></div><p>${escapeHTML(running.message)}${queued ? ` · ${queued} queued` : ''}</p>`;
+}
+function receiveJob(job) {
+ if (!state) return;
+ state.jobs = [job, ...state.jobs.filter(j => j.id !== job.id)];
+ showJob(job); renderActiveJob(); renderActivity();
+}
+function renderActivity() {
+ const jobs = state.jobs.slice(0, 6);
+ $('#activity-content').innerHTML = jobs.length ? jobs.map(job => `<div class="activity-row"><span class="activity-state ${job.status === 'failed' ? 'error-text' : ''}">${escapeHTML(job.status)}</span><div><strong>${escapeHTML(job.model || job.models.join(', ') || 'Ollama')}</strong><p>${escapeHTML(job.message)}</p></div>${active(job) ? `<button data-cancel="${escapeHTML(job.id)}">Cancel</button>` : '<span class="activity-time">' + new Date(job.completedAt || job.createdAt).toLocaleTimeString([], { hour: '2-digit', minute: '2-digit' }) + '</span>'}</div>`).join('') : '<p class="secondary">Ask the agent to run a model. Its progress will appear here.</p>';
+}
+async function sendCommand(text) {
+ if (pending || !text.trim()) return;
+ pending = true; $('#send-button').disabled = true; $('#suggestions').hidden = true;
+ appendMessage(text, true); $('#command-input').value = '';
+ try {
+ const result = await api('/api/command', { text, selectedModel: selectedModel || undefined, history: chatHistory.get(selectedModel) || [] });
+ if (result.job) {
+ if (result.job.type === 'chat') { const prompt = text.match(/^(?:ask|tell|chat with)\s+.+?\s*:\s*([\s\S]+)$/i)?.[1] || text; chatRequests.set(result.job.id, prompt); }
+ receiveJob(result.job);
+ } else {
+ const el = appendMessage(result.message || 'Choose a model to get started.');
+ if (result.models) el.insertAdjacentHTML('beforeend', result.models.map(m => `<div class="catalog-card"><strong>${escapeHTML(m.name)}</strong><p>${escapeHTML(m.description)}</p><p class="secondary">About ${number(m.downloadGB, 2)} GB to download</p><button data-action="deploy" data-target="${escapeHTML(m.id)}">Run this model</button></div>`).join(''));
+ }
+ } catch (error) { appendMessage(error.message).classList.add('error-text'); }
+ finally { pending = false; $('#send-button').disabled = false; scrollConversation(); }
+}
+async function runAction(type, model) {
+ $('#model-dialog').close();
+ const action = type === 'benchmark' ? { type, models: [model] } : { type, model };
+ appendMessage(`${type === 'deploy' ? 'Run' : type[0].toUpperCase() + type.slice(1)} ${model}`, true);
+ try { const result = await api('/api/action', action); receiveJob(result.job); return { jobId: result.job.id, status: result.job.status }; }
+ catch (error) { appendMessage(error.message).classList.add('error-text'); throw error; }
+}
+function inspectModel(id) {
+ const model = state.runtime.models.find(m => m.id === id); if (!model) return;
+ selectModel(id);
+ const resident = model.resident;
+ $('#model-detail').innerHTML = `<p class="eyebrow">OLLAMA / MODEL DETAILS</p><h2 id="detail-title">${escapeHTML(model.name)}</h2><p class="secondary">${model.running ? 'Loaded and ready for inference' : 'Downloaded and available to load'}</p><div class="detail-grid"><div><span>Model parameters</span><strong>${escapeHTML(model.parameters || 'Unknown')}</strong></div><div><span>Quantization</span><strong>${escapeHTML(model.quantization || 'Unknown')}</strong></div><div><span>Size on disk</span><strong>${gib(model.size)} GiB</strong></div><div><span>Runtime allocation</span><strong>${resident ? `${gib(resident.size)} GiB` : 'Not loaded'}</strong></div><div><span>Context window</span><strong>${resident?.context_length?.toLocaleString() || 'Set when loaded'}</strong></div><div><span>Idle expiry</span><strong>${resident?.expires_at ? new Date(resident.expires_at).toLocaleTimeString() : '—'}</strong></div></div><p class="secondary">Ollama reports runtime allocation. On Apple Silicon, CPU and GPU share unified memory; this is not a separate physical VRAM pool.</p><div class="actions"><button class="primary" data-action="deploy" data-target="${escapeHTML(id)}">${model.running ? 'Keep ready for 15m' : 'Load model'}</button><button data-action="benchmark" data-target="${escapeHTML(id)}">Benchmark</button><button data-chat="${escapeHTML(id)}">Chat</button>${model.running ? `<button data-action="unload" data-target="${escapeHTML(id)}">Unload</button>` : ''}</div>`;
+ $('#model-dialog').showModal();
+}
+
+$('#refresh').addEventListener('click', async () => { if (await refresh()) toast('Model status refreshed'); });
+$('#selected-model').addEventListener('change', event => selectModel(event.target.value));
+$('#command-form').addEventListener('submit', event => { event.preventDefault(); void sendCommand($('#command-input').value); });
+$('#command-input').addEventListener('keydown', event => { if (event.key === 'Enter' && !event.shiftKey && !event.isComposing) { event.preventDefault(); $('#command-form').requestSubmit(); } });
+$('.dialog-close').addEventListener('click', () => $('#model-dialog').close());
+document.addEventListener('click', event => {
+ const button = event.target.closest('button'); if (!button) return;
+ if (button.dataset.prompt) void sendCommand(button.dataset.prompt);
+ if (button.dataset.model) inspectModel(button.dataset.model);
+ if (button.dataset.action) void runAction(button.dataset.action, button.dataset.target).catch(() => {});
+ if (button.dataset.chat) { selectModel(button.dataset.chat); $('#model-dialog').close(); $('#command-input').value = 'Ask it: '; $('#command-input').focus(); }
+ if (button.dataset.cancel) void api('/api/cancel', { id: button.dataset.cancel }).then(result => receiveJob(result.job)).catch(error => toast(error.message));
+});
+
+await refresh();
+const events = new EventSource('/api/events');
+events.addEventListener('state', event => { try { applyState(JSON.parse(event.data)); } catch { void refresh(); } });
+events.addEventListener('job', event => { lastReceived = Date.now(); receiveJob(JSON.parse(event.data)); });
+events.addEventListener('error', () => { $('#connection-status').textContent = 'Reconnecting…'; });
+setInterval(() => { if (Date.now() - lastReceived > 12000) void refresh(); }, 10000);
+
+// The same visible UI actions are available to supported agent browsers.
+const lifecycle = new AbortController();
+if (document.modelContext?.registerTool) {
+ const register = tool => document.modelContext.registerTool(tool, { signal: lifecycle.signal });
+ try {
+ await register({ name: 'harness_inventory', title: 'Inspect local models', description: 'Read Ollama models, residency, recent benchmarks and job status from this viewer.', inputSchema: { type: 'object', properties: {}, additionalProperties: false }, annotations: { readOnlyHint: true, untrustedContentHint: true }, async execute() { if (!await refresh()) throw new Error('The workspace service is disconnected.'); return { runtime: state.runtime, benchmarks: state.benchmarks, jobs: state.jobs.map(({ id, type, model, status, message }) => ({ id, type, model, status, message })) }; } });
+ await register({ name: 'harness_start_action', title: 'Control a local model', description: 'Start one local Ollama action. Deploy downloads a missing model. Benchmark runs three warm measurements. Returns a job ID; inspect inventory to see completion.', inputSchema: { type: 'object', properties: { type: { type: 'string', enum: ['deploy', 'download', 'unload', 'benchmark'] }, model: { type: 'string', description: 'Exact local Ollama model name and tag.' } }, required: ['type', 'model'], additionalProperties: false }, annotations: { readOnlyHint: false, untrustedContentHint: true }, async execute(input) { if (!input || !['deploy', 'download', 'unload', 'benchmark'].includes(input.type) || typeof input.model !== 'string') throw new Error('Provide an allowed action and model.'); return runAction(input.type, input.model); } });
+ } catch (error) { console.info('WebMCP tools unavailable:', error.message); }
+}
+window.addEventListener('pagehide', () => { lifecycle.abort(); events.close(); });
diff --git a/store/tools/local-ai/dist/assets/README.md b/store/tools/local-ai/dist/assets/README.md
new file mode 100644
index 000000000..5ca42176a
--- /dev/null
+++ b/store/tools/local-ai/dist/assets/README.md
@@ -0,0 +1,3 @@
+Ollama logo source: https://github.com/ollama/ollama/blob/main/docs/ollama-logo.svg
+Retrieved 2026-09-19. The official path is preserved in ollama.svg.
+ollama-icon.svg packages that mark on a light rounded tile for dark and light app chrome.
diff --git a/store/tools/local-ai/dist/assets/ollama-icon.png b/store/tools/local-ai/dist/assets/ollama-icon.png
new file mode 100644
index 000000000..130cb9ba4
Binary files /dev/null and b/store/tools/local-ai/dist/assets/ollama-icon.png differ
diff --git a/store/tools/local-ai/dist/assets/ollama-icon.svg b/store/tools/local-ai/dist/assets/ollama-icon.svg
new file mode 100644
index 000000000..24a21f56a
--- /dev/null
+++ b/store/tools/local-ai/dist/assets/ollama-icon.svg
@@ -0,0 +1,3 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="512" height="512" viewBox="0 0 36 36"><rect width="36" height="36" rx="9" fill="#f4f6ef"/><g transform="translate(9.5 5.5)">
+<path fill-rule="evenodd" clip-rule="evenodd" d="M4.40517 0.102088C4.62117 0.198678 4.81617 0.357766 4.99317 0.56799C5.28817 0.915712 5.53718 1.41342 5.72718 2.00318C5.91818 2.59635 6.04218 3.25316 6.08918 3.91224C6.71878 3.5075 7.41754 3.26103 8.13818 3.18953L8.18918 3.18498C9.05919 3.10544 9.91919 3.28384 10.6692 3.72361C10.7702 3.78384 10.8692 3.84861 10.9662 3.91679C11.0162 3.27021 11.1382 2.62817 11.3262 2.04863C11.5162 1.45773 11.7652 0.961166 12.0592 0.612308C12.2235 0.410338 12.4245 0.251368 12.6482 0.146406C12.9052 0.032771 13.1782 0.0123167 13.4442 0.098679C13.8452 0.228223 14.1892 0.516855 14.4602 0.936167C14.7082 1.3191 14.8942 1.81 15.0212 2.39863C15.2512 3.45998 15.2912 4.85655 15.1362 6.54061L15.1892 6.58607L15.2152 6.60766C15.9722 7.26219 16.4992 8.19513 16.7782 9.27807C17.2133 10.9678 16.9943 12.8632 16.2442 13.9235L16.2262 13.9473L16.2282 13.9507C16.6453 14.8166 16.8983 15.7314 16.9523 16.678L16.9543 16.7121C17.0183 17.9223 16.7542 19.1404 16.1402 20.337L16.1332 20.3484L16.1432 20.3756C16.6152 21.6904 16.7632 23.0142 16.5812 24.3369L16.5752 24.3813C16.547 24.5744 16.4525 24.7472 16.3125 24.8612C16.1725 24.9753 15.9983 25.0219 15.8282 24.9903C15.744 24.9753 15.6632 24.9417 15.5904 24.8912C15.5177 24.8408 15.4544 24.7744 15.4042 24.696C15.3541 24.6178 15.318 24.529 15.2981 24.4347C15.2782 24.3406 15.2748 24.2428 15.2882 24.1472C15.4552 22.9733 15.2982 21.7961 14.8082 20.5984C14.7625 20.4871 14.7422 20.3645 14.7492 20.242C14.7562 20.1194 14.7902 20.0009 14.8482 19.8972L14.8522 19.8904C15.4562 18.8404 15.7062 17.8109 15.6522 16.7996C15.6062 15.9143 15.3272 15.045 14.8522 14.2166C14.7598 14.0556 14.7269 13.8597 14.7606 13.6713C14.7943 13.4829 14.8918 13.3171 15.0322 13.2098L15.0412 13.203C15.2842 13.0223 15.5082 12.561 15.6212 11.9303C15.7459 11.1846 15.7133 10.4159 15.5262 9.68716C15.3212 8.89171 14.9462 8.22809 14.4212 7.77468C13.8262 7.25878 13.0382 7.00992 12.0412 7.08151C11.9108 7.09115 11.7809 7.05613 11.6682 6.98097C11.5556 6.90581 11.4653 6.79399 11.4092 6.65993C11.0952 5.90426 10.6372 5.36336 10.0662 5.02814C9.51799 4.71723 8.90425 4.58657 8.29418 4.65087C7.04918 4.76337 5.95118 5.56108 5.62418 6.56675C5.57792 6.70829 5.4947 6.8304 5.38568 6.91672C5.27666 7.00301 5.14703 7.04942 5.01417 7.0497C3.94717 7.05197 3.12117 7.33606 2.51717 7.84855C1.99517 8.29172 1.63916 8.91103 1.45116 9.65307C1.28104 10.3515 1.25774 11.0857 1.38316 11.7962C1.49516 12.4303 1.71416 12.9553 1.96517 13.2382L1.97317 13.2462C2.18517 13.4814 2.23017 13.8485 2.08217 14.1382C1.72216 14.845 1.45316 15.8984 1.40916 16.9109C1.35916 18.0677 1.59516 19.0722 2.12817 19.7927L2.14417 19.8143C2.22461 19.9208 2.27633 20.0514 2.29319 20.1905C2.31003 20.3295 2.29127 20.4711 2.23917 20.5984C1.66316 22.0029 1.48616 23.1574 1.67716 24.0665C1.71148 24.2556 1.67954 24.4524 1.58812 24.6149C1.4967 24.7776 1.35302 24.8933 1.18766 24.9374C1.0223 24.9817 0.848322 24.9506 0.702741 24.8512C0.557141 24.7517 0.451463 24.5917 0.408163 24.4051C0.165162 23.2483 0.330162 21.9233 0.881162 20.4302L0.895162 20.3904L0.887162 20.3768C0.616341 19.9222 0.414243 19.4195 0.289162 18.8893L0.284162 18.8677C0.132362 18.2062 0.0726416 17.5218 0.107162 16.8393C0.151162 15.8052 0.385163 14.7462 0.729162 13.8962L0.741162 13.8666L0.739162 13.8644C0.446163 13.3894 0.229162 12.7814 0.109162 12.1087L0.104162 12.0814C-0.0611788 11.1431 -0.0293187 10.1737 0.197162 9.25194C0.459163 8.21218 0.974162 7.31901 1.73316 6.67356C1.79316 6.62243 1.85616 6.57129 1.91916 6.52357C1.76016 4.827 1.80016 3.42134 2.03117 2.35317C2.15817 1.76455 2.34517 1.27365 2.59317 0.890713C2.86317 0.472537 3.20717 0.183905 3.60817 0.0532252C3.87417 -0.0331371 4.14817 -0.0126829 4.40517 0.102088ZM8.52118 10.4315C9.45719 10.4315 10.3212 10.7871 10.9672 11.403C11.5972 12.0019 11.9722 12.8064 11.9722 13.6076C11.9722 14.6166 11.5662 15.403 10.8392 15.9052C10.2192 16.3314 9.38819 16.5382 8.43618 16.5382C7.42718 16.5382 6.56518 16.2439 5.94318 15.7041C5.32618 15.17 4.98017 14.42 4.98017 13.6076C4.98017 12.8042 5.37818 11.9973 6.03618 11.3962C6.70418 10.786 7.58618 10.4315 8.52118 10.4315ZM8.52118 11.4496C7.82742 11.4428 7.15204 11.7031 6.60518 12.1883C6.14418 12.6087 5.88318 13.1371 5.88318 13.6087C5.88318 14.095 6.09318 14.5507 6.49318 14.8973C6.94818 15.2916 7.61718 15.52 8.43618 15.52C9.23519 15.52 9.90919 15.353 10.3682 15.0359C10.8312 14.7178 11.0682 14.2564 11.0682 13.6076C11.0682 13.1269 10.8222 12.5962 10.3852 12.1803C9.90119 11.7201 9.24519 11.4496 8.52118 11.4496ZM9.18319 12.8246L9.18719 12.8292C9.30719 13.0007 9.28219 13.2496 9.13119 13.386L8.83919 13.6473V14.1541C8.83865 14.267 8.79877 14.375 8.72829 14.4544C8.6578 14.5339 8.56246 14.5783 8.46318 14.578C8.3639 14.5783 8.26856 14.5339 8.19808 14.4544C8.12758 14.375 8.0877 14.267 8.08718 14.1541V13.6314L7.81618 13.3837C7.78042 13.3511 7.7507 13.3109 7.72872 13.2652C7.70674 13.2195 7.69294 13.1694 7.6881 13.1176C7.68326 13.0658 7.6875 13.0135 7.70056 12.9636C7.71362 12.9137 7.73524 12.8672 7.76418 12.8269C7.8232 12.7452 7.9082 12.6934 8.0007 12.6825C8.09318 12.6717 8.18572 12.7027 8.25818 12.7689L8.47318 12.9644L8.69318 12.7667C8.76538 12.7018 8.85702 12.6716 8.94854 12.6825C9.04009 12.6933 9.12427 12.7443 9.18319 12.8246ZM4.14317 10.644C4.62117 10.644 5.01017 11.0871 5.01017 11.6337C5.01043 11.8957 4.91917 12.1471 4.75641 12.3327C4.59365 12.5183 4.37273 12.6229 4.14217 12.6235C3.91195 12.6226 3.69143 12.518 3.52893 12.3327C3.36641 12.1474 3.27517 11.8965 3.27517 11.6349C3.27463 11.3729 3.36565 11.1213 3.52821 10.9355C3.69079 10.7497 3.91261 10.6449 4.14317 10.644ZM12.8492 10.644C13.3292 10.644 13.7172 11.0871 13.7172 11.6337C13.7175 11.8957 13.6262 12.1471 13.4634 12.3327C13.3007 12.5183 13.0798 12.6229 12.8492 12.6235C12.619 12.6226 12.3985 12.518 12.236 12.3327C12.0734 12.1474 11.9822 11.8965 11.9822 11.6349C11.9817 11.3729 12.0727 11.1213 12.2352 10.9355C12.3978 10.7497 12.6186 10.6449 12.8492 10.644ZM3.94017 1.47705L3.93717 1.47932C3.82131 1.53657 3.72239 1.63046 3.65217 1.74977L3.64717 1.75659C3.50917 1.97136 3.38917 2.28727 3.29917 2.70203C3.12917 3.48839 3.08317 4.55541 3.17517 5.86335C3.60517 5.7179 4.07417 5.62699 4.57917 5.59404L4.58917 5.5929L4.60817 5.55426C4.65417 5.46108 4.70317 5.37131 4.75617 5.28268C4.87917 4.40655 4.77817 3.35998 4.50317 2.50545C4.36917 2.09182 4.20617 1.76682 4.05017 1.5816C4.01797 1.5431 3.98207 1.5088 3.94317 1.47932L3.94017 1.47705ZM13.1142 1.52251L13.1122 1.52364C13.0733 1.55312 13.0374 1.58741 13.0052 1.62591C12.8492 1.81114 12.6852 2.13727 12.5522 2.5509C12.2622 3.45316 12.1652 4.56905 12.3222 5.47358L12.3802 5.58381L12.3882 5.59972H12.4182C12.9145 5.59988 13.4082 5.68101 13.8842 5.84062C13.9702 4.56337 13.9222 3.51907 13.7562 2.74749C13.6662 2.33272 13.5462 2.01682 13.4072 1.80205L13.4032 1.79523C13.3331 1.67548 13.2342 1.58121 13.1182 1.52364L13.1142 1.52251Z" fill="black"/>
+</g></svg>
diff --git a/store/tools/local-ai/dist/assets/ollama.svg b/store/tools/local-ai/dist/assets/ollama.svg
new file mode 100644
index 000000000..b215c89b9
--- /dev/null
+++ b/store/tools/local-ai/dist/assets/ollama.svg
@@ -0,0 +1,3 @@
+<svg width="17" height="25" viewBox="0 0 17 25" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path fill-rule="evenodd" clip-rule="evenodd" d="M4.40517 0.102088C4.62117 0.198678 4.81617 0.357766 4.99317 0.56799C5.28817 0.915712 5.53718 1.41342 5.72718 2.00318C5.91818 2.59635 6.04218 3.25316 6.08918 3.91224C6.71878 3.5075 7.41754 3.26103 8.13818 3.18953L8.18918 3.18498C9.05919 3.10544 9.91919 3.28384 10.6692 3.72361C10.7702 3.78384 10.8692 3.84861 10.9662 3.91679C11.0162 3.27021 11.1382 2.62817 11.3262 2.04863C11.5162 1.45773 11.7652 0.961166 12.0592 0.612308C12.2235 0.410338 12.4245 0.251368 12.6482 0.146406C12.9052 0.032771 13.1782 0.0123167 13.4442 0.098679C13.8452 0.228223 14.1892 0.516855 14.4602 0.936167C14.7082 1.3191 14.8942 1.81 15.0212 2.39863C15.2512 3.45998 15.2912 4.85655 15.1362 6.54061L15.1892 6.58607L15.2152 6.60766C15.9722 7.26219 16.4992 8.19513 16.7782 9.27807C17.2133 10.9678 16.9943 12.8632 16.2442 13.9235L16.2262 13.9473L16.2282 13.9507C16.6453 14.8166 16.8983 15.7314 16.9523 16.678L16.9543 16.7121C17.0183 17.9223 16.7542 19.1404 16.1402 20.337L16.1332 20.3484L16.1432 20.3756C16.6152 21.6904 16.7632 23.0142 16.5812 24.3369L16.5752 24.3813C16.547 24.5744 16.4525 24.7472 16.3125 24.8612C16.1725 24.9753 15.9983 25.0219 15.8282 24.9903C15.744 24.9753 15.6632 24.9417 15.5904 24.8912C15.5177 24.8408 15.4544 24.7744 15.4042 24.696C15.3541 24.6178 15.318 24.529 15.2981 24.4347C15.2782 24.3406 15.2748 24.2428 15.2882 24.1472C15.4552 22.9733 15.2982 21.7961 14.8082 20.5984C14.7625 20.4871 14.7422 20.3645 14.7492 20.242C14.7562 20.1194 14.7902 20.0009 14.8482 19.8972L14.8522 19.8904C15.4562 18.8404 15.7062 17.8109 15.6522 16.7996C15.6062 15.9143 15.3272 15.045 14.8522 14.2166C14.7598 14.0556 14.7269 13.8597 14.7606 13.6713C14.7943 13.4829 14.8918 13.3171 15.0322 13.2098L15.0412 13.203C15.2842 13.0223 15.5082 12.561 15.6212 11.9303C15.7459 11.1846 15.7133 10.4159 15.5262 9.68716C15.3212 8.89171 14.9462 8.22809 14.4212 7.77468C13.8262 7.25878 13.0382 7.00992 12.0412 7.08151C11.9108 7.09115 11.7809 7.05613 11.6682 6.98097C11.5556 6.90581 11.4653 6.79399 11.4092 6.65993C11.0952 5.90426 10.6372 5.36336 10.0662 5.02814C9.51799 4.71723 8.90425 4.58657 8.29418 4.65087C7.04918 4.76337 5.95118 5.56108 5.62418 6.56675C5.57792 6.70829 5.4947 6.8304 5.38568 6.91672C5.27666 7.00301 5.14703 7.04942 5.01417 7.0497C3.94717 7.05197 3.12117 7.33606 2.51717 7.84855C1.99517 8.29172 1.63916 8.91103 1.45116 9.65307C1.28104 10.3515 1.25774 11.0857 1.38316 11.7962C1.49516 12.4303 1.71416 12.9553 1.96517 13.2382L1.97317 13.2462C2.18517 13.4814 2.23017 13.8485 2.08217 14.1382C1.72216 14.845 1.45316 15.8984 1.40916 16.9109C1.35916 18.0677 1.59516 19.0722 2.12817 19.7927L2.14417 19.8143C2.22461 19.9208 2.27633 20.0514 2.29319 20.1905C2.31003 20.3295 2.29127 20.4711 2.23917 20.5984C1.66316 22.0029 1.48616 23.1574 1.67716 24.0665C1.71148 24.2556 1.67954 24.4524 1.58812 24.6149C1.4967 24.7776 1.35302 24.8933 1.18766 24.9374C1.0223 24.9817 0.848322 24.9506 0.702741 24.8512C0.557141 24.7517 0.451463 24.5917 0.408163 24.4051C0.165162 23.2483 0.330162 21.9233 0.881162 20.4302L0.895162 20.3904L0.887162 20.3768C0.616341 19.9222 0.414243 19.4195 0.289162 18.8893L0.284162 18.8677C0.132362 18.2062 0.0726416 17.5218 0.107162 16.8393C0.151162 15.8052 0.385163 14.7462 0.729162 13.8962L0.741162 13.8666L0.739162 13.8644C0.446163 13.3894 0.229162 12.7814 0.109162 12.1087L0.104162 12.0814C-0.0611788 11.1431 -0.0293187 10.1737 0.197162 9.25194C0.459163 8.21218 0.974162 7.31901 1.73316 6.67356C1.79316 6.62243 1.85616 6.57129 1.91916 6.52357C1.76016 4.827 1.80016 3.42134 2.03117 2.35317C2.15817 1.76455 2.34517 1.27365 2.59317 0.890713C2.86317 0.472537 3.20717 0.183905 3.60817 0.0532252C3.87417 -0.0331371 4.14817 -0.0126829 4.40517 0.102088ZM8.52118 10.4315C9.45719 10.4315 10.3212 10.7871 10.9672 11.403C11.5972 12.0019 11.9722 12.8064 11.9722 13.6076C11.9722 14.6166 11.5662 15.403 10.8392 15.9052C10.2192 16.3314 9.38819 16.5382 8.43618 16.5382C7.42718 16.5382 6.56518 16.2439 5.94318 15.7041C5.32618 15.17 4.98017 14.42 4.98017 13.6076C4.98017 12.8042 5.37818 11.9973 6.03618 11.3962C6.70418 10.786 7.58618 10.4315 8.52118 10.4315ZM8.52118 11.4496C7.82742 11.4428 7.15204 11.7031 6.60518 12.1883C6.14418 12.6087 5.88318 13.1371 5.88318 13.6087C5.88318 14.095 6.09318 14.5507 6.49318 14.8973C6.94818 15.2916 7.61718 15.52 8.43618 15.52C9.23519 15.52 9.90919 15.353 10.3682 15.0359C10.8312 14.7178 11.0682 14.2564 11.0682 13.6076C11.0682 13.1269 10.8222 12.5962 10.3852 12.1803C9.90119 11.7201 9.24519 11.4496 8.52118 11.4496ZM9.18319 12.8246L9.18719 12.8292C9.30719 13.0007 9.28219 13.2496 9.13119 13.386L8.83919 13.6473V14.1541C8.83865 14.267 8.79877 14.375 8.72829 14.4544C8.6578 14.5339 8.56246 14.5783 8.46318 14.578C8.3639 14.5783 8.26856 14.5339 8.19808 14.4544C8.12758 14.375 8.0877 14.267 8.08718 14.1541V13.6314L7.81618 13.3837C7.78042 13.3511 7.7507 13.3109 7.72872 13.2652C7.70674 13.2195 7.69294 13.1694 7.6881 13.1176C7.68326 13.0658 7.6875 13.0135 7.70056 12.9636C7.71362 12.9137 7.73524 12.8672 7.76418 12.8269C7.8232 12.7452 7.9082 12.6934 8.0007 12.6825C8.09318 12.6717 8.18572 12.7027 8.25818 12.7689L8.47318 12.9644L8.69318 12.7667C8.76538 12.7018 8.85702 12.6716 8.94854 12.6825C9.04009 12.6933 9.12427 12.7443 9.18319 12.8246ZM4.14317 10.644C4.62117 10.644 5.01017 11.0871 5.01017 11.6337C5.01043 11.8957 4.91917 12.1471 4.75641 12.3327C4.59365 12.5183 4.37273 12.6229 4.14217 12.6235C3.91195 12.6226 3.69143 12.518 3.52893 12.3327C3.36641 12.1474 3.27517 11.8965 3.27517 11.6349C3.27463 11.3729 3.36565 11.1213 3.52821 10.9355C3.69079 10.7497 3.91261 10.6449 4.14317 10.644ZM12.8492 10.644C13.3292 10.644 13.7172 11.0871 13.7172 11.6337C13.7175 11.8957 13.6262 12.1471 13.4634 12.3327C13.3007 12.5183 13.0798 12.6229 12.8492 12.6235C12.619 12.6226 12.3985 12.518 12.236 12.3327C12.0734 12.1474 11.9822 11.8965 11.9822 11.6349C11.9817 11.3729 12.0727 11.1213 12.2352 10.9355C12.3978 10.7497 12.6186 10.6449 12.8492 10.644ZM3.94017 1.47705L3.93717 1.47932C3.82131 1.53657 3.72239 1.63046 3.65217 1.74977L3.64717 1.75659C3.50917 1.97136 3.38917 2.28727 3.29917 2.70203C3.12917 3.48839 3.08317 4.55541 3.17517 5.86335C3.60517 5.7179 4.07417 5.62699 4.57917 5.59404L4.58917 5.5929L4.60817 5.55426C4.65417 5.46108 4.70317 5.37131 4.75617 5.28268C4.87917 4.40655 4.77817 3.35998 4.50317 2.50545C4.36917 2.09182 4.20617 1.76682 4.05017 1.5816C4.01797 1.5431 3.98207 1.5088 3.94317 1.47932L3.94017 1.47705ZM13.1142 1.52251L13.1122 1.52364C13.0733 1.55312 13.0374 1.58741 13.0052 1.62591C12.8492 1.81114 12.6852 2.13727 12.5522 2.5509C12.2622 3.45316 12.1652 4.56905 12.3222 5.47358L12.3802 5.58381L12.3882 5.59972H12.4182C12.9145 5.59988 13.4082 5.68101 13.8842 5.84062C13.9702 4.56337 13.9222 3.51907 13.7562 2.74749C13.6662 2.33272 13.5462 2.01682 13.4072 1.80205L13.4032 1.79523C13.3331 1.67548 13.2342 1.58121 13.1182 1.52364L13.1142 1.52251Z" fill="black"/>
+</svg>
diff --git a/store/tools/local-ai/dist/index.html b/store/tools/local-ai/dist/index.html
new file mode 100644
index 000000000..4ec0310ac
--- /dev/null
+++ b/store/tools/local-ai/dist/index.html
@@ -0,0 +1,59 @@
+<!doctype html>
+<html lang="en">
+<head>
+ <meta charset="utf-8">
+ <meta name="viewport" content="width=device-width, initial-scale=1">
+ <meta name="theme-color" content="#101516">
+ <meta name="description" content="Run, inspect, and benchmark your Ollama models on this Mac using natural language.">
+ <title>Ollama — Your local model space</title>
+ <link rel="icon" type="image/svg+xml" href="/assets/ollama-icon.svg">
+ <link rel="apple-touch-icon" href="/assets/ollama-icon.png">
+ <link rel="stylesheet" href="/styles.css">
+ <script type="module" src="/app.js"></script>
+</head>
+<body>
+ <header class="topbar">
+ <a class="brand" href="/?view=grid" aria-label="Ollama model space"><img class="brand-mark" src="/assets/ollama-icon.svg" width="40" height="40" alt=""><span>Ollama</span><span class="pilot">LOCAL AI</span></a>
+ <div class="top-status"><span class="local-symbol" aria-hidden="true">◈</span> On your machine <span class="separator"></span><span id="connection-status">Connecting</span></div>
+ </header>
+ <main class="workspace">
+ <section class="control-room" aria-label="Ollama control room">
+ <div class="page-heading"><div><p class="eyebrow">OLLAMA / ON THIS MAC</p><h1>Your Ollama models.</h1></div><button id="refresh" class="icon-button" aria-label="Refresh model status" title="Refresh model status">↻</button></div>
+ <div id="runtime-notice" class="runtime-notice" role="status" hidden></div>
+ <div class="stats-row" id="stats-row">
+ <div class="stat"><span>Models on disk</span><strong id="model-count">—</strong></div>
+ <div class="stat"><span>In memory</span><strong id="running-count">—</strong></div>
+ <div class="stat"><span>Last generation</span><strong id="last-speed">— <small>tok/s</small></strong></div>
+ <div class="stat"><span>Model allocation</span><strong id="model-memory">— <small>GiB</small></strong></div>
+ </div>
+ <section class="map-panel" aria-labelledby="map-heading">
+ <div class="panel-heading"><h2 id="map-heading">Ollama runtime</h2><span class="mono live-label" id="map-status">DISCOVERING</span></div>
+ <div id="runtime-map" class="runtime-map"><div class="map-empty">Discovering your Ollama models…</div></div>
+ <div class="map-footer"><span><i class="legend-dot loaded"></i> In memory <i class="legend-dot"></i> On disk</span><span id="machine-info">Apple Silicon</span></div>
+ </section>
+ <section class="models-panel" aria-labelledby="models-heading">
+ <div class="panel-heading"><h2 id="models-heading">Ollama models <span id="models-total" class="count-pill">0</span></h2><span class="secondary">Select a model to inspect</span></div>
+ <div id="models-list" class="models-list"></div>
+ </section>
+ <section class="performance-panel" aria-labelledby="performance-heading">
+ <div class="panel-heading"><h2 id="performance-heading">Performance</h2><a id="export-benchmarks" class="secondary" href="/api/export" download hidden>Export results ↗</a></div>
+ <div id="performance-content" class="performance-content"><div class="empty-small"><span class="empty-icon" aria-hidden="true">▥</span><p>Your first run sets the baseline.</p><span>Benchmark a model to compare speed and response time.</span></div></div>
+ </section>
+ <section class="activity-panel" aria-labelledby="activity-heading"><div class="panel-heading"><h2 id="activity-heading">Activity</h2><span class="secondary">Live from Ollama</span></div><div id="activity-content"></div></section>
+ </section>
+ <aside class="assistant-panel" aria-label="Natural language model controls">
+ <div class="assistant-header"><img class="assistant-avatar" src="/assets/ollama-icon.svg" width="36" height="36" alt=""><div><h2>Ollama operator</h2><span>Tell it what you want to run.</span></div><span class="private-badge">LOCAL</span></div>
+ <div class="conversation" id="conversation" role="log" aria-live="polite" aria-relevant="additions">
+ <div class="assistant-message"><span class="message-label">OLLAMA</span><p>Your Ollama models.<br>At your command.</p><p class="secondary">I can find a model, deploy it with Ollama, and measure how it performs here.</p></div>
+ <div class="suggestions" id="suggestions"><button data-prompt="Run a lightweight model">Run a lightweight model <span>↗</span></button><button data-prompt="Find a small model for coding">Find a model for coding <span>↗</span></button><button data-prompt="Show my models">Show my models <span>↗</span></button></div>
+ </div>
+ <div id="job-panel" class="job-panel" hidden></div>
+ <label class="model-picker" for="selected-model">Chat / follow-up model<select id="selected-model"><option value="">Auto · running model</option></select></label>
+ <form id="command-form" class="composer"><label class="sr-only" for="command-input">Ask the Ollama operator</label><textarea id="command-input" rows="3" placeholder="Run an Ollama model, compare speeds, free up memory…" maxlength="4000"></textarea><div class="composer-footer"><span id="operator-status">Ollama harness</span><button id="send-button" type="submit" aria-label="Send request">↑</button></div></form>
+ <div class="assistant-footer">Ollama inference stays on this Mac.<span>↵ Send · ⇧↵ New line</span></div>
+ </aside>
+ </main>
+ <dialog id="model-dialog" aria-labelledby="detail-title"><div id="model-detail"></div><button class="dialog-close" aria-label="Close model details">×</button></dialog>
+ <div id="toast" class="toast" role="status" hidden></div>
+</body>
+</html>
diff --git a/store/tools/local-ai/dist/styles.css b/store/tools/local-ai/dist/styles.css
new file mode 100644
index 000000000..cee872f50
--- /dev/null
+++ b/store/tools/local-ai/dist/styles.css
@@ -0,0 +1,21 @@
+:root{color-scheme:dark;--bg:#101516;--surface:#171d1e;--surface-2:#1d2526;--line:#303939;--muted:#9ba9a7;--text:#edf3ee;--accent:#c5f87e;--cyan:#7ed8d1;--error:#ffafa4;--mono:ui-monospace,SFMono-Regular,Menlo,monospace;font-family:Inter,-apple-system,BlinkMacSystemFont,"Segoe UI",sans-serif;font-size:16px}
+*{box-sizing:border-box}body{margin:0;background:var(--bg);color:var(--text)}button,textarea,input,select{font:inherit}button{cursor:pointer}button:disabled{cursor:wait;opacity:.55}button,a,textarea,select{outline-offset:4px}button:focus-visible,a:focus-visible{outline:2px solid var(--accent)}button{color:inherit}a{color:inherit}button{border:1px solid var(--line);background:var(--surface);border-radius:8px}button:hover{border-color:#617166}h1,h2,p{margin:0}h2{font-size:1rem;font-weight:600;letter-spacing:-.02em}.mono{font-family:var(--mono)}.secondary{color:var(--muted);font-size:.875rem;line-height:1.6}.topbar{height:76px;border-bottom:1px solid var(--line);display:flex;justify-content:space-between;align-items:center;padding:0 32px}.brand{text-decoration:none;font-size:1rem;font-weight:750;letter-spacing:.15em;display:flex;align-items:center;gap:13px}.brand-mark{background:var(--accent);color:#111b0c;width:30px;height:32px;display:grid;place-items:center;clip-path:polygon(0 0,80% 0,100% 25%,100% 100%,20% 100%,0 75%);font-size:1.125rem;font-weight:900;letter-spacing:-.15em;padding-right:4px}.pilot{font-family:var(--mono);letter-spacing:.05em;font-size:.75rem;font-weight:400;color:var(--muted);padding-left:16px;margin-left:8px;border-left:1px solid var(--line)}.top-status{display:flex;gap:10px;align-items:center;font-size:.8125rem;color:var(--muted)}.local-symbol{color:var(--accent);font-size:1.1rem}.separator{height:16px;width:1px;background:var(--line);margin:0 6px}.workspace{display:grid;grid-template-columns:minmax(0,1fr) 360px;min-height:calc(100vh - 76px);max-width:1900px;margin:auto}.control-room{padding:34px 32px 40px;min-width:0}.page-heading{display:flex;justify-content:space-between;align-items:center;margin-bottom:30px}.eyebrow{font-family:var(--mono);font-size:.75rem;color:var(--accent);letter-spacing:.15em;margin-bottom:10px}h1{font-size:clamp(1.75rem,2.6vw,2.5rem);letter-spacing:-.06em;line-height:1.15;font-weight:550}.icon-button{border-radius:50%;width:38px;height:38px;font-size:1.4rem;background:transparent;color:var(--muted)}.stats-row{display:grid;grid-template-columns:repeat(4,1fr);margin-bottom:30px}.stat{padding:0 22px;border-right:1px solid var(--line);min-width:0}.stat:first-child{padding-left:0}.stat:last-child{border:0}.stat>span{font-size:.8125rem;color:var(--muted);display:block;margin-bottom:9px}.stat strong{font-family:var(--mono);font-size:1.75rem;font-weight:400;letter-spacing:-.07em}.stat small{font-family:inherit;color:var(--muted);font-size:.8rem;letter-spacing:-.02em}.panel-heading{display:flex;justify-content:space-between;align-items:center;gap:12px;margin-bottom:18px}.map-panel{border:1px solid var(--line);background:var(--surface);border-radius:12px;overflow:hidden}.map-panel .panel-heading{padding:20px 20px 0;margin-bottom:0}.live-label{font-size:.6875rem;letter-spacing:.09em;color:var(--muted)}.live-label.online{color:var(--accent)}.runtime-map{min-height:252px;position:relative;padding:25px 22px;background-image:radial-gradient(#3a454238 .8px,transparent .8px);background-size:16px 16px;display:flex;align-items:center;gap:36px}.runtime-node{border:1px solid #596a50;background:#202c21;border-radius:14px;padding:18px;min-width:150px;box-shadow:0 0 40px #c5f87e07;z-index:1;position:relative}.runtime-node::after{content:'';width:37px;height:1px;background:#617751;position:absolute;right:-37px;top:50%}.runtime-node.offline{border-color:var(--line);background:var(--surface-2)}.runtime-node .node-icon{width:32px;height:32px;display:grid;place-items:center;background:var(--accent);color:#233014;border-radius:9px;font-weight:700;margin-bottom:13px}.runtime-node strong{font-size:1rem;display:block;margin-bottom:6px}.runtime-node small{font-family:var(--mono);color:var(--muted);font-size:.75rem}.model-nodes{display:flex;gap:10px;flex-wrap:wrap;align-content:center;flex:1;border-left:1px solid #3c4d3c;padding-left:22px;min-height:120px}.model-node{position:relative;text-align:left;padding:13px 15px;border-radius:8px;min-width:160px;max-width:240px;background:#1b2222}.model-node::before{content:'';position:absolute;left:-23px;top:50%;height:1px;width:22px;background:#3c4d3c}.model-node.active{border-color:#7a995a;background:#263324}.model-node strong{display:block;font-size:.875rem;overflow-wrap:anywhere;line-height:1.4}.model-node span{display:block;color:var(--muted);font-size:.75rem;margin-top:6px}.model-node.active span{color:var(--accent)}.map-empty{margin:auto;color:var(--muted);font-size:.875rem;max-width:330px;line-height:1.7;text-align:center}.empty-node{text-align:left;margin:0;padding:12px 0}.empty-node strong{display:block;color:var(--text);font-size:1rem;margin-bottom:6px}.map-footer{border-top:1px solid var(--line);display:flex;align-items:center;justify-content:space-between;gap:12px;padding:12px 20px;font-family:var(--mono);font-size:.6875rem;color:var(--muted)}.map-footer>span:first-child{display:flex;align-items:center;gap:6px}.legend-dot{width:5px;height:5px;display:inline-block;border-radius:50%;background:#6a7a74;margin-left:12px}.legend-dot:first-child{margin-left:0}.legend-dot.loaded{background:var(--accent)}.models-panel,.performance-panel{margin-top:30px}.count-pill{display:inline-grid;place-items:center;font-family:var(--mono);font-size:.75rem;min-width:23px;height:21px;border-radius:5px;border:1px solid var(--line);color:var(--muted);margin-left:6px}.model-row{width:100%;display:grid;grid-template-columns:minmax(0,1fr) 85px 80px 80px;gap:14px;align-items:center;text-align:left;border-width:0 0 1px;border-radius:0;background:transparent;padding:17px 4px;font-size:.875rem}.model-row:hover{background:var(--surface)}.model-title strong{font-weight:500;overflow-wrap:anywhere;display:block}.model-title small{font-size:.75rem;color:var(--muted);display:block;margin-top:6px}.model-status{font-size:.75rem;color:var(--muted)}.model-status.loaded{color:var(--accent)}.model-numeric{font-family:var(--mono);font-size:.75rem;color:var(--muted)}.empty-small{padding:30px 18px;border:1px dashed var(--line);border-radius:10px;color:var(--muted);text-align:center;font-size:.875rem}.empty-small p{color:var(--text);margin:8px 0;font-size:1rem}.empty-small span:not(.empty-icon){font-size:.8125rem;line-height:1.6}.empty-icon{font-size:1.5rem;color:#667973}.assistant-panel{border-left:1px solid var(--line);background:#131a1b;display:flex;flex-direction:column;min-width:0;position:sticky;top:0;height:calc(100vh - 76px);max-height:1200px}.assistant-header{display:flex;gap:12px;align-items:center;padding:26px 22px;border-bottom:1px solid var(--line)}.assistant-avatar{height:35px;width:35px;display:grid;place-items:center;color:var(--accent);background:#293526;font-size:1.5rem;border-radius:10px}.assistant-header h2{font-size:.9375rem}.assistant-header div>span{font-size:.75rem;color:var(--muted);display:block;margin-top:4px}.private-badge{font-size:.625rem;letter-spacing:.08em;color:var(--muted);border:1px solid var(--line);border-radius:4px;padding:4px 5px;margin-left:auto;font-family:var(--mono)}.conversation{padding:26px 22px;overflow:auto;flex:1;min-height:200px;scroll-behavior:smooth}.message-label{font-family:var(--mono);font-size:.625rem;letter-spacing:.1em;color:var(--accent);display:block;margin-bottom:13px}.assistant-message{font-size:.9375rem;line-height:1.65;margin-bottom:25px}.assistant-message p+p{margin-top:12px}.assistant-message .secondary{font-size:.875rem}.suggestions{display:flex;flex-direction:column;gap:8px;margin-top:25px}.suggestions button{display:flex;justify-content:space-between;align-items:center;text-align:left;font-size:.8125rem;padding:13px 12px;background:transparent;color:#c4d0cb}.suggestions button span{color:var(--accent)}.composer{margin:0 18px;border:1px solid #485441;background:#202829;border-radius:11px;overflow:hidden}.composer:focus-within{border-color:var(--accent)}textarea{color:var(--text);display:block;resize:none;background:transparent;width:100%;border:0;padding:15px 15px 5px;line-height:1.6;outline:none;font-size:.875rem}textarea::placeholder{color:#91a29b}.composer-footer{padding:8px 11px 10px 15px;display:flex;justify-content:space-between;align-items:center}.composer-footer>span{font-size:.6875rem;font-family:var(--mono);color:var(--muted)}.composer-footer button{background:var(--accent);color:#243315;border:0;border-radius:7px;font-size:1.4rem;width:31px;height:31px}.assistant-footer{display:flex;flex-wrap:wrap;gap:8px;justify-content:space-between;padding:14px 20px 20px;font-size:.625rem;color:#98a79f;line-height:1.5}.assistant-footer span{font-size:.625rem}.user-message{padding:12px 14px;background:#263127;border:1px solid #3b4e35;border-radius:9px 9px 2px 9px;margin:15px 0 25px;white-space:pre-wrap;overflow-wrap:anywhere;font-size:.875rem;line-height:1.6}.message-text{white-space:pre-wrap;overflow-wrap:anywhere}.job-panel{padding:15px;margin:0 18px 15px;background:var(--surface-2);border:1px solid var(--line);border-radius:10px;font-size:.8125rem}.job-panel strong{font-size:.8125rem;font-weight:500}.job-panel p{color:var(--muted);font-size:.75rem;margin-top:7px}.job-top{display:flex;justify-content:space-between;align-items:center}.job-top button{font-size:.6875rem;padding:4px 7px}.progress-track{height:4px;background:var(--line);border-radius:4px;margin:12px 0 2px;overflow:hidden}.progress-fill{height:100%;width:0;background:var(--accent);transition:width .3s}.toast{position:fixed;bottom:20px;left:50%;transform:translateX(-50%);padding:12px 18px;background:#33402f;color:var(--text);border:1px solid #60784d;border-radius:8px;max-width:90vw;z-index:10;font-size:.875rem}dialog{background:var(--surface);color:var(--text);border:1px solid var(--line);border-radius:16px;padding:30px;max-width:600px;width:calc(100% - 32px)}dialog::backdrop{background:#000a;backdrop-filter:blur(3px)}.dialog-close{position:absolute;right:12px;top:10px;border:0;background:transparent;font-size:1.5rem}.sr-only{position:absolute;width:1px;height:1px;overflow:hidden;clip:rect(0,0,0,0);white-space:nowrap}.detail-grid{display:grid;grid-template-columns:repeat(2,1fr);gap:16px;margin:22px 0}.detail-grid>div{border:1px solid var(--line);padding:14px;border-radius:8px}.detail-grid span{font-size:.75rem;color:var(--muted);display:block;margin-bottom:7px}.detail-grid strong{font-size:1rem;font-weight:500}.actions{display:flex;flex-wrap:wrap;gap:8px;margin-top:18px}.actions button{padding:9px 13px;font-size:.875rem}.actions button.primary{background:var(--accent);color:#233014;border-color:var(--accent)}.chart-bars{display:flex;flex-direction:column;gap:15px;padding:8px 0}.chart-row{display:grid;grid-template-columns:minmax(120px,160px) 1fr 65px;gap:16px;align-items:center;font-size:.8125rem}.chart-row>span:first-child{overflow-wrap:anywhere}.bar-track{height:22px;background:var(--surface-2);border-radius:4px;overflow:hidden}.bar-fill{height:100%;background:var(--accent);border-radius:3px;min-width:2px}.bar-value{font-family:var(--mono);font-size:.8125rem}.benchmark-meta{color:var(--muted);font-size:.75rem;margin-top:15px;line-height:1.8}.plan-card{border:1px solid #536846;border-radius:10px;background:#222e20;padding:15px;margin-top:12px}.plan-card p{font-size:.8125rem;line-height:1.6}.plan-card .actions{margin-top:10px}.error-text{color:var(--error)}.catalog-card{border:1px solid var(--line);border-radius:9px;padding:13px;margin:10px 0;font-size:.8125rem}.catalog-card strong{display:block;margin-bottom:5px}.catalog-card button{margin-top:10px;padding:6px 10px;background:var(--accent);color:#233014;font-size:.75rem;border-color:var(--accent)}.stream-output{max-height:340px;overflow:auto;border-left:2px solid #526c3f;padding-left:12px;font-size:.875rem}.warning-note{font-size:.75rem;color:#d4cca8;line-height:1.6;margin-top:10px}.mini-label{font-family:var(--mono);font-size:.6875rem;color:var(--muted);letter-spacing:.03em}
+@media(min-width:1450px){.workspace{grid-template-columns:minmax(0,1fr) 420px}.control-room{padding:40px 48px}.runtime-map{min-height:295px}.assistant-header,.conversation{padding:28px}.composer{margin:0 24px}.model-node{min-width:180px}}
+@media(max-width:1080px){.workspace{grid-template-columns:minmax(0,1fr) 320px}.control-room{padding:28px 22px}.topbar{padding:0 22px}.stat{padding:0 12px}.stat strong{font-size:1.5rem}.stat>span{font-size:.75rem}.runtime-map{gap:25px;padding:22px 16px}.runtime-node{min-width:118px;padding:14px}.runtime-node::after{width:26px;right:-26px}.model-node{min-width:130px;padding:12px;max-width:100%}.model-nodes{padding-left:14px}.model-node::before{left:-15px;width:14px}.model-row{grid-template-columns:minmax(0,1fr) 68px 65px;gap:7px}.model-row .model-numeric:last-child{display:none}.map-footer{font-size:.625rem;padding:12px}.top-status{font-size:.75rem}.assistant-header{padding:22px 16px}}
+@media(max-width:820px){.workspace{display:flex;flex-direction:column}.assistant-panel{position:relative;height:auto;max-height:none;border-left:0;border-top:1px solid var(--line);min-height:500px}.control-room{padding:26px 20px}.conversation{max-height:480px}.topbar{height:64px;padding:0 20px}.top-status .separator,.top-status #connection-status{display:none}.pilot{font-size:.625rem;padding-left:10px;margin-left:0}.brand{font-size:.875rem;gap:9px}.runtime-map{min-height:230px}.model-nodes{gap:8px}.model-node{max-width:220px}.stats-row{margin-bottom:24px}.stat{padding:0 10px}.stat>span{font-size:.6875rem}.map-footer{font-size:.625rem}.assistant-footer{font-size:.6875rem}.top-status{font-size:.6875rem;gap:5px}.assistant-header h2{font-size:1rem}.assistant-header div>span{font-size:.875rem}.private-badge{font-size:.75rem}.suggestions button{font-size:.875rem}textarea{font-size:1rem}.assistant-message{font-size:1rem}.model-row{grid-template-columns:minmax(0,1fr) 75px 80px}.model-node strong{font-size:.875rem}.model-node span{font-size:.75rem}}
+@media(max-width:430px){.top-status{display:none}.stat strong{font-size:1.35rem}.stat>span{min-height:30px;line-height:1.4}.stat small{font-size:.6875rem}.runtime-map{gap:20px;padding:18px 12px}.runtime-node{min-width:106px;padding:12px}.runtime-node::after{right:-21px;width:21px}.model-node{min-width:120px}.map-footer{flex-wrap:wrap}.panel-heading>.secondary{font-size:.75rem}.chart-row{grid-template-columns:110px 1fr 52px;gap:8px}}
+@media(prefers-reduced-motion:reduce){*{scroll-behavior:auto!important;transition:none!important;animation:none!important}}
+/* Live control and measurement surfaces. */
+[hidden]{display:none!important}.model-picker{display:flex;flex-direction:column;gap:7px;margin:0 18px 12px;color:var(--muted);font-size:.75rem}.model-picker select{width:100%;color:var(--text);background:var(--surface);border:1px solid var(--line);border-radius:7px;padding:9px;font-size:.875rem}.model-node.selected{box-shadow:0 0 0 2px var(--cyan)}.model-row.selected{background:#1a2826}.job-state{color:var(--muted);margin-left:8px;letter-spacing:0;font-family:inherit;text-transform:lowercase}.run-metrics{font-family:var(--mono);font-size:.75rem;color:var(--cyan);margin-top:12px!important}.stream-output{margin-top:14px;line-height:1.7}.chart-heading{display:flex;justify-content:space-between;gap:12px;align-items:center;flex-wrap:wrap;margin-bottom:10px}.chart-heading h3{font-size:.875rem;margin:0;font-weight:500}.chart-heading>span{font-size:.75rem;color:var(--muted)}.second-chart{margin-top:26px}.bar-fill.latency{background:var(--cyan)}.benchmark-details{margin-top:16px;border-top:1px solid var(--line);padding-top:14px;font-size:.875rem}.benchmark-details summary{cursor:pointer;color:var(--muted)}.sample-record{display:flex;flex-direction:column;gap:5px;padding:18px 0;border-bottom:1px solid var(--line);font-size:.875rem;overflow-wrap:anywhere}.sample-record span{color:var(--muted);font-size:.75rem}.chart-row{grid-template-columns:minmax(125px,170px) 1fr 64px}.bar-value{text-align:right}.bar-value small{font-size:.75rem}.detail-grid strong,#detail-title{overflow-wrap:anywhere}#operator-status{overflow:hidden;text-overflow:ellipsis;white-space:nowrap;max-width:calc(100% - 44px)}.job-top{gap:12px}.job-top button{flex-shrink:0}.assistant-message,.stream-output,.user-message{font-size:1rem}.live-label,.map-footer,.private-badge,.message-label,.composer-footer>span,.assistant-footer,.assistant-footer span,.job-top button,.mini-label,.pilot,.stat>span,.top-status,.stat small{font-size:.75rem}.stat>span,.suggestions button,.top-status{font-size:.875rem}.composer textarea{font-size:1rem}.model-node span,.model-title small,.model-status,.model-numeric{font-size:.8125rem}.map-footer{line-height:1.6;flex-wrap:wrap}.topbar{gap:14px}.assistant-panel{min-height:650px}.job-panel p{overflow-wrap:anywhere}.catalog-card p{line-height:1.6}.model-nodes{min-width:0}.model-node{min-width:0;flex:1 1 150px}button,a,select{-webkit-tap-highlight-color:transparent}
+@media(max-width:1080px){.stat strong{font-size:1.45rem}.stat>span{min-height:32px}.model-row{grid-template-columns:minmax(0,1fr) 76px 76px}.chart-row{grid-template-columns:130px 1fr 60px;gap:10px}}
+@media(max-width:820px){.assistant-panel{min-height:650px}.top-status{font-size:.75rem}.stat>span{font-size:.75rem}.stat small{font-size:.75rem}.model-picker select{font-size:1rem}.assistant-panel{height:auto}.model-nodes{flex-basis:140px}.model-node{min-width:0}.map-footer{font-size:.75rem}}
+@media(max-width:430px){.chart-row{grid-template-columns:100px 1fr 54px;gap:8px}.model-row{grid-template-columns:minmax(0,1fr) 72px 62px}.model-numeric,.model-status{font-size:.75rem}.stats-row{gap:0}.stat{padding:0 7px}.stat strong{font-size:1.3rem}.stat>span{min-height:36px}.runtime-node{min-width:100px}.control-room{padding:24px 16px}.runtime-map{gap:17px}.runtime-node::after{width:18px;right:-18px}.map-footer>span{flex-wrap:wrap}.assistant-header{padding:20px 16px}.conversation{padding:22px 18px}}
+@media(pointer:coarse){button{min-height:44px}.model-picker select{min-height:44px}.composer-footer button{min-width:44px}.icon-button{width:44px;height:44px}}
+.activity-panel{margin-top:32px}.activity-row{display:flex;align-items:flex-start;gap:16px;padding:17px 0;border-top:1px solid var(--line);font-size:.875rem}.activity-state{min-width:78px;font-family:var(--mono);font-size:.75rem;color:var(--muted);padding-top:3px}.activity-row>div{flex:1;min-width:0}.activity-row strong{font-weight:500;overflow-wrap:anywhere}.activity-row p{font-size:.8125rem;line-height:1.6;margin-top:5px;color:var(--muted);overflow-wrap:anywhere}.activity-time{font-size:.75rem;white-space:nowrap;color:var(--muted)}.activity-row button{padding:5px 9px;font-size:.75rem}.embedded-viewer .workspace{display:block;max-width:1440px}.embedded-viewer .assistant-panel{display:none}.embedded-viewer .control-room{padding:34px clamp(20px,4vw,60px)}.embedded-viewer .runtime-map{min-height:240px}.embedded-viewer .topbar{height:64px}.embedded-viewer .workspace{min-height:calc(100vh - 64px)}.embedded-viewer .stats-row{max-width:1050px}.embedded-viewer .model-nodes{flex:1}.embedded-viewer .model-node{flex:0 1 230px}.embedded-viewer .model-row{grid-template-columns:minmax(0,1fr) 100px 110px 100px}.embedded-viewer .runtime-node{min-width:160px}.embedded-viewer .model-row .model-numeric:last-child{display:block}
+@media(max-width:600px){.embedded-viewer .model-row{grid-template-columns:minmax(0,1fr) 76px 76px}.embedded-viewer .model-row .model-numeric:last-child{display:none}.embedded-viewer .runtime-node{min-width:100px}.activity-row{gap:10px}.activity-state{min-width:64px}.activity-time{display:none}.embedded-viewer .model-node{flex-basis:160px}.embedded-viewer .control-room{padding:24px 16px}}
+.embedded-viewer [data-chat]{display:none}
+
+/* Ollama identity: use the official mark at every logo size. */
+.brand{font-size:1.45rem;letter-spacing:-.04em;text-transform:none;gap:12px}.brand .brand-mark{display:block;width:40px;height:40px;padding:0;background:none;clip-path:none;border-radius:10px;object-fit:contain}.brand .pilot{font-size:.65rem;letter-spacing:.13em;margin-left:10px;padding-left:20px}.runtime-node .node-icon{display:block;width:42px;height:42px;padding:0;margin:0 auto 9px;background:none;border-radius:10px;object-fit:contain}.assistant-header .assistant-avatar{padding:0;object-fit:contain;background:none;border-radius:9px}
+
+.runtime-notice{margin:0 0 20px;padding:16px 18px;border:1px solid #614840;border-radius:12px;background:#241e1a}.runtime-notice[hidden]{display:none}.runtime-notice strong{color:var(--error);font-size:.9rem}.runtime-notice p{margin:8px 0;color:var(--text);font-size:.85rem;line-height:1.5}.runtime-notice summary{cursor:pointer;color:var(--muted);font-size:.8rem}.runtime-notice pre{white-space:pre-wrap;overflow-wrap:anywhere;font: .75rem/1.5 var(--mono);color:var(--error)}
diff --git a/store/tools/local-ai/package.json b/store/tools/local-ai/package.json
new file mode 100644
index 000000000..fe68d1357
--- /dev/null
+++ b/store/tools/local-ai/package.json
@@ -0,0 +1,23 @@
+{
+ "name": "local-ai-harnesses",
+ "version": "0.1.0",
+ "private": true,
+ "type": "module",
+ "description": "A local control room for running AI models in natural language.",
+ "engines": {
+ "node": ">=22"
+ },
+ "scripts": {
+ "build:mlx": "node scripts/build-mlx.mjs",
+ "start:mlx": "node ../../agents/mlx-lm/main.mjs",
+ "test:mlx-live": "node test/mlx-live.mjs",
+ "start": "node ../../agents/ollama/src/server.mjs",
+ "test": "node --test test/*.test.mjs",
+ "check": "node --check src/server.mjs && node --check src/controller.mjs && node --check src/intents.mjs && node --check src/ollama.mjs && node --check dist/app.js && node --check bin/harness.mjs",
+ "test:live": "node test/live.mjs",
+ "build:vllm": "node scripts/build-vllm.mjs",
+ "start:vllm": "node ../../agents/vllm/main.mjs",
+ "test:vllm-live": "node test/vllm-live.mjs",
+ "build": "node scripts/build-ollama.mjs && node scripts/build-mlx.mjs && node scripts/build-vllm.mjs"
+ }
+}
diff --git a/store/tools/local-ai/scripts/build-mlx.mjs b/store/tools/local-ai/scripts/build-mlx.mjs
new file mode 100644
index 000000000..a8adea1e2
--- /dev/null
+++ b/store/tools/local-ai/scripts/build-mlx.mjs
@@ -0,0 +1,35 @@
+// Package the shared control plane and viewer with MLX-LM's own identity and measurements.
+import { readFile, writeFile, copyFile, mkdir } from 'node:fs/promises';
+import { dirname, join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+const root = dirname(dirname(fileURLToPath(import.meta.url)));
+const target = join(root, '../../agents/mlx-lm');
+for (const directory of ['src', 'bin', 'dist/assets', 'toolchain']) await mkdir(join(target, directory), { recursive: true });
+for (const name of ['catalog', 'controller', 'intents', 'mlx', 'ollama', 'server', 'store', 'viewer-relay']) {
+ await copyFile(join(root, `src/${name}.mjs`), join(target, `src/${name}.mjs`));
+}
+await copyFile(join(root, 'bin/harness.mjs'), join(target, 'bin/harness.mjs'));
+await copyFile(join(root, 'toolchain/node.sh'), join(target, 'toolchain/node.sh'));
+const brand = text => text.replaceAll('Ollama', 'MLX-LM').replaceAll('OLLAMA', 'MLX-LM').replaceAll('ollama-icon', 'mlx-icon');
+let html = brand(await readFile(join(root, 'dist/index.html'), 'utf8'));
+html = html.replaceAll('#101516', '#10151f')
+ .replace('Your MLX-LM models.', 'Models. Made for this Mac.')
+ .replace('MLX-LM runtime</h2>', 'MLX on Apple Silicon</h2>')
+ .replace('Model allocation</span>', 'MLX allocation</span>')
+ .replace('Select a model to inspect', 'One model in memory at a time')
+ .replace('Run an MLX-LM model', 'Run an MLX model');
+await writeFile(join(target, 'dist/index.html'), html);
+let app = brand(await readFile(join(root, 'dist/app.js'), 'utf8'));
+app = app.replace('Repeated prompts can use the prefix cache.', 'Each sample starts with a fresh prompt cache.')
+ .replace('MLX-LM reports runtime allocation.', 'MLX reports active allocations for this workspace, excluding its reusable memory cache.')
+ .replace('Context window</span>', 'Harness context budget</span>')
+ .replace('Exact local MLX-LM model name and tag.', 'Exact Hugging Face MLX model ID, including its owner.')
+ .replace('${escapeHTML(b.model)}</span><div class="bar-track">', '${escapeHTML(b.model.split("/").at(-1))}</span><div class="bar-track">');
+await writeFile(join(target, 'dist/app.js'), app);
+let css = brand(await readFile(join(root, 'dist/styles.css'), 'utf8'));
+css = css.replaceAll('#101516', '#10151f').replaceAll('#171d1e', '#171e2a').replaceAll('#1d2526', '#202a38')
+ .replaceAll('#303939', '#344154').replaceAll('#9ba9a7', '#a2aec0').replaceAll('#c5f87e', '#a9caff').replaceAll('#7ed8d1', '#8ae8d4');
+css += '\n/* Hugging Face model names are longer than Ollama tags. */\n.model-node strong,.model-title strong,.chart-row>span:first-child,.activity-row strong{overflow-wrap:anywhere}.model-node{min-width:0}.chart-row>span:first-child{font-size:.72rem}.activity-row>div{min-width:0}.model-title strong{white-space:normal}.sample-record strong{overflow-wrap:anywhere}\n';
+await writeFile(join(target, 'dist/styles.css'), css);
+for (const name of ['mlx-icon.svg', 'mlx-icon.png', 'README.md']) await copyFile(join(target, 'assets', name), join(target, 'dist/assets', name));
+console.log('Built MLX-LM from the shared queue, controls, measurements and viewer.');
diff --git a/store/tools/local-ai/scripts/build-ollama.mjs b/store/tools/local-ai/scripts/build-ollama.mjs
new file mode 100644
index 000000000..485b7c8d6
--- /dev/null
+++ b/store/tools/local-ai/scripts/build-ollama.mjs
@@ -0,0 +1,13 @@
+// Package the shared sources so Store installs need only this harness directory.
+import { cp, copyFile, mkdir } from 'node:fs/promises';
+import { dirname, join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const root = dirname(dirname(fileURLToPath(import.meta.url)));
+const target = join(root, '../../agents/ollama');
+await mkdir(join(target, 'toolchain'), { recursive: true });
+for (const directory of ['src', 'bin', 'dist']) {
+ await cp(join(root, directory), join(target, directory), { recursive: true });
+}
+await copyFile(join(root, 'toolchain/node.sh'), join(target, 'toolchain/node.sh'));
+console.log('Built Ollama with the shared control plane and viewer.');
diff --git a/store/tools/local-ai/scripts/build-vllm.mjs b/store/tools/local-ai/scripts/build-vllm.mjs
new file mode 100644
index 000000000..ceea62af7
--- /dev/null
+++ b/store/tools/local-ai/scripts/build-vllm.mjs
@@ -0,0 +1,40 @@
+// Reuse the shared viewer and control plane; specialize the vLLM serving surface.
+import { readFile, writeFile, copyFile, mkdir } from 'node:fs/promises';
+import { dirname, join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+const root = dirname(dirname(fileURLToPath(import.meta.url)));
+const target = join(root, '../../agents/vllm');
+for (const directory of ['src', 'bin', 'dist/assets', 'toolchain']) await mkdir(join(target, directory), { recursive: true });
+for (const name of ['catalog', 'controller', 'intents', 'mlx', 'ollama', 'server', 'store', 'viewer-relay', 'json-worker', 'vllm']) await copyFile(join(root, `src/${name}.mjs`), join(target, `src/${name}.mjs`));
+await copyFile(join(root, 'bin/harness.mjs'), join(target, 'bin/harness.mjs'));
+await copyFile(join(root, 'toolchain/node.sh'), join(target, 'toolchain/node.sh'));
+const brand = text => text.replaceAll('Ollama', 'vLLM').replaceAll('OLLAMA', 'vLLM').replaceAll('ollama-icon.svg', 'vllm-icon.png').replaceAll('ollama-icon.png', 'vllm-icon.png');
+let html = brand(await readFile(join(root, 'dist/index.html'), 'utf8'))
+ .replace('image/svg+xml', 'image/png').replaceAll('#101516', '#10151d')
+ .replace('vLLM / ON THIS MAC', 'vLLM / METAL ON THIS MAC').replace('Your vLLM models.', 'Models, ready to serve.')
+ .replace('vLLM runtime</h2>', 'vLLM · Metal backend</h2>').replace('Model allocation</span>', 'Server RSS</span>')
+ .replace('Last generation</span>', 'Last request</span>').replace('Select a model to inspect', 'One server · continuous batching')
+ .replace('<section class="models-panel"', '<section class="serving-panel" aria-labelledby="serving-heading"><div class="panel-heading"><h2 id="serving-heading">Serving</h2><span class="secondary" id="serving-state">Idle</span></div><div id="serving-content"></div></section>\n <section class="models-panel"')
+ .replace('<section class="performance-panel"', '<section class="concurrency-panel" aria-labelledby="concurrency-heading"><div class="panel-heading"><h2 id="concurrency-heading">Concurrent performance</h2><span class="secondary">Throughput &amp; latency</span></div><div id="concurrency-content"></div></section>\n <section class="performance-panel"')
+ .replace('id="performance-heading">Performance', 'id="performance-heading">Single-request baseline');
+await writeFile(join(target, 'dist/index.html'), html);
+let app = brand(await readFile(join(root, 'dist/app.js'), 'utf8'))
+ .replace("const gib = bytes => ((bytes || 0) / 1024 ** 3).toFixed(1);", "const gib = bytes => Number.isFinite(bytes) ? (bytes / 1024 ** 3).toFixed(1) : '—';")
+ .replace('Generation speed</h3>', 'Request throughput</h3>').replace('Measured median generation speed', 'Measured median end-to-end request throughput')
+ .replace('Speed measures inference, not answer quality.', 'Token throughput includes prompt processing and HTTP; it is not native decode speed or answer quality.')
+ .replace('vLLM reports runtime allocation.', 'RSS sums this workspace’s server processes and may count shared pages more than once. It is not GPU allocation.')
+ .replace('Runtime allocation</span>', 'Server process RSS</span>').replace('allocation: ${gib(b.residentBytes)}', 'server RSS: ${gib(b.residentBytes)}')
+ .replace('Exact local vLLM model name and tag.', 'Exact Hugging Face text model ID, including its owner.')
+ .replace('${escapeHTML(b.model)}</span><div class="bar-track">', '${escapeHTML(b.model.split("/").at(-1))}</span><div class="bar-track">')
+ .replace("$('#export-benchmarks').hidden = benchmarks.length === 0;", "$('#export-benchmarks').hidden = !benchmarks.length && !state.loadTests?.length; renderServing(); renderLoadTests();")
+ .replace("['deploy', 'chat', 'benchmark'].includes(job.type)", "['deploy', 'serve', 'chat', 'benchmark', 'load_test'].includes(job.type)")
+ .replace('>Benchmark</button><button data-chat=', '>Benchmark</button><button data-action="load_test" data-target="${escapeHTML(id)}">Test 4 concurrent</button><button data-chat=')
+ .replace('await refresh();\nconst events', `${await readFile(join(target, 'viewer-extra.js'), 'utf8')}\nawait refresh();\nconst events`);
+await writeFile(join(target, 'dist/app.js'), app);
+let css = brand(await readFile(join(root, 'dist/styles.css'), 'utf8'))
+ .replaceAll('#101516', '#10151d').replaceAll('#171d1e', '#171e29').replaceAll('#1d2526', '#202a38')
+ .replaceAll('#303939', '#344052').replaceAll('#9ba9a7', '#a2adbf').replaceAll('#c5f87e', '#ffc743').replaceAll('#7ed8d1', '#7bb1ff');
+css += await readFile(join(target, 'viewer-extra.css'), 'utf8');
+await writeFile(join(target, 'dist/styles.css'), css);
+for (const name of ['vllm-icon.png', 'README.md']) await copyFile(join(target, 'assets', name), join(target, 'dist/assets', name));
+console.log('Built vLLM: shared viewer, Metal serving telemetry and concurrent measurements.');
diff --git a/store/tools/local-ai/src/catalog.mjs b/store/tools/local-ai/src/catalog.mjs
new file mode 100644
index 000000000..616380e9d
--- /dev/null
+++ b/store/tools/local-ai/src/catalog.mjs
@@ -0,0 +1,6 @@
+export const CATALOG = [
+ { id: 'qwen3:0.6b', name: 'Qwen 3 · 0.6B', purpose: 'A lightweight first model', description: 'Quick to download. Good for trying the harness, short answers, and simple tasks.', downloadGB: 0.523, use: 'general', small: true, source: 'https://ollama.com/library/qwen3:0.6b' },
+ { id: 'qwen2.5-coder:1.5b', name: 'Qwen 2.5 Coder · 1.5B', purpose: 'Small coding assistant', description: 'A compact model for code snippets, explanations, and simple programming tasks.', downloadGB: 0.986, use: 'coding', small: true, source: 'https://ollama.com/library/qwen2.5-coder:1.5b' },
+ { id: 'gemma3:1b', name: 'Gemma 3 · 1B', purpose: 'Everyday text tasks', description: 'A small text-only model for summarizing, rewriting, and everyday questions.', downloadGB: 0.815, use: 'writing', small: true, source: 'https://ollama.com/library/gemma3:1b' },
+];
+export const DEFAULT_MODEL = CATALOG[0].id;
diff --git a/store/tools/local-ai/src/controller.mjs b/store/tools/local-ai/src/controller.mjs
new file mode 100644
index 000000000..79613900b
--- /dev/null
+++ b/store/tools/local-ai/src/controller.mjs
@@ -0,0 +1,191 @@
+import { EventEmitter } from 'node:events';
+import { randomUUID, createHash } from 'node:crypto';
+import os from 'node:os';
+import { setTimeout as delay } from 'node:timers/promises';
+import * as ollama from './ollama.mjs';
+import { resolveIntent, validateAction } from './intents.mjs';
+import { CATALOG } from './catalog.mjs';
+
+export const BENCHMARK_PROMPT = 'Explain how a bicycle works to a curious beginner. Discuss the pedals, chain, gears, wheels, brakes, and balance. Write a clear explanation of at least 250 words without a heading.';
+export const BENCHMARK_SETTINGS = { context: 2048, maxTokens: 128, seed: 42, temperature: 0, thinking: false };
+export const median = values => { const v = values.filter(Number.isFinite).sort((a, b) => a - b); return v.length ? (v[Math.floor((v.length - 1) / 2)] + v[Math.ceil((v.length - 1) / 2)]) / 2 : null; };
+const terminal = job => !['queued', 'running'].includes(job.status);
+export const safeError = error => /TLS handshake timeout|fetch failed/.test(error.message) ? 'The model download or runtime connection timed out. Check your connection and retry; partial downloads can resume.' : String(error.message || error).replace(/https?:\/\/[^\s"<>]+/g, url => url.split('?')[0]).slice(0, 700);
+
+export class Controller extends EventEmitter {
+ constructor(store, adapter = ollama, profile = {}) {
+ super();
+ this.profile = { id: 'ollama', name: 'Ollama', catalog: CATALOG, ...profile };
+ this.store = store; this.adapter = adapter; this.queue = []; this.aborts = new Map(); this.actions = new Map(); this.processing = false; this.closing = false;
+ this.system = { platform: os.platform(), arch: os.arch(), chip: os.cpus()[0]?.model || 'Local machine', totalMemory: os.totalmem(), cores: os.cpus().length };
+ this.runtime = { id: this.profile.id, name: this.profile.name, online: false, models: [] };
+ this.updatedAt = null;
+ }
+ snapshot() { return { runtime: this.runtime, system: { ...this.system, freeMemory: os.freemem() }, benchmarks: this.store.data.benchmarks, loadTests: this.store.data.loadTests, runs: this.store.data.runs, jobs: this.store.data.jobs, catalog: this.profile.catalog, updatedAt: this.updatedAt }; }
+ async refresh() {
+ if (this.refreshing) return this.refreshing;
+ this.refreshing = this.adapter.discover().then(runtime => { this.runtime = runtime; this.updatedAt = new Date().toISOString(); this.emit('state', this.snapshot()); }).finally(() => { this.refreshing = null; });
+ return this.refreshing;
+ }
+ async command(text, { selectedModel, history } = {}) {
+ await this.refresh();
+ const intent = (this.profile.resolveIntent || resolveIntent)(text, { models: this.runtime.models, selectedModel, profile: this.profile });
+ if (intent.kind === 'action') {
+ if (intent.action.type === 'chat') intent.action.history = history;
+ return { kind: 'job', job: await this.submit(intent.action) };
+ }
+ if (intent.kind === 'inventory') {
+ const models = this.runtime.models;
+ return { kind: 'reply', message: !this.runtime.online ? `${this.profile.name} is not running. Say “start ${this.profile.name}” or “run a lightweight model”.` : models.length ? `${models.length} model${models.length === 1 ? '' : 's'} on disk, ${models.filter(m => m.running).length} in memory.\n${models.map(m => `${m.name} — ${m.running ? 'in memory' : 'on disk'}`).join('\n')}` : `${this.profile.name} is ready. Say “run a lightweight model” to download your first model.` };
+ }
+ return intent;
+ }
+ async submit(input) {
+ if (this.closing) throw new Error('Harness is shutting down. Try again after it restarts.');
+ if (this.queue.length >= 8) throw new Error('There are already eight jobs waiting. Let them finish or cancel one.');
+ const action = this.profile.validateAction ? this.profile.validateAction(input) : validateAction(input, this.profile.validateModel);
+ const job = { id: randomUUID(), type: action.type, model: action.model || null, models: action.models || [], status: 'queued', progress: 0, message: 'Waiting for the model operator', createdAt: new Date().toISOString(), output: '', result: null };
+ this.store.data.jobs.unshift(job);
+ this.store.data.jobs = this.store.data.jobs.filter((j, i) => i < 40 || !terminal(j));
+ this.actions.set(job.id, action); this.queue.push(job.id);
+ await this.store.save(); this.emit('job', job);
+ void this.drain();
+ return job;
+ }
+ async cancel(id) {
+ const job = this.store.data.jobs.find(j => j.id === id);
+ if (!job) throw new Error('Job not found.');
+ if (terminal(job)) return job;
+ this.aborts.get(id)?.abort(new Error('Cancelled by you.'));
+ if (job.status === 'queued') { job.status = 'cancelled'; job.message = 'Cancelled before starting.'; job.completedAt = new Date().toISOString(); this.queue = this.queue.filter(queued => queued !== id); this.actions.delete(id); await this.store.save(); }
+ else job.message = 'Cancelling…';
+ this.emit('job', job); return job;
+ }
+ async drain() {
+ if (this.processing) return;
+ this.processing = true;
+ try {
+ while (this.queue.length && !this.closing) {
+ const id = this.queue.shift(); const job = this.store.data.jobs.find(j => j.id === id); const action = this.actions.get(id);
+ if (!job || job.status !== 'queued') continue;
+ const abort = new AbortController(); this.aborts.set(id, abort);
+ job.status = 'running'; job.startedAt = new Date().toISOString();
+ const progress = (message, value, extra) => { job.message = message; if (Number.isFinite(value)) job.progress = Math.max(0, Math.min(100, value)); if (extra) job.download = extra; this.emit('job', job); };
+ try {
+ await this.store.save();
+ progress(`Connecting to ${this.profile.name}`, 1);
+ await this.adapter.ensureRunning(this.store.directory, { signal: abort.signal, onProgress: progress });
+ await this.refresh();
+ job.result = await this.execute(action, { signal: abort.signal, progress, job });
+ abort.signal.throwIfAborted();
+ job.status = 'succeeded'; job.progress = 100; job.message = job.result.message;
+ } catch (error) {
+ job.status = abort.signal.aborted ? 'cancelled' : 'failed';
+ job.message = abort.signal.aborted ? 'Cancelled. Any downloaded files stay cached; retrying can resume the download.' : safeError(error);
+ } finally {
+ job.completedAt = new Date().toISOString(); this.aborts.delete(id); this.actions.delete(id);
+ try { await this.refresh(); await this.store.save(); } catch (error) { job.persistenceError = error.message; }
+ this.emit('job', job);
+ }
+ }
+ } finally { this.processing = false; }
+ }
+ async deploy(model, { signal, progress }, load = true, context) {
+ await this.refresh();
+ if (!this.runtime.models.some(m => m.id === model)) {
+ progress(`Inspecting ${model} before downloading`, 2);
+ await this.adapter.pullModel(model, { signal, onProgress: (message, value, extra) => progress(message, 3 + (value || 0) * 0.75, extra) });
+ }
+ await this.adapter.localModelDetails(model, { signal });
+ if (load) {
+ progress(`Loading ${model} into memory`, 82);
+ await this.adapter.loadModel(model, { signal, context: context ?? (this.profile.preserveServingConfig ? undefined : 4096) });
+ for (let i = 0; i < 15; i++) { await this.refresh(); if (this.runtime.models.some(m => m.id === model && m.running)) return; await delay(200, undefined, { signal }); }
+ throw new Error(`${this.profile.name} accepted the load request, but the model did not appear in memory. Check the runtime log and try again.`);
+ }
+ await this.refresh();
+ if (!this.runtime.models.some(m => m.id === model)) throw new Error(`${this.profile.name} did not report the model as installed after the download.`);
+ }
+ async execute(action, ctx) {
+ const { signal, progress, job } = ctx;
+ if (action.model) job.activeModel = action.model;
+ if (action.type === 'start') return { message: `${this.profile.name} ${this.runtime.version} is ready. ${this.runtime.models.length} models found on disk.` };
+ if (action.type === 'serve') {
+ await this.deploy(action.model, ctx, false);
+ progress(`Starting ${action.model} with the requested serving settings`, 80);
+ await this.adapter.loadModel(action.model, { ...action, signal });
+ await this.refresh();
+ return { model: action.model, serving: this.runtime.serving, message: `${action.model} is serving · ${this.runtime.serving.context.toLocaleString()} context · ${this.runtime.serving.maxSequences} parallel sequences · prefix cache ${this.runtime.serving.prefixCaching ? 'on' : 'off'}.` };
+ }
+ if (action.type === 'load_test') {
+ await this.deploy(action.model, ctx);
+ progress('Warming the server before the concurrent test', 5);
+ await this.adapter.generate(action.model, [{ role: 'user', content: BENCHMARK_PROMPT }], { ...BENCHMARK_SETTINGS, maxTokens: 32, signal });
+ const serving = { ...this.runtime.serving }; const samples = []; let next = 0; let complete = 0;
+ const began = performance.now();
+ const worker = async () => {
+ while (next < action.requests) {
+ signal.throwIfAborted(); const index = next++;
+ try {
+ const { metrics } = await this.adapter.generate(action.model, [{ role: 'user', content: `${BENCHMARK_PROMPT} Reference number: ${index + 1}.` }], { ...BENCHMARK_SETTINGS, signal });
+ samples.push({ index, status: 'succeeded', ...metrics });
+ } catch (error) { signal.throwIfAborted(); samples.push({ index, status: 'failed', error: safeError(error) }); }
+ complete++; progress(`${complete} of ${action.requests} requests completed · concurrency ${action.concurrency}`, 10 + complete / action.requests * 85);
+ }
+ };
+ const outcomes = await Promise.allSettled(Array.from({ length: action.concurrency }, worker)); signal.throwIfAborted();
+ const rejected = outcomes.find(outcome => outcome.status === 'rejected'); if (rejected) throw rejected.reason;
+ const totalMs = performance.now() - began;
+ const passed = samples.filter(s => s.status === 'succeeded');
+ const latency = passed.map(s => s.totalMs).sort((a, b) => a - b);
+ const result = { id: randomUUID(), at: new Date().toISOString(), model: action.model, digest: this.runtime.models.find(m => m.id === action.model)?.digest, runtime: { name: this.profile.name, version: this.runtime.version, backend: this.runtime.backend, backendVersion: this.runtime.backendVersion }, hardware: this.system, concurrency: action.concurrency, requests: action.requests, succeeded: passed.length, failed: samples.length - passed.length, totalMs, aggregateTokensPerSecond: passed.reduce((sum, s) => sum + s.tokens, 0) / (totalMs / 1000), medianLatencyMs: median(latency), p95LatencyMs: latency.length ? latency[Math.ceil(latency.length * 0.95) - 1] : null, medianFirstTokenMs: median(passed.map(s => s.firstTokenMs)), settings: { ...BENCHMARK_SETTINGS, context: serving.context }, serving, prompt: BENCHMARK_PROMPT, promptVariation: 'Reference number: {1-based request index}.', mode: 'one discarded warmup; repeated prompt prefixes may be cached; client end-to-end measurements', samples: samples.sort((a, b) => a.index - b.index) };
+ this.store.data.loadTests.unshift(result); this.store.data.loadTests = this.store.data.loadTests.slice(0, 200); await this.store.save();
+ if (!passed.length) throw new Error(`All ${action.requests} requests failed. The load-test record contains each error.`);
+ return { model: action.model, loadTestId: result.id, message: `${passed.length}/${action.requests} requests succeeded at concurrency ${action.concurrency} · ${result.aggregateTokensPerSecond.toFixed(1)} aggregate tokens/s · ${Math.round(result.p95LatencyMs)} ms p95 request latency · ${result.failed} errors.` };
+ }
+ if (['deploy', 'download'].includes(action.type)) {
+ await this.deploy(action.model, ctx, action.type === 'deploy');
+ return { model: action.model, message: action.type === 'deploy' ? `${action.model} is ready on this Mac. Ask a question, or say “benchmark it”. ${this.profile.readyMessage || 'Idle models unload after 15 minutes; their files stay on disk.'}` : `${action.model} is downloaded. Say “run ${action.model}” to load it into memory.` };
+ }
+ if (['unload', 'unload_all'].includes(action.type)) {
+ const targets = action.type === 'unload_all' ? this.runtime.models.filter(m => m.running).map(m => m.id) : [action.model];
+ for (const model of targets) { progress(`Releasing ${model} from memory`, 30); await this.adapter.unloadModel(model, { signal }); }
+ await this.refresh();
+ if (targets.some(model => this.runtime.models.some(m => m.id === model && m.running))) throw new Error('The runtime still reports a requested model in memory. It may be in use by another client.');
+ return { message: targets.length ? `${targets.length === 1 ? targets[0] : `${targets.length} models`} unloaded. Downloaded files are still on disk.` : 'No models were in memory.' };
+ }
+ if (action.type === 'chat') {
+ await this.deploy(action.model, ctx);
+ progress(`Generating with ${action.model}`, 90);
+ let lastEmit = 0;
+ const generated = await this.adapter.generate(action.model, [...action.history, { role: 'user', content: action.prompt }], { signal, onToken: text => { job.output += text; if (Date.now() - lastEmit > 80) { this.emit('job', job); lastEmit = Date.now(); } } });
+ const run = { id: randomUUID(), at: new Date().toISOString(), ...generated.metrics };
+ this.store.data.runs.unshift(run); this.store.data.runs = this.store.data.runs.slice(0, 200);
+ return { model: action.model, metrics: generated.metrics, message: `Response from ${action.model}.` };
+ }
+ if (action.type === 'benchmark') {
+ const results = [];
+ for (const [modelIndex, model] of action.models.entries()) {
+ job.activeModel = model;
+ await this.deploy(model, ctx, true, BENCHMARK_SETTINGS.context);
+ const metadata = this.runtime.models.find(m => m.id === model);
+ progress(`Warming up ${model} (not included in results)`, 5 + modelIndex / action.models.length * 90);
+ await this.adapter.generate(model, [{ role: 'user', content: BENCHMARK_PROMPT }], { ...BENCHMARK_SETTINGS, maxTokens: 32, signal });
+ const samples = [];
+ for (let i = 0; i < 3; i++) {
+ progress(`${model} · measured run ${i + 1} of 3`, 10 + (modelIndex + i / 3) / action.models.length * 85);
+ const { metrics } = await this.adapter.generate(model, [{ role: 'user', content: BENCHMARK_PROMPT }], { ...BENCHMARK_SETTINGS, signal });
+ if (!Number.isFinite(metrics.tokensPerSecond) || !Number.isFinite(metrics.firstTokenMs)) throw new Error(`${this.profile.name} did not return usable timing data; no benchmark was recorded.`);
+ samples.push(metrics);
+ }
+ await this.refresh();
+ const result = { id: randomUUID(), model, digest: metadata?.digest || null, at: new Date().toISOString(), runtime: { name: this.profile.name, version: this.runtime.version, mlxVersion: this.runtime.mlxVersion, backend: this.runtime.backend, backendVersion: this.runtime.backendVersion }, hardware: this.system, settings: BENCHMARK_SETTINGS, prompt: BENCHMARK_PROMPT, promptHash: createHash('sha256').update(BENCHMARK_PROMPT).digest('hex'), mode: this.profile.benchmarkMode || 'warm; repeated prompt may use prefix cache', samples, medianTokensPerSecond: median(samples.map(s => s.tokensPerSecond)), medianFirstTokenMs: median(samples.map(s => s.firstTokenMs)), medianFirstOutputMs: median(samples.map(s => s.firstOutputMs)), residentBytes: this.runtime.models.find(m => m.id === model)?.resident?.size || null };
+ this.store.data.benchmarks.unshift(result); this.store.data.benchmarks = this.store.data.benchmarks.slice(0, 200); results.push(result);
+ await this.store.save();
+ }
+ return { model: action.models.at(-1), benchmarkIds: results.map(r => r.id), message: results.map(r => `${r.model}: ${r.medianTokensPerSecond.toFixed(1)} tokens/s · ${Math.round(r.medianFirstTokenMs)} ms to first token (median of 3 warm runs).`).join('\n') };
+ }
+ throw new Error('Unknown action.');
+ }
+ async close() { this.closing = true; for (const abort of this.aborts.values()) abort.abort(new Error('Harness shutting down.')); await this.store.save(); await this.adapter.close?.(); }
+}
diff --git a/store/tools/local-ai/src/intents.mjs b/store/tools/local-ai/src/intents.mjs
new file mode 100644
index 000000000..7aa35e8df
--- /dev/null
+++ b/store/tools/local-ai/src/intents.mjs
@@ -0,0 +1,91 @@
+import { CATALOG, DEFAULT_MODEL } from './catalog.mjs';
+import { validateModel } from './ollama.mjs';
+
+export const HELP = 'Try “run a lightweight model”, “find a model for coding”, “benchmark it”, “ask it: explain unified memory”, or “free up memory”. You can also name any local Ollama model, such as qwen3:0.6b.';
+
+// Only explicit operational requests enter the executor. Model replies are text,
+// never instructions. There is no shell-command generation or eval here.
+export function resolveIntent(input, { models = [], selectedModel, profile = {} } = {}) {
+ const catalog = profile.catalog || CATALOG;
+ const defaultModel = profile.defaultModel || DEFAULT_MODEL;
+ const normalize = profile.validateModel || validateModel;
+ const help = profile.help || HELP;
+ if (typeof input !== 'string' || !input.trim() || input.length > 4000) throw new Error('Enter a request between 1 and 4,000 characters.');
+ const text = input.trim();
+ const lower = text.toLowerCase();
+ const selected = selectedModel ? normalize(selectedModel) : null;
+ const preferred = selected || models.find(m => m.running)?.id || models[0]?.id;
+ const action = (type, props = {}) => ({ kind: 'action', action: { type, ...props } });
+ const reply = message => ({ kind: 'reply', message });
+ const choose = () => reply('Which model? Select one in the model map, or include its name in your request.');
+ // Prompt contents must never be parsed as commands or model identifiers.
+ const ask = text.match(/^(?:(?:please |can you )?)(?:ask|tell|chat with)\s+(.+?)\s*:\s*([\s\S]+)$/i);
+ if (ask) {
+ const target = /^(it|me|us|the model|my model)$/i.test(ask[1]) ? preferred : normalize(ask[1].trim());
+ return target ? action('chat', { model: target, prompt: ask[2] }) : choose();
+ }
+ const names = [...new Set([...models.map(m => m.id), ...catalog.map(m => m.id)])];
+ const mentioned = names.filter(name => lower.includes(name.toLowerCase()));
+ for (const match of text.matchAll(profile.modelPattern || /\b(?:[a-z0-9][a-z0-9._-]*\/)?[a-z0-9][a-z0-9._-]*:[a-z0-9][a-z0-9._-]*\b/g)) {
+ if (!mentioned.includes(match[0]) && !/^\d+:\d+$/.test(match[0])) mentioned.push(normalize(match[0]));
+ }
+ const modelForTask = /cod(?:e|ing)|programming/.test(lower) ? catalog[1].id : /writing|summari[sz]|rewrite/.test(lower) ? catalog[2].id : defaultModel;
+ const lead = lower.replace(/^(?:please\s+|(?:can|could|would) you\s+|i(?:'d| would) like (?:you )?to\s+|i want (?:you )?to\s+)+/, '');
+ if (/\b(don['’]?t|do not|never|not yet|cancel|abort)\b/.test(lower) && /\b(run|load|download|deploy|start|stop|unload|benchmark|compare|cancel|abort)\b/.test(lower)) return reply('I haven’t started an action. Use the Cancel button on an active job to stop it.');
+ if (/^(how (?:do|would|can|should)|what (?:if|happens)|why|explain|help)\b/.test(lead)) return reply(help);
+ if (/^(show|list|what|which|check|status|refresh)\b/.test(lead) && /models?|running|loaded|status|memory|installed/.test(lead) || /^(status|models|refresh)[.!?]?$/.test(lead)) return { kind: 'inventory' };
+ if (/^(find|recommend|suggest|pick|choose|what['’]?s|what is)\b/.test(lead) && /model|coding|writing/.test(lead)) return { kind: 'catalog', message: 'Start small, then choose a larger model once you have a baseline. These downloads run locally on this Mac.', models: /coding|code|programming/.test(lower) ? [catalog[1]] : /writing|summari[sz]/.test(lower) ? [catalog[2]] : catalog };
+ if (/^(delete|remove|erase)\b/.test(lead)) return reply('This pilot keeps downloaded models on disk. To release their memory, say “unload it” or “free up memory”.');
+ if (/^(free (?:up )?(?:my |the )?memory|unload all(?: models)?|stop all(?: models)?|release (?:all )?memory)[.!?]?$/.test(lead)) return action('unload_all');
+ if (/^(start|launch|open) (?:the )?(?:ollama|mlx(?:-lm)?|runtime|server)[.!?]?$/.test(lead)) return action('start');
+
+ const operation = lead.match(/^(run|deploy|load|start|download|pull|install|unload|stop|benchmark|test|measure|compare)\b/);
+ if (operation) {
+ if (/\b(?:and then|then|after that)\b/.test(lead) || /\band\s+(run|deploy|download|stop|unload|benchmark|ask)\b/.test(lead)) return reply('Let’s do one operation at a time. Start with the first request, then ask for the next step when it finishes.');
+ const verb = operation[1];
+ const rest = lead.slice(verb.length).trim().replace(/[.!?]+$/, '');
+ let model = mentioned[0];
+ if (!model && /^[a-z0-9][a-z0-9._-]*(?:\/[a-z0-9][a-z0-9._-]*)?$/.test(rest) && !/^(it|this|that|model|ollama|speed|performance|all)$/.test(rest)) {
+ model = names.find(name => name.split(':')[0] === rest) || normalize(rest);
+ }
+ if (!model && /\b(it|this model|that model|my model|the model|the running model)\b/.test(rest)) model = preferred;
+ if (['benchmark', 'test', 'measure', 'compare'].includes(verb)) {
+ if (verb === 'compare' && /\ball\b/.test(rest)) return models.length > 4 ? reply('This pilot compares up to four models at a time. Name the models to compare.') : models.length ? action('benchmark', { models: models.map(m => m.id) }) : choose();
+ const targets = mentioned.length ? mentioned : model ? [model] : preferred ? [preferred] : [];
+ if (verb === 'compare' && targets.length < 2) return reply('Name two models from this framework to compare. Each gets the same prompt and settings.');
+ return targets.length ? action('benchmark', { models: targets }) : choose();
+ }
+ if (['unload', 'stop'].includes(verb)) return model ? action('unload', { model }) : choose();
+ if (!model && /\b(lightweight|small|tiny|fast|coding|writing|starter|first)\b/.test(rest)) model = modelForTask;
+ if (!model) return reply('Name the model you want, or say “run a lightweight model” and I’ll use Qwen 3 0.6B.');
+ return action(['download', 'pull', 'install'].includes(verb) ? 'download' : 'deploy', { model });
+ }
+ if (/\b(benchmark|speed|fast|tokens per second|performance)\b/.test(lower) && /^(how|test|measure)\b/.test(lead)) return preferred ? action('benchmark', { models: [preferred] }) : choose();
+ if (/^(chat|talk)(?: with| to)?(?: it| the model)?[.!?]?$/.test(lead)) return preferred ? reply(`${preferred} is selected. Ask a question below, or say “ask it: …”.`) : choose();
+ if (/^(run|deploy|download|load|start|stop|unload|benchmark|compare|install|delete|remove)\b/.test(lead) || /\b(shell|terminal|sudo|rm -rf)\b/.test(lower)) return reply(help);
+ return preferred ? action('chat', { model: preferred, prompt: text }) : reply(help);
+}
+
+export function validateAction(input, normalize = validateModel) {
+ if (!input || typeof input !== 'object' || Array.isArray(input)) throw new Error('Provide a model action.');
+ const { type } = input;
+ if (!['start', 'download', 'deploy', 'unload', 'unload_all', 'benchmark', 'chat'].includes(type)) throw new Error('That action is not supported by this pilot.');
+ const action = { type };
+ if (['download', 'deploy', 'unload', 'chat'].includes(type)) action.model = normalize(input.model);
+ if (type === 'benchmark') {
+ if (!Array.isArray(input.models) || !input.models.length || input.models.length > 4) throw new Error('Benchmark between one and four models at a time.');
+ action.models = [...new Set(input.models.map(normalize))];
+ }
+ if (type === 'chat') {
+ if (typeof input.prompt !== 'string' || !input.prompt.trim() || input.prompt.length > 4000) throw new Error('Enter a prompt between 1 and 4,000 characters.');
+ action.prompt = input.prompt.trim();
+ action.history = [];
+ if (Array.isArray(input.history)) {
+ action.history = input.history.slice(-6).map(m => {
+ if (!['user', 'assistant'].includes(m.role) || typeof m.content !== 'string' || m.content.length > 6000) throw new Error('Invalid chat history.');
+ return { role: m.role, content: m.content };
+ });
+ }
+ }
+ return action;
+}
diff --git a/store/tools/local-ai/src/json-worker.mjs b/store/tools/local-ai/src/json-worker.mjs
new file mode 100644
index 000000000..7129ce6a1
--- /dev/null
+++ b/store/tools/local-ai/src/json-worker.mjs
@@ -0,0 +1,80 @@
+import { spawn } from 'node:child_process';
+import { access, mkdir, open } from 'node:fs/promises';
+import { constants } from 'node:fs';
+import { createInterface } from 'node:readline';
+import { randomUUID } from 'node:crypto';
+import { join } from 'node:path';
+
+// Private, request-correlated JSON-lines transport. It never invokes a shell.
+export class JsonWorker {
+ constructor({ python, script, directory, env = {}, name }) {
+ Object.assign(this, { python, script, directory, env, name });
+ this.child = null; this.ready = false; this.pending = new Map(); this.state = {}; this.closed = false;
+ }
+ async start() {
+ if (this.closed) throw new Error(`${this.name} is closing.`);
+ if (this.stopping) await this.stopping;
+ if (this.child && this.ready) return;
+ if (this.starting) return this.starting;
+ this.starting = this.launch().finally(() => { this.starting = null; });
+ return this.starting;
+ }
+ async launch() {
+ await access(this.python, constants.X_OK).catch(() => { throw new Error(`${this.name} needs setup. Run this harness's setup command.`); });
+ await mkdir(this.directory, { recursive: true });
+ const log = await open(join(this.directory, 'model-cache.log'), 'a', 0o600);
+ const child = spawn(this.python, ['-u', this.script], { stdio: ['pipe', 'pipe', log.fd], env: { ...process.env, ...this.env } });
+ await log.close(); this.child = child; this.ready = false;
+ child.stdin.on('error', () => {});
+ await new Promise((resolve, reject) => {
+ const timer = setTimeout(() => { this.stop(child); reject(new Error(`${this.name} startup timed out.`)); }, 45000);
+ const fail = error => { clearTimeout(timer); reject(error); };
+ child.once('error', error => { if (this.child === child) this.child = null; fail(error); });
+ const lines = createInterface({ input: child.stdout });
+ lines.on('line', line => {
+ if (this.child !== child) return;
+ try {
+ if (line.length > 8 * 1024 * 1024) throw new Error('Oversized worker event.');
+ const event = JSON.parse(line);
+ if (event.event === 'ready') { clearTimeout(timer); this.ready = true; this.state = event; resolve(); return; }
+ if (event.state) this.state = { ...this.state, ...event.state };
+ const pending = this.pending.get(event.id); if (!pending) return;
+ if (event.event === 'done') pending.resolve(event.result);
+ else if (event.event === 'error') pending.reject(new Error(event.message));
+ else pending.onEvent(event);
+ } catch (error) { this.stop(child); fail(error); }
+ });
+ child.once('close', (code, signal) => {
+ clearTimeout(timer); lines.close();
+ if (this.child === child) { this.child = null; this.ready = false; }
+ const error = new Error(`${this.name} stopped (${signal || code}).`);
+ for (const pending of this.pending.values()) if (pending.child === child) pending.reject(error);
+ fail(error);
+ });
+ });
+ }
+ stop(child = this.child) {
+ if (!child || this.stopping) return this.stopping;
+ this.ready = false;
+ this.stopping = new Promise(resolve => {
+ const timer = setTimeout(() => child.kill('SIGKILL'), 1500);
+ child.once('close', () => { clearTimeout(timer); resolve(); });
+ child.kill('SIGTERM');
+ }).finally(() => { this.stopping = null; });
+ return this.stopping;
+ }
+ async request(action, body = {}, { signal, onEvent = () => {}, timeout = 180000 } = {}) {
+ signal?.throwIfAborted(); await this.start(); signal?.throwIfAborted();
+ const child = this.child; const id = randomUUID();
+ return new Promise((resolve, reject) => {
+ let settled = false;
+ const finish = (fn, value) => { if (settled) return; settled = true; clearTimeout(timer); signal?.removeEventListener('abort', abort); this.pending.delete(id); fn(value); };
+ const abort = () => { this.stop(child); finish(reject, signal.reason || new Error('Cancelled.')); };
+ const timer = setTimeout(() => { this.stop(child); finish(reject, new Error(`${this.name} ${action} timed out.`)); }, timeout);
+ this.pending.set(id, { child, resolve: value => finish(resolve, value), reject: error => finish(reject, error), onEvent });
+ signal?.addEventListener('abort', abort, { once: true });
+ child.stdin.write(JSON.stringify({ id, action, ...body }) + '\n', error => { if (error) finish(reject, error); });
+ });
+ }
+ async close() { this.closed = true; await this.starting?.catch(() => {}); await this.stop(); }
+}
diff --git a/store/tools/local-ai/src/mlx.mjs b/store/tools/local-ai/src/mlx.mjs
new file mode 100644
index 000000000..a347b2ea1
--- /dev/null
+++ b/store/tools/local-ai/src/mlx.mjs
@@ -0,0 +1,156 @@
+import { spawn } from 'node:child_process';
+import { access, mkdir, open } from 'node:fs/promises';
+import { constants } from 'node:fs';
+import { createInterface } from 'node:readline';
+import { join } from 'node:path';
+import { randomUUID } from 'node:crypto';
+import { totalmem } from 'node:os';
+import { performance } from 'node:perf_hooks';
+
+export function validateMlxModel(model) {
+ if (typeof model !== 'string' || model.length > 180 || !/^[A-Za-z0-9][A-Za-z0-9_.-]*\/[A-Za-z0-9][A-Za-z0-9_.-]*$/.test(model) || model.includes('..')) throw new Error('Use a Hugging Face model ID, such as mlx-community/Qwen3-0.6B-4bit.');
+ return model;
+}
+
+export const MLX_CATALOG = [
+ { id: 'mlx-community/Qwen3-0.6B-4bit', name: 'Qwen 3 · 0.6B', purpose: 'A lightweight first model', description: 'A small MLX model for trying local chat and measuring this Mac.', downloadGB: 0.4, use: 'general', small: true },
+ { id: 'mlx-community/Qwen2.5-Coder-1.5B-Instruct-4bit', name: 'Qwen 2.5 Coder · 1.5B', purpose: 'Small coding assistant', description: 'A compact MLX coding model for explanations and short snippets.', downloadGB: 1.0, use: 'coding', small: true },
+ { id: 'mlx-community/Qwen2.5-0.5B-Instruct-4bit', name: 'Qwen 2.5 · 0.5B', purpose: 'Small baseline', description: 'A compact instruction model for short answers and a second performance baseline.', downloadGB: 0.3, use: 'writing', small: true },
+].map(model => ({ ...model, source: `https://huggingface.co/${model.id}` }));
+
+export const MLX_PROFILE = {
+ id: 'mlx-lm', name: 'MLX-LM', catalog: MLX_CATALOG, defaultModel: MLX_CATALOG[0].id,
+ validateModel: validateMlxModel, modelPattern: /\b[A-Za-z0-9][A-Za-z0-9_.-]*\/[A-Za-z0-9][A-Za-z0-9_.-]*\b/g,
+ help: 'Try “run a lightweight model”, “benchmark it”, “ask it: explain unified memory”, or “free up memory”. Use Hugging Face MLX model IDs, such as mlx-community/Qwen3-0.6B-4bit.',
+ benchmarkMode: 'warm model; fresh prompt cache for each sample',
+ readyMessage: 'MLX-LM keeps one model loaded per workspace. Switching models releases the previous model; idle models unload after 15 minutes.',
+};
+
+export class MlxAdapter {
+ constructor({ packageRoot, dataDir, python, interpreterArgs = ['-u'] } = {}) {
+ this.python = python || join(packageRoot, '.venv/bin/python');
+ this.interpreterArgs = interpreterArgs;
+ this.workerPath = join(packageRoot, 'worker.py');
+ this.dataDir = dataDir;
+ this.child = null; this.starting = null; this.pending = new Map(); this.generation = 0;
+ this.current = { models: [] }; this.version = null; this.mlxVersion = null; this.expiresAt = null;
+ this.installed = false; this.closed = false;
+ }
+ async ensureRunning(_directory, { signal } = {}) {
+ signal?.throwIfAborted();
+ if (this.closed) throw new Error('The MLX-LM workspace is closing.');
+ if (this.stopping) await this.stopping;
+ signal?.throwIfAborted();
+ if (this.closed) throw new Error('The MLX-LM workspace is closing.');
+ if (this.child && this.ready) return;
+ if (this.starting) return this.starting;
+ this.starting = this.startWorker().finally(() => { this.starting = null; });
+ return this.starting;
+ }
+ async startWorker() {
+ try { await access(this.python, constants.X_OK); this.installed = true; }
+ catch { throw new Error('MLX-LM needs setup. Run this harness’s setup command to install its isolated Python environment.'); }
+ await mkdir(this.dataDir, { recursive: true });
+ const log = await open(join(this.dataDir, 'mlx-lm.log'), 'a', 0o600);
+ const child = spawn(this.python, [...this.interpreterArgs, this.workerPath], { stdio: ['pipe', 'pipe', log.fd], env: { ...process.env, HF_HUB_DISABLE_TELEMETRY: '1', TOKENIZERS_PARALLELISM: 'false', HARNESS_MLX_REGISTRY: join(this.dataDir, 'mlx-models.json') } });
+ await log.close();
+ this.child = child; this.ready = false;
+ child.stdin.on('error', () => {}); // The write callback rejects the corresponding request.
+ const generation = ++this.generation;
+ await new Promise((resolve, reject) => {
+ const timer = setTimeout(() => { child.kill('SIGKILL'); reject(new Error('MLX-LM did not start within 45 seconds. Check .harness/mlx-lm.log.')); }, 45000);
+ const startupFailure = error => { clearTimeout(timer); reject(error); };
+ child.once('error', startupFailure);
+ const lines = createInterface({ input: child.stdout });
+ lines.on('line', line => {
+ try {
+ if (line.length > 8 * 1024 * 1024) throw new Error('MLX-LM returned an oversized event.');
+ const event = JSON.parse(line);
+ if (this.generation !== generation) return;
+ if (event.event === 'ready') {
+ clearTimeout(timer); this.ready = true; this.current = event;
+ this.version = event.version; this.mlxVersion = event.mlxVersion;
+ resolve(); return;
+ }
+ if (event.state) this.current = { ...this.current, ...event.state };
+ const pending = this.pending.get(event.id); if (!pending) return;
+ if (event.event === 'error') pending.reject(new Error(event.message));
+ else if (event.event === 'done') pending.resolve(event.result);
+ else pending.onEvent(event);
+ } catch (error) { child.kill('SIGKILL'); startupFailure(error); }
+ });
+ child.once('exit', (code, signal) => {
+ clearTimeout(timer); lines.close();
+ const error = new Error(`MLX-LM worker stopped (${signal || code}). Its loaded model was released. Check .harness/mlx-lm.log if unexpected.`);
+ if (this.generation === generation) { this.child = null; this.ready = false; this.expiresAt = null; this.current = { ...this.current, models: this.current.models.map(m => ({ ...m, running: false, resident: null })) }; }
+ for (const request of this.pending.values()) if (request.child === child) request.reject(error);
+ startupFailure(error);
+ });
+ });
+ }
+ stopWorker(child, signal = 'SIGTERM') {
+ if (child !== this.child) return;
+ this.ready = false;
+ if (this.stopping) return;
+ this.stopping = new Promise(resolve => {
+ const timer = setTimeout(() => child.kill('SIGKILL'), 1500);
+ child.once('close', () => { clearTimeout(timer); resolve(); });
+ child.kill(signal);
+ }).finally(() => { this.stopping = null; });
+ }
+ async request(action, body = {}, { signal, onEvent = () => {}, timeout = 180000 } = {}) {
+ await this.ensureRunning(undefined, { signal }); signal?.throwIfAborted();
+ const child = this.child; const id = randomUUID();
+ return new Promise((resolve, reject) => {
+ let settled = false;
+ const finish = (fn, value) => { if (settled) return; settled = true; clearTimeout(timer); signal?.removeEventListener('abort', abort); this.pending.delete(id); fn(value); };
+ const abort = () => { this.stopWorker(child); finish(reject, signal.reason || new Error('Cancelled.')); };
+ const timer = setTimeout(() => { this.stopWorker(child, 'SIGKILL'); finish(reject, new Error(`MLX-LM ${action} timed out. Its worker was stopped to release memory.`)); }, timeout);
+ this.pending.set(id, { child, resolve: value => finish(resolve, value), reject: error => finish(reject, error), onEvent });
+ signal?.addEventListener('abort', abort, { once: true });
+ child.stdin.write(JSON.stringify({ id, action, ...body }) + '\n', error => { if (error) finish(reject, error); });
+ });
+ }
+ async discover() {
+ let error;
+ try {
+ await this.ensureRunning();
+ if (!this.pending.size) {
+ const loaded = this.current.models.find(m => m.running);
+ if (loaded && this.expiresAt && Date.now() > this.expiresAt) await this.unloadModel(loaded.id);
+ await this.request('inventory');
+ }
+ } catch (e) { error = e.message; }
+ return { id: 'mlx-lm', name: 'MLX-LM', online: Boolean(this.child && this.ready), installed: this.installed, version: this.version, mlxVersion: this.mlxVersion, error,
+ endpoint: 'Private local worker', cacheRoot: this.current.cacheRoot, memoryScope: 'MLX active allocations; one model per workspace',
+ models: this.current.models.map(m => ({ ...m, resident: m.resident ? { ...m.resident, expires_at: this.expiresAt ? new Date(this.expiresAt).toISOString() : null } : null })) };
+ }
+ async pullModel(model, { signal, onProgress = () => {} } = {}) {
+ return this.request('download', { model: validateMlxModel(model), memoryBudget: Math.floor(totalmem() * 0.8) }, { signal, timeout: 30 * 60 * 1000, onEvent: event => { if (event.event === 'progress') onProgress(event.message, event.percent, { completed: event.completed, total: event.total }); } });
+ }
+ localModelDetails(model, options) { return this.request('details', { model: validateMlxModel(model) }, options); }
+ async loadModel(model, { signal, context = 4096 } = {}) {
+ const result = await this.request('load', { model: validateMlxModel(model), context }, { signal });
+ this.expiresAt = Date.now() + 15 * 60 * 1000; return result;
+ }
+ async unloadModel(model, { signal } = {}) {
+ const result = await this.request('unload', { model: validateMlxModel(model) }, { signal });
+ this.expiresAt = null; return result;
+ }
+ async generate(model, messages, { signal, onToken = () => {}, ...options } = {}) {
+ const started = performance.now(); let firstTokenMs = null; let firstOutputMs = null;
+ const result = await this.request('generate', { model: validateMlxModel(model), messages, options }, { signal, onEvent: event => {
+ if (event.event === 'first_token' && firstTokenMs === null) firstTokenMs = performance.now() - started;
+ if (event.event === 'token') { if (firstOutputMs === null && /\S/.test(event.text)) firstOutputMs = performance.now() - started; onToken(event.text); }
+ } });
+ this.expiresAt = Date.now() + 15 * 60 * 1000;
+ result.metrics.firstTokenMs = firstTokenMs; result.metrics.firstOutputMs = firstOutputMs;
+ return result;
+ }
+ async close() {
+ this.closed = true;
+ if (this.starting) await this.starting.catch(() => {});
+ if (this.child) this.stopWorker(this.child);
+ await this.stopping;
+ }
+}
diff --git a/store/tools/local-ai/src/ollama.mjs b/store/tools/local-ai/src/ollama.mjs
new file mode 100644
index 000000000..17063efec
--- /dev/null
+++ b/store/tools/local-ai/src/ollama.mjs
@@ -0,0 +1,187 @@
+import { access, mkdir, open, statfs } from 'node:fs/promises';
+import { constants } from 'node:fs';
+import { homedir, totalmem } from 'node:os';
+import { join } from 'node:path';
+import { spawn } from 'node:child_process';
+import { setTimeout as delay } from 'node:timers/promises';
+import { performance } from 'node:perf_hooks';
+
+export const OLLAMA_URL = 'http://127.0.0.1:11434';
+export const DEFAULT_CONTEXT = 4096;
+let starting;
+
+export function validateModel(model) {
+ if (typeof model !== 'string' || model.length > 180 || !/^[a-zA-Z0-9][a-zA-Z0-9._-]*(?:\/[a-zA-Z0-9][a-zA-Z0-9._-]*)?(?::[a-zA-Z0-9][a-zA-Z0-9._-]*)?$/.test(model) || model.includes('..')) throw new Error('Use an Ollama model name, such as qwen3:0.6b.');
+ if (/(?:^|[-:])cloud(?:$|[-:])/i.test(model)) throw new Error('This harness runs local models. Choose a model without a cloud tag.');
+ return model.includes(':') ? model : `${model}:latest`;
+}
+
+export async function* readNDJSON(body) {
+ const decoder = new TextDecoder();
+ let buffer = '';
+ for await (const chunk of body) {
+ buffer += typeof chunk === 'string' ? chunk : decoder.decode(chunk, { stream: true });
+ if (buffer.length > 8 * 1024 * 1024) throw new Error('The runtime returned an oversized event.');
+ let newline;
+ while ((newline = buffer.indexOf('\n')) !== -1) {
+ const line = buffer.slice(0, newline).trim(); buffer = buffer.slice(newline + 1);
+ if (line) { const event = JSON.parse(line); if (event.error) throw new Error(event.error); yield event; }
+ }
+ }
+ buffer += decoder.decode();
+ if (buffer.trim()) { const event = JSON.parse(buffer); if (event.error) throw new Error(event.error); yield event; }
+}
+
+export async function streamRequest(path, body, { signal, timeout = 30 * 60 * 1000, onEvent = () => {} } = {}) {
+ const response = await fetch(`${OLLAMA_URL}${path}`, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ ...body, stream: true }), signal: signal ? AbortSignal.any([signal, AbortSignal.timeout(timeout)]) : AbortSignal.timeout(timeout) });
+ if (!response.ok) { const message = await response.text(); throw new Error(message.slice(0, 600) || `Ollama returned ${response.status}`); }
+ let last;
+ for await (const event of readNDJSON(response.body)) { last = event; await onEvent(event); }
+ return last;
+}
+
+export async function ensureRunning(dataDir, { signal, onProgress = () => {} } = {}) {
+ try { return await ollamaJSON('/api/version', { timeout: 1500, signal }); } catch {}
+ signal?.throwIfAborted();
+ if (starting) return starting;
+ starting = (async () => {
+ const binary = await findOllama();
+ if (!binary) throw new Error('Ollama is not installed. Install the official Ollama app from ollama.com, then try again.');
+ onProgress('Starting Ollama on this Mac');
+ await mkdir(dataDir, { recursive: true });
+ const log = await open(join(dataDir, 'ollama.log'), 'a', 0o600);
+ let failure;
+ const child = spawn(binary, ['serve'], { env: { ...process.env, OLLAMA_HOST: '127.0.0.1:11434', OLLAMA_NO_CLOUD: '1' }, detached: true, stdio: ['ignore', log.fd, log.fd] });
+ child.on('error', error => { failure = error; });
+ child.unref();
+ await log.close();
+ const deadline = Date.now() + 30000;
+ while (Date.now() < deadline) {
+ if (failure) throw failure;
+ signal?.throwIfAborted();
+ try { return await ollamaJSON('/api/version', { timeout: 1000, signal }); } catch {}
+ if (child.exitCode !== null) throw new Error(`Ollama exited during startup. Details are in ${join(dataDir, 'ollama.log')}.`);
+ await delay(300, undefined, { signal });
+ }
+ throw new Error('Ollama did not become ready within 30 seconds. Check the runtime log and try again.');
+ })().finally(() => { starting = null; });
+ return starting;
+}
+
+export async function modelManifest(model, { signal } = {}) {
+ const normalized = validateModel(model);
+ const [path, tag] = normalized.split(':');
+ const repository = path.includes('/') ? path : `library/${path}`;
+ const url = `https://registry.ollama.ai/v2/${repository}/manifests/${tag}`;
+ const response = await fetch(url, { signal: signal ? AbortSignal.any([signal, AbortSignal.timeout(12000)]) : AbortSignal.timeout(12000), headers: { Accept: 'application/vnd.docker.distribution.manifest.v2+json' } });
+ if (!response.ok) throw new Error(response.status === 404 ? `The model ${model} was not found in Ollama's registry. Check its name and tag.` : `Could not inspect the download size (${response.status}). Try again when the registry is reachable.`);
+ const manifest = await response.json();
+ const layers = [...(manifest.layers || []), ...(manifest.config ? [manifest.config] : [])];
+ if (!layers.length || layers.some(layer => !Number.isSafeInteger(layer.size) || layer.size < 0)) throw new Error('The model registry returned an invalid manifest.');
+ return { bytes: layers.reduce((total, layer) => total + layer.size, 0), repository, tag };
+}
+
+export async function pullModel(model, options = {}) {
+ model = validateModel(model);
+ const manifest = await modelManifest(model, options);
+ if (manifest.bytes > totalmem() * 0.8) throw new Error(`This model download is ${(manifest.bytes / 1024 ** 3).toFixed(1)} GiB, too large for the pilot's memory budget on this Mac. Choose a smaller quantization or model.`);
+ try {
+ const disk = await statfs(process.env.OLLAMA_MODELS || join(homedir(), '.ollama'));
+ if (disk.bavail * disk.bsize < manifest.bytes + 2 * 1024 ** 3) throw new Error('There is not enough free disk space for this download plus 2 GiB of headroom.');
+ } catch (error) { if (!['ENOENT', 'EPERM', 'EACCES'].includes(error.code)) throw error; }
+ const layers = new Map();
+ const last = await streamRequest('/api/pull', { model }, { ...options, onEvent: event => {
+ if (event.digest && event.total) layers.set(event.digest, { total: event.total, completed: event.completed || 0 });
+ const known = [...layers.values()];
+ const completed = known.reduce((sum, layer) => sum + layer.completed, 0);
+ options.onProgress?.(event.status || 'Downloading model', Math.min(99, Math.floor(completed / manifest.bytes * 100)), { completed, total: manifest.bytes });
+ } });
+ if (last?.status !== 'success') throw new Error('The model download ended before Ollama confirmed success. You can retry to resume it.');
+ return manifest;
+}
+
+export async function localModelDetails(model, options = {}) {
+ const details = await ollamaJSON('/api/show', { method: 'POST', body: { model: validateModel(model) }, timeout: 15000, ...options });
+ if (details.remote_host || details.remote_model) throw new Error('This model forwards inference to a remote service. Choose a local model for this harness.');
+ if (Array.isArray(details.capabilities) && !details.capabilities.includes('completion')) throw new Error('This pilot supports language generation models. The selected model does not expose completion capability.');
+ return details;
+}
+
+export async function loadModel(model, { signal, context = DEFAULT_CONTEXT, keepAlive = '15m' } = {}) {
+ await localModelDetails(model, { signal });
+ return ollamaJSON('/api/generate', { method: 'POST', body: { model, prompt: '', stream: false, keep_alive: keepAlive, options: { num_ctx: context } }, timeout: 180000, signal });
+}
+
+export async function unloadModel(model, { signal } = {}) {
+ return ollamaJSON('/api/generate', { method: 'POST', body: { model: validateModel(model), stream: false, keep_alive: 0 }, timeout: 30000, signal });
+}
+
+export async function generate(model, messages, { signal, maxTokens = 512, context = DEFAULT_CONTEXT, keepAlive = '15m', seed = 42, temperature = 0.3, thinking = true, onToken = () => {} } = {}) {
+ const details = await localModelDetails(model, { signal });
+ const start = performance.now();
+ let firstTokenMs = null;
+ let firstOutputMs = null;
+ let text = '';
+ const body = { model, messages, options: { num_predict: maxTokens, num_ctx: context, temperature, seed }, keep_alive: keepAlive };
+ if (details.capabilities?.includes('thinking')) body.think = /^gpt-oss(?::|$)/.test(model) ? 'low' : Boolean(thinking);
+ const last = await streamRequest('/api/chat', body, { signal, timeout: 180000, onEvent: event => {
+ const piece = event.message?.content || '';
+ if (firstTokenMs === null && (piece || event.message?.thinking)) firstTokenMs = performance.now() - start;
+ if (piece) { if (firstOutputMs === null) firstOutputMs = performance.now() - start; text += piece; onToken(piece); }
+ } });
+ if (!last?.done) throw new Error('The model response ended unexpectedly. Partial output was not counted as a successful run.');
+ const elapsedMs = performance.now() - start;
+ return { text, metrics: {
+ model, ...responseMetrics(last), firstTokenMs, firstOutputMs, elapsedMs,
+ context, maxTokens, temperature, seed, thinking: body.think ?? null,
+ } };
+}
+
+export function responseMetrics(last) {
+ return {
+ tokens: last.eval_count ?? null,
+ promptTokens: last.prompt_eval_count ?? null,
+ tokensPerSecond: last.eval_duration > 0 && Number.isFinite(last.eval_count) ? last.eval_count / (last.eval_duration / 1e9) : null,
+ promptTokensPerSecond: last.prompt_eval_duration > 0 && Number.isFinite(last.prompt_eval_count) ? last.prompt_eval_count / (last.prompt_eval_duration / 1e9) : null,
+ loadMs: Number.isFinite(last.load_duration) ? last.load_duration / 1e6 : null,
+ evalMs: Number.isFinite(last.eval_duration) ? last.eval_duration / 1e6 : null,
+ doneReason: last.done_reason || null,
+ };
+}
+
+export async function findOllama() {
+ const candidates = [process.env.OLLAMA_BIN, '/usr/local/bin/ollama', '/opt/homebrew/bin/ollama', '/Applications/Ollama.app/Contents/Resources/ollama', ...String(process.env.PATH || '').split(':').map(p => join(p, 'ollama'))].filter(Boolean);
+ for (const candidate of candidates) {
+ try { await access(candidate, constants.X_OK); return candidate; } catch {}
+ }
+ return null;
+}
+
+export async function ollamaJSON(path, { method = 'GET', body, timeout = 5000, signal } = {}) {
+ const response = await fetch(`${OLLAMA_URL}${path}`, {
+ method, headers: { 'Content-Type': 'application/json' },
+ body: body === undefined ? undefined : JSON.stringify(body),
+ signal: signal ? AbortSignal.any([signal, AbortSignal.timeout(timeout)]) : AbortSignal.timeout(timeout),
+ });
+ const result = await response.json();
+ if (!response.ok || result.error) throw new Error(result.error || `Ollama returned ${response.status}`);
+ return result;
+}
+
+export async function discover() {
+ const binary = await findOllama();
+ try {
+ const [version, tags, running] = await Promise.all([ollamaJSON('/api/version'), ollamaJSON('/api/tags'), ollamaJSON('/api/ps')]);
+ return {
+ id: 'ollama', name: 'Ollama', online: true, installed: Boolean(binary), version: version.version, endpoint: OLLAMA_URL,
+ models: (tags.models || []).map(m => ({
+ id: m.name, name: m.name, size: m.size, digest: m.digest, modifiedAt: m.modified_at,
+ family: m.details?.family, parameters: m.details?.parameter_size, quantization: m.details?.quantization_level,
+ running: Boolean((running.models || []).find(r => r.name === m.name || r.model === m.name)),
+ resident: (running.models || []).find(r => r.name === m.name || r.model === m.name) || null,
+ })).filter(m => !/(?:^|[-:])cloud(?:$|[-:])/i.test(m.name)),
+ };
+ } catch (error) {
+ return { id: 'ollama', name: 'Ollama', online: false, installed: Boolean(binary), version: null, endpoint: OLLAMA_URL, models: [], error: error.message };
+ }
+}
diff --git a/store/tools/local-ai/src/server.mjs b/store/tools/local-ai/src/server.mjs
new file mode 100644
index 000000000..06984c602
--- /dev/null
+++ b/store/tools/local-ai/src/server.mjs
@@ -0,0 +1,143 @@
+import http from 'node:http';
+import { readFile, writeFile, rename } from 'node:fs/promises';
+import { fileURLToPath, pathToFileURL } from 'node:url';
+import { join, dirname, resolve } from 'node:path';
+import { randomBytes, timingSafeEqual } from 'node:crypto';
+import { Store } from './store.mjs';
+import { Controller } from './controller.mjs';
+import { createViewerRelay } from './viewer-relay.mjs';
+
+export const ROOT = dirname(dirname(fileURLToPath(import.meta.url)));
+const assets = { '/': ['index.html', 'text/html'], '/app.js': ['app.js', 'text/javascript'], '/styles.css': ['styles.css', 'text/css'], '/assets/ollama-icon.svg': ['assets/ollama-icon.svg', 'image/svg+xml'], '/assets/ollama-icon.png': ['assets/ollama-icon.png', 'image/png'], '/favicon.ico': ['assets/ollama-icon.png', 'image/png'], '/assets/mlx-icon.svg': ['assets/mlx-icon.svg', 'image/svg+xml'], '/assets/mlx-icon.png': ['assets/mlx-icon.png', 'image/png'] };
+assets['/assets/vllm-icon.png'] = ['assets/vllm-icon.png', 'image/png'];
+const json = (res, status, value) => { res.writeHead(status, { 'Content-Type': 'application/json; charset=utf-8' }); res.end(JSON.stringify(value)); };
+
+async function readBody(req) {
+ if (!/^application\/json(?:\s*;|$)/i.test(req.headers['content-type'] || '')) throw Object.assign(new Error('Use application/json for control requests.'), { status: 415 });
+ if (Number(req.headers['content-length']) > 65536) throw Object.assign(new Error('Request too large.'), { status: 413 });
+ const chunks = []; let size = 0;
+ for await (const chunk of req) { size += chunk.length; if (size > 65536) throw Object.assign(new Error('Request too large.'), { status: 413 }); chunks.push(chunk); }
+ try { return JSON.parse(Buffer.concat(chunks).toString('utf8')); } catch { throw Object.assign(new Error('Invalid JSON request.'), { status: 400 }); }
+}
+
+export function createServer(controller, { assetRoot = join(ROOT, 'dist') } = {}) {
+ const token = randomBytes(32).toString('hex');
+ const events = new Set();
+ const sendEvent = (res, type, data) => { if (res.writableLength > 1024 * 1024) { res.destroy(); events.delete(res); } else if (!res.destroyed) res.write(`event: ${type}\ndata: ${JSON.stringify(data)}\n\n`); };
+ const onState = state => { for (const res of events) sendEvent(res, 'state', state); };
+ const onJob = job => { for (const res of events) sendEvent(res, 'job', job); };
+ controller.on('state', onState); controller.on('job', onJob);
+ const server = http.createServer(async (req, res) => {
+ try {
+ const port = server.address()?.port;
+ const hosts = [`127.0.0.1:${port}`, `localhost:${port}`];
+ if (!hosts.includes(req.headers.host)) return json(res, 403, { error: 'Local requests only.' });
+ const origin = `http://${req.headers.host}`;
+ if (req.headers.origin && req.headers.origin !== origin || req.headers['sec-fetch-site'] === 'cross-site') return json(res, 403, { error: 'Cross-origin control is disabled.' });
+ const url = new URL(req.url, origin);
+ res.setHeader('X-Content-Type-Options', 'nosniff');
+ res.setHeader('Referrer-Policy', 'no-referrer');
+ res.setHeader('Cache-Control', 'no-store');
+ res.setHeader('Content-Security-Policy', "default-src 'self'; script-src 'self'; style-src 'self' 'unsafe-inline'; connect-src 'self'; img-src 'self' data:; frame-ancestors 'self'; base-uri 'none'; form-action 'self'");
+ if (req.method === 'GET' && url.pathname === '/api/health') return json(res, 200, { application: 'local-ai-harnesses', protocol: 1, workspace: controller.workspace, runtime: controller.profile?.id || 'ollama', pid: process.pid });
+ if (req.method === 'POST') {
+ const submitted = Buffer.from(String(req.headers['x-harness-token'] || ''));
+ const expected = Buffer.from(token);
+ if (submitted.length !== expected.length || !timingSafeEqual(submitted, expected)) return json(res, 403, { error: 'Refresh the viewer before sending a control request.' });
+ const body = await readBody(req);
+ if (!body || typeof body !== 'object' || Array.isArray(body)) return json(res, 400, { error: 'Expected a JSON object.' });
+ if (url.pathname === '/api/command') return json(res, 200, await controller.command(body.text, { selectedModel: body.selectedModel, history: body.history }));
+ if (url.pathname === '/api/action') return json(res, 202, { kind: 'job', job: await controller.submit(body) });
+ if (url.pathname === '/api/cancel') return json(res, 200, { job: await controller.cancel(body.id) });
+ }
+ if (req.method === 'GET' && url.pathname === '/api/state') { await controller.refresh(); return json(res, 200, { ...controller.snapshot(), token }); }
+ if (req.method === 'GET' && url.pathname === '/api/jobs') return json(res, 200, { jobs: controller.snapshot().jobs });
+ if (req.method === 'GET' && url.pathname === '/api/events') {
+ if (events.size >= 32) return json(res, 503, { error: 'Too many viewer connections.' });
+ res.writeHead(200, { 'Content-Type': 'text/event-stream', Connection: 'keep-alive', 'X-Accel-Buffering': 'no' });
+ res.write('retry: 2000\n\n'); events.add(res); sendEvent(res, 'state', controller.snapshot());
+ req.on('close', () => events.delete(res)); return;
+ }
+ if (req.method === 'GET' && url.pathname === '/api/export') {
+ res.setHeader('Content-Disposition', `attachment; filename="${controller.profile?.id || 'ollama'}-benchmarks.json"`);
+ return json(res, 200, { exportedAt: new Date().toISOString(), system: controller.system, benchmarks: controller.snapshot().benchmarks, loadTests: controller.snapshot().loadTests });
+ }
+ if (req.method === 'GET' && Object.hasOwn(assets, url.pathname)) {
+ let [file, type] = assets[url.pathname];
+ if (url.pathname === '/favicon.ico' && controller.profile?.id === 'mlx-lm') file = 'assets/mlx-icon.png';
+ if (url.pathname === '/favicon.ico' && controller.profile?.id === 'vllm') file = 'assets/vllm-icon.png';
+ const data = await readFile(join(assetRoot, file));
+ res.writeHead(200, { 'Content-Type': `${type}; charset=utf-8` }); return res.end(data);
+ }
+ json(res, 404, { error: 'Not found.' });
+ } catch (error) { if (!res.headersSent) json(res, error.status || 400, { error: error.message }); else res.end(); }
+ });
+ const heartbeat = setInterval(() => { for (const res of events) res.write(': heartbeat\n\n'); }, 15000); heartbeat.unref();
+ server.on('close', () => { clearInterval(heartbeat); controller.off('state', onState); controller.off('job', onJob); });
+ server.closeEvents = () => { for (const res of events) res.end(); events.clear(); };
+ server.requestTimeout = 30000;
+ return server;
+}
+
+export async function start(port = Number(process.env.HARNESS_VIEWER_PORT || process.env.HARNESS_PORT || 4310), options = {}) {
+ if (!Number.isInteger(port) || port < 0 || port > 65535) throw new Error('The viewer port must be between 0 and 65535.');
+ const workspace = resolve(process.env.HARNESS_WORKSPACE || options.workspaceRoot || ROOT);
+ const runtime = options.profile?.id || 'ollama';
+ let marker;
+ try { marker = JSON.parse(await readFile(join(workspace, 'local-ai.json'), 'utf8')); }
+ catch (error) { if (error.code !== 'ENOENT') throw error; }
+ if (marker?.runtime && marker.runtime !== runtime) throw new Error(`This workspace belongs to ${marker.runtime}. Open a separate ${runtime} workspace.`);
+ let existing;
+ let conflictingRuntime;
+ try {
+ const endpoint = JSON.parse(await readFile(join(workspace, '.harness/endpoint.json'), 'utf8'));
+ if (endpoint.workspace === workspace && Number.isInteger(endpoint.port) && endpoint.port > 0 && endpoint.port <= 65535) {
+ const health = await (await fetch(`http://127.0.0.1:${endpoint.port}/api/health`, { signal: AbortSignal.timeout(1200) })).json();
+ if (health.application === 'local-ai-harnesses' && health.workspace === workspace && health.pid === endpoint.pid) {
+ if ((health.runtime || 'ollama') === runtime) existing = endpoint;
+ else conflictingRuntime = health.runtime || 'ollama';
+ }
+ }
+ } catch {}
+ if (conflictingRuntime) throw new Error(`This workspace is already serving ${conflictingRuntime}. Open a separate ${runtime} workspace.`);
+ if (existing) {
+ if (existing.port === port) throw new Error(`This workspace is already running at http://127.0.0.1:${port}. Open that viewer instead of starting a second copy.`);
+ const relay = createViewerRelay(existing.port);
+ await new Promise((resolve, reject) => { relay.once('error', reject); relay.listen(port, '127.0.0.1', resolve); });
+ console.log(`Local: http://127.0.0.1:${relay.address().port} (shared workspace service)`);
+ const stopRelay = () => { relay.closeAllConnections(); relay.close(); };
+ process.once('SIGINT', stopRelay); process.once('SIGTERM', stopRelay);
+ return { server: relay, controller: null };
+ }
+ const store = await new Store(join(workspace, '.harness')).init();
+ const controller = new Controller(store, options.adapter, options.profile);
+ controller.workspace = workspace;
+ const server = createServer(controller, options);
+ await new Promise((resolve, reject) => { server.once('error', reject); server.listen(port, '127.0.0.1', resolve); });
+ const boundPort = server.address().port;
+ await writeFile(join(store.directory, 'endpoint.json'), JSON.stringify({ port: boundPort, pid: process.pid, workspace, runtime, startedAt: new Date().toISOString() }), { mode: 0o600 });
+ console.log(`Local: http://127.0.0.1:${boundPort}`);
+ let verdictWriting = Promise.resolve();
+ let lastVerdict;
+ const verdict = async snapshot => {
+ const last = snapshot.jobs[0];
+ const failed = last?.status === 'failed';
+ const affected = last?.activeModel || last?.model || last?.models?.join(', ') || controller.profile.name;
+ const failure = failed ? `${affected} · ${last.type} failed: ${last.message}` : undefined;
+ const data = { spec: 1, ready: snapshot.runtime.online && !failed, summary: failed ? failure : snapshot.runtime.online ? `${snapshot.runtime.models.length} local models · ${controller.profile.name} connected` : `${controller.profile.name} is ready to start`, findings: failed ? [{ severity: 'error', kind: 'runtime', message: failure }] : [], updatedAt: new Date().toISOString() };
+ const key = JSON.stringify({ ...data, updatedAt: undefined });
+ if (key === lastVerdict) return;
+ lastVerdict = key;
+ verdictWriting = verdictWriting.catch(() => {}).then(async () => { const temp = join(store.directory, 'verdict.json.tmp'); await writeFile(temp, JSON.stringify(data, null, 2), { mode: 0o600 }); await rename(temp, join(store.directory, 'verdict.json')); });
+ await verdictWriting;
+ };
+ controller.on('state', snapshot => { void verdict(snapshot).catch(() => {}); });
+ controller.on('job', () => { void verdict(controller.snapshot()).catch(() => {}); });
+ await controller.refresh();
+ const poll = setInterval(() => { void controller.refresh().catch(error => console.error('Discovery:', error.message)); }, 5000); poll.unref();
+ let stopping = false;
+ const stop = async () => { if (stopping) return; stopping = true; clearInterval(poll); server.closeEvents(); server.close(); await controller.close(); };
+ process.once('SIGINT', stop); process.once('SIGTERM', stop);
+ return { server, controller };
+}
+if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) await start();
diff --git a/store/tools/local-ai/src/store.mjs b/store/tools/local-ai/src/store.mjs
new file mode 100644
index 000000000..59957fa1b
--- /dev/null
+++ b/store/tools/local-ai/src/store.mjs
@@ -0,0 +1,21 @@
+import { mkdir, readFile, writeFile, rename } from 'node:fs/promises';
+import { join } from 'node:path';
+
+export class Store {
+ constructor(directory) { this.directory = directory; this.file = join(directory, 'state.json'); this.data = { benchmarks: [], loadTests: [], runs: [], jobs: [] }; this.writing = Promise.resolve(); }
+ async init() {
+ await mkdir(this.directory, { recursive: true });
+ try {
+ const parsed = JSON.parse(await readFile(this.file, 'utf8'));
+ for (const key of ['benchmarks', 'loadTests', 'runs', 'jobs']) if (Array.isArray(parsed[key])) this.data[key] = parsed[key];
+ this.data.jobs = this.data.jobs.map(job => ['queued', 'running'].includes(job.status) ? { ...job, status: 'interrupted', message: 'Harness restarted before this job completed. Retry the action to continue.', completedAt: new Date().toISOString() } : { ...job, message: String(job.message || '').replace(/https?:\/\/[^\s"<>]+/g, url => url.split('?')[0]).slice(0, 700) });
+ } catch (error) { if (error.code !== 'ENOENT') throw new Error(`Could not read saved Harness data: ${error.message}`); }
+ await this.save();
+ return this;
+ }
+ save() {
+ const content = JSON.stringify(this.data, null, 2);
+ this.writing = this.writing.catch(() => {}).then(async () => { const temp = `${this.file}.tmp`; await writeFile(temp, content, { mode: 0o600 }); await rename(temp, this.file); });
+ return this.writing;
+ }
+}
diff --git a/store/tools/local-ai/src/viewer-relay.mjs b/store/tools/local-ai/src/viewer-relay.mjs
new file mode 100644
index 000000000..7aa643ddd
--- /dev/null
+++ b/store/tools/local-ai/src/viewer-relay.mjs
@@ -0,0 +1,26 @@
+import http from 'node:http';
+
+// Multiple views of one workspace share one controller and job queue. The native
+// host assigns a fresh viewer port; this loopback relay serves that port without
+// opening a second writer against the workspace's history.
+export function createViewerRelay(targetPort) {
+ if (!Number.isInteger(targetPort) || targetPort < 1 || targetPort > 65535) throw new Error('Invalid local service port.');
+ const server = http.createServer((req, res) => {
+ const port = server.address().port;
+ const host = req.headers.host;
+ const reject = (status, message) => { res.writeHead(status, { 'Content-Type': 'application/json' }); res.end(JSON.stringify({ error: message })); };
+ if (![`localhost:${port}`, `127.0.0.1:${port}`].includes(host)) return reject(403, 'Local requests only.');
+ if (req.headers.origin && req.headers.origin !== `http://${host}` || req.headers['sec-fetch-site'] === 'cross-site') return reject(403, 'Cross-origin control is disabled.');
+ if (!req.url.startsWith('/') || req.url.startsWith('//')) return reject(400, 'Invalid local path.');
+ const headers = { ...req.headers, host: `127.0.0.1:${targetPort}` };
+ if (headers.origin) headers.origin = `http://127.0.0.1:${targetPort}`;
+ const upstream = http.request({ hostname: '127.0.0.1', port: targetPort, path: req.url, method: req.method, headers }, response => {
+ res.writeHead(response.statusCode || 502, response.headers); response.pipe(res);
+ });
+ upstream.on('error', () => { if (!res.headersSent) reject(502, 'The workspace service stopped. Reopen this viewer to reconnect.'); else res.end(); });
+ req.on('aborted', () => upstream.destroy()); res.on('close', () => upstream.destroy());
+ req.pipe(upstream);
+ });
+ server.requestTimeout = 30000;
+ return server;
+}
diff --git a/store/tools/local-ai/src/vllm.mjs b/store/tools/local-ai/src/vllm.mjs
new file mode 100644
index 000000000..8603b27f4
--- /dev/null
+++ b/store/tools/local-ai/src/vllm.mjs
@@ -0,0 +1,218 @@
+import { spawn } from 'node:child_process';
+import { mkdir, open, readFile } from 'node:fs/promises';
+import net from 'node:net';
+import { join } from 'node:path';
+import { randomBytes } from 'node:crypto';
+import { totalmem } from 'node:os';
+import { setTimeout as delay } from 'node:timers/promises';
+import { JsonWorker } from './json-worker.mjs';
+import { MLX_CATALOG, validateMlxModel } from './mlx.mjs';
+import { resolveIntent, validateAction } from './intents.mjs';
+
+export const DEFAULT_SERVING = Object.freeze({ context: 4096, maxSequences: 4, memoryFraction: 0.1, prefixCaching: true });
+const bounded = (value, name, min, max, integer = true) => {
+ if (!Number.isFinite(value) || (integer && !Number.isInteger(value)) || value < min || value > max) throw new Error(`${name} must be ${integer ? 'an integer ' : ''}between ${min} and ${max}.`);
+ return value;
+};
+export function validateServing(input = {}, base = DEFAULT_SERVING) {
+ const config = { ...base };
+ if (input.context !== undefined) config.context = bounded(input.context, 'Context', 512, 16384);
+ if (input.maxSequences !== undefined) config.maxSequences = bounded(input.maxSequences, 'Parallel sequences', 1, 8);
+ if (input.memoryFraction !== undefined) config.memoryFraction = bounded(input.memoryFraction, 'Metal memory fraction', 0.05, 0.8, false);
+ if (input.prefixCaching !== undefined) {
+ if (typeof input.prefixCaching !== 'boolean') throw new Error('Prefix caching must be true or false.');
+ config.prefixCaching = input.prefixCaching;
+ }
+ return config;
+}
+export function validateVllmAction(input) {
+ if (!input || !['serve', 'load_test'].includes(input.type)) return validateAction(input, validateMlxModel);
+ const action = { type: input.type, model: validateMlxModel(input.model) };
+ if (input.type === 'serve') {
+ const allowed = ['type', 'model', ...Object.keys(DEFAULT_SERVING)];
+ if (Object.keys(input).some(key => !allowed.includes(key))) throw new Error('Unknown serving setting.');
+ validateServing(input);
+ for (const key of Object.keys(DEFAULT_SERVING)) if (input[key] !== undefined) action[key] = input[key];
+ } else {
+ if (Object.keys(input).some(key => !['type', 'model', 'concurrency', 'requests'].includes(key))) throw new Error('Unknown load-test setting.');
+ action.concurrency = bounded(input.concurrency ?? 4, 'Concurrency', 1, 8);
+ action.requests = bounded(input.requests ?? 8, 'Requests', action.concurrency, 32);
+ }
+ return action;
+}
+export const VLLM_PROFILE = {
+ id: 'vllm', name: 'vLLM', catalog: MLX_CATALOG, defaultModel: MLX_CATALOG[0].id,
+ preserveServingConfig: true,
+ validateModel: validateMlxModel, validateAction: validateVllmAction,
+ modelPattern: /\b[A-Za-z0-9][A-Za-z0-9_.-]*\/[A-Za-z0-9][A-Za-z0-9_.-]*\b/g,
+ help: 'Try “run a lightweight model”, “benchmark it”, “test 4 concurrent requests”, or “free up memory”. Ask the agent to change context, parallel sequences, Metal memory budget or prefix caching.',
+ benchmarkMode: 'warm server; repeated prompt may use prefix cache; client end-to-end token throughput, including prompt processing and HTTP',
+ readyMessage: 'This workspace owns one vLLM server using the Metal backend. Idle servers stop after 15 minutes.',
+ resolveIntent(text, options) {
+ const match = typeof text === 'string' && text.trim().match(/^(?:please )?(?:test|benchmark) (\d+) concurrent requests[.!?]?$/i);
+ if (match) {
+ const model = options.selectedModel || options.models.find(m => m.running)?.id;
+ return model ? { kind: 'action', action: { type: 'load_test', model, concurrency: Number(match[1]), requests: Math.max(8, Number(match[1])) } } : { kind: 'reply', message: 'Load or select a model first.' };
+ }
+ if (typeof text === 'string' && /^(?:please )?start vllm[.!?]?$/i.test(text.trim())) return { kind: 'action', action: { type: 'start' } };
+ return resolveIntent(text, options);
+ },
+};
+
+// Decode streaming UTF-8 before splitting frames. Chunk boundaries are not tokens.
+export async function readCompletion(body, onToken = () => {}, now = () => performance.now()) {
+ const start = now(); const decoder = new TextDecoder(); let buffer = ''; let text = ''; let usage; let finishReason; let firstTokenMs = null; let firstOutputMs = null; let done = false;
+ const frame = data => {
+ if (data === '[DONE]') { done = true; return; }
+ const event = JSON.parse(data);
+ if (event.error) throw new Error(event.error.message || 'vLLM generation failed.');
+ if (event.usage) usage = event.usage;
+ for (const choice of event.choices || []) {
+ const delta = choice.delta || {};
+ const content = delta.content || '';
+ const reasoning = delta.reasoning_content || delta.reasoning || '';
+ if ((content.length || reasoning.length) && firstTokenMs === null) firstTokenMs = now() - start;
+ if (/\S/.test(content) && firstOutputMs === null) firstOutputMs = now() - start;
+ if (content) { text += content; onToken(content); }
+ if (choice.finish_reason) finishReason = choice.finish_reason;
+ }
+ };
+ const consume = () => {
+ let boundary;
+ while ((boundary = buffer.indexOf('\n\n')) >= 0) {
+ const raw = buffer.slice(0, boundary); buffer = buffer.slice(boundary + 2);
+ const data = raw.split('\n').filter(line => line.startsWith('data:')).map(line => line.slice(5).trimStart()).join('\n');
+ if (data) frame(data);
+ }
+ if (buffer.length > 4 * 1024 * 1024) throw new Error('Oversized vLLM stream frame.');
+ };
+ for await (const bytes of body) { buffer += decoder.decode(bytes, { stream: true }).replace(/\r/g, ''); consume(); }
+ buffer += decoder.decode(); consume();
+ if (!done || !Number.isInteger(usage?.completion_tokens) || !Number.isInteger(usage?.prompt_tokens)) throw new Error('vLLM returned an incomplete stream or missing token usage; no measurement was recorded.');
+ return { text, usage, firstTokenMs, firstOutputMs, finishReason };
+}
+
+async function freePort() {
+ const server = net.createServer();
+ await new Promise((resolve, reject) => { server.once('error', reject); server.listen(0, '127.0.0.1', resolve); });
+ const port = server.address().port;
+ await new Promise((resolve, reject) => server.close(error => error ? reject(error) : resolve()));
+ return port;
+}
+export function parsePrometheus(text) {
+ const sums = {};
+ for (const line of text.split('\n')) {
+ const match = line.match(/^(vllm:[a-z_]+)(?:\{.*\})?\s+([\d.eE+-]+)$/);
+ if (match && Number.isFinite(Number(match[2]))) sums[match[1]] = (sums[match[1]] || 0) + Number(match[2]);
+ }
+ return { runningRequests: sums['vllm:num_requests_running'] ?? null, waitingRequests: sums['vllm:num_requests_waiting'] ?? null, cacheUsage: sums['vllm:kv_cache_usage_perc'] ?? null, prefixQueries: sums['vllm:prefix_cache_queries_total'] ?? null, prefixHits: sums['vllm:prefix_cache_hits_total'] ?? null };
+}
+export class VllmAdapter {
+ constructor({ packageRoot, dataDir, python } = {}) {
+ Object.assign(this, { packageRoot, dataDir }); this.python = python || join(packageRoot, '.venv/bin/python');
+ this.worker = new JsonWorker({ python: this.python, script: join(packageRoot, 'cache.py'), directory: dataDir, name: 'vLLM model cache', env: { HARNESS_MODEL_REGISTRY: join(dataDir, 'vllm-models.json'), HF_HUB_DISABLE_TELEMETRY: '1' } });
+ this.config = { ...DEFAULT_SERVING }; this.server = null; this.loaded = null; this.inflight = 0; this.closed = false; this.expiresAt = null;
+ }
+ async ensureRunning(_directory, { signal } = {}) { signal?.throwIfAborted(); if (this.closed) throw new Error('vLLM is closing.'); await this.worker.start(); signal?.throwIfAborted(); }
+ async discover() {
+ try {
+ await this.ensureRunning();
+ if (this.loaded && !this.inflight && Date.now() > this.expiresAt) await this.stopServer();
+ // Avoid putting inventory requests behind a long model download.
+ if (!this.worker.pending.size) await this.worker.request('inventory', {}, { timeout: 10000 });
+ let rss = null; let metrics = {};
+ if (this.loaded && this.server) {
+ const record = this.server;
+ try {
+ metrics = parsePrometheus(await (await this.fetchNative('/metrics', { signal: AbortSignal.timeout(1500) }, record)).text());
+ if (!this.worker.pending.size) rss = (await this.worker.request('memory', { pid: record.child.pid }, { timeout: 5000 })).rssBytes;
+ } catch {} // Scrape delay does not change server health or invent measurements.
+ }
+ const versions = this.worker.state.versions;
+ const models = (this.worker.state.models || []).map(model => ({ ...model, running: model.id === this.loaded, resident: model.id === this.loaded ? { size: rss, context_length: this.config.context, expires_at: new Date(this.expiresAt).toISOString() } : null }));
+ return { id: 'vllm', name: 'vLLM', online: true, version: versions?.vllm, backend: 'Metal', backendVersion: versions?.['vllm-metal'], mlxVersion: versions?.mlx, models, serving: { ...this.config, status: this.loaded ? 'serving' : this.server ? 'starting' : 'idle', model: this.loaded, processRssBytes: rss, ...metrics } };
+ } catch (error) { return { id: 'vllm', name: 'vLLM', online: false, models: [], error: error.message, backend: 'Metal', serving: { ...this.config, status: 'offline' } }; }
+ }
+ async localModelDetails(model, { signal } = {}) { return this.worker.request('details', { model: validateMlxModel(model) }, { signal }); }
+ async pullModel(model, { signal, onProgress = () => {} } = {}) {
+ return this.worker.request('download', { model: validateMlxModel(model), memoryBudget: Math.floor(totalmem() * 0.8) }, { signal, timeout: 2 * 60 * 60 * 1000, onEvent: event => { if (event.event === 'progress') onProgress(event.message, event.percent, { completed: event.completed, total: event.total }); } });
+ }
+ async fetchNative(path, options = {}, record = this.server) {
+ if (!record) throw new Error('The vLLM server is not running.');
+ const response = await fetch(`http://127.0.0.1:${record.port}${path}`, { ...options, headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${record.key}`, ...options.headers } });
+ if (!response.ok) throw new Error(`vLLM HTTP ${response.status}: ${(await response.text()).slice(0, 700)}`);
+ return response;
+ }
+ async loadModel(model, options = {}) {
+ const config = validateServing(options, this.config); validateMlxModel(model); options.signal?.throwIfAborted();
+ if (this.loaded === model && JSON.stringify(config) === JSON.stringify(this.config)) { this.touch(); return; }
+ if (this.inflight) throw new Error('Wait for current requests before changing the server.');
+ await this.stopServer();
+ if (this.closed) throw new Error('vLLM is closing.');
+ const details = await this.localModelDetails(model, options);
+ const port = await freePort(); const key = randomBytes(32).toString('hex');
+ await mkdir(this.dataDir, { recursive: true });
+ const log = await open(join(this.dataDir, 'vllm.log'), 'a', 0o600);
+ const args = ['-u', join(this.packageRoot, 'server.py'), 'serve', details.path, '--host', '127.0.0.1', '--port', String(port), '--served-model-name', model, '--tokenizer', details.path, '--max-model-len', String(config.context), '--max-num-seqs', String(config.maxSequences), '--max-num-batched-tokens', String(config.context), '--gpu-memory-utilization', String(config.memoryFraction), config.prefixCaching ? '--enable-prefix-caching' : '--no-enable-prefix-caching', '--middleware', 'guard.LocalOnlyMiddleware'];
+ if (details.family === 'qwen3') args.push('--reasoning-parser', 'qwen3');
+ options.signal?.throwIfAborted();
+ const child = spawn(this.python, args, { detached: true, stdio: ['ignore', log.fd, log.fd], cwd: this.packageRoot, env: { ...process.env, PYTHONPATH: this.packageRoot, VLLM_API_KEY: key, HARNESS_VLLM_PORT: String(port), HARNESS_PARENT_PID: String(process.pid), HF_HUB_OFFLINE: '1', HF_HUB_DISABLE_TELEMETRY: '1', TOKENIZERS_PARALLELISM: 'false', VLLM_NO_USAGE_STATS: '1', DO_NOT_TRACK: '1', VLLM_METAL_MEMORY_FRACTION: 'auto', VLLM_MLX_DEVICE: 'gpu' } });
+ const record = { child, port, key, exited: false, error: null };
+ record.exit = new Promise(resolve => { child.once('error', error => { record.error = error; }); child.once('close', () => { record.exited = true; if (child.pid) { try { process.kill(-child.pid, 'SIGKILL'); } catch {} } if (this.server === record) { this.server = null; this.loaded = null; } resolve(); }); });
+ this.server = record; this.config = config; await log.close();
+ const abort = () => { void this.stopServer(record); };
+ options.signal?.addEventListener('abort', abort, { once: true });
+ try {
+ const deadline = Date.now() + 180000;
+ while (Date.now() < deadline) {
+ options.signal?.throwIfAborted();
+ if (record.exited) {
+ const tail = (await readFile(join(this.dataDir, 'vllm.log'), 'utf8')).slice(-2500);
+ throw new Error(`vLLM did not start. ${record.error?.message || tail}`);
+ }
+ try {
+ const response = await this.fetchNative('/v1/models', { signal: AbortSignal.timeout(1000) }, record);
+ const inventory = await response.json();
+ if (inventory.data?.some(item => item.id === model)) { options.signal?.throwIfAborted(); this.loaded = model; this.touch(); return; }
+ } catch (error) { if (options.signal?.aborted) throw error; }
+ await delay(300, undefined, { signal: options.signal });
+ }
+ throw new Error('vLLM startup exceeded three minutes. Check .harness/vllm.log.');
+ } catch (error) { await this.stopServer(record); throw error; }
+ finally { options.signal?.removeEventListener('abort', abort); }
+ }
+ touch() { this.expiresAt = Date.now() + 15 * 60 * 1000; }
+ async stopServer(record = this.server) {
+ if (!record) return;
+ if (record.stopping) return record.stopping;
+ if (this.server === record) this.loaded = null;
+ record.stopping = (async () => {
+ // Only this detached, harness-owned process group is eligible for termination.
+ const kill = signal => { if (!record.child.pid) return; try { process.kill(-record.child.pid, signal); } catch (error) { if (error.code !== 'ESRCH') throw error; } };
+ if (!record.exited) kill('SIGTERM');
+ const timer = setTimeout(() => { try { kill('SIGKILL'); } catch {} }, 4000);
+ try { await record.exit; } finally { clearTimeout(timer); }
+ // The close handler reaps remaining group members once, including after crashes.
+ if (this.server === record) { this.server = null; this.loaded = null; }
+ })();
+ return record.stopping;
+ }
+ async unloadModel(model) { if (this.loaded === model || !this.loaded) await this.stopServer(); }
+ async generate(model, messages, { signal, onToken, maxTokens = 512, temperature = 0.7, seed = 42, thinking = false } = {}) {
+ signal?.throwIfAborted();
+ if (this.loaded !== model) await this.loadModel(model, { signal });
+ const record = this.server; this.inflight++; this.touch();
+ const requestSignal = AbortSignal.any([...(signal ? [signal] : []), AbortSignal.timeout(180000)]);
+ const start = performance.now(); const abort = () => { void this.stopServer(record); };
+ signal?.addEventListener('abort', abort, { once: true });
+ try {
+ const response = await this.fetchNative('/v1/chat/completions', { method: 'POST', signal: requestSignal, body: JSON.stringify({ model, messages, max_tokens: maxTokens, temperature, seed, stream: true, stream_options: { include_usage: true }, chat_template_kwargs: { enable_thinking: thinking } }) }, record);
+ const headersMs = performance.now() - start;
+ const completion = await readCompletion(response.body, onToken);
+ signal?.throwIfAborted();
+ const totalMs = performance.now() - start;
+ return { text: completion.text, metrics: { model, tokens: completion.usage.completion_tokens, promptTokens: completion.usage.prompt_tokens, tokensPerSecond: completion.usage.completion_tokens / (totalMs / 1000), firstTokenMs: completion.firstTokenMs === null ? null : headersMs + completion.firstTokenMs, firstOutputMs: completion.firstOutputMs === null ? null : headersMs + completion.firstOutputMs, totalMs, thinking, doneReason: completion.finishReason, metric: 'client_end_to_end', serving: { ...this.config } } };
+ } finally { signal?.removeEventListener('abort', abort); if (requestSignal.aborted) await this.stopServer(record); this.inflight--; this.touch(); }
+ }
+ async close() { this.closed = true; await this.stopServer(); await this.worker.close(); }
+}
diff --git a/store/tools/local-ai/test/core.test.mjs b/store/tools/local-ai/test/core.test.mjs
new file mode 100644
index 000000000..4e92b08d8
--- /dev/null
+++ b/store/tools/local-ai/test/core.test.mjs
@@ -0,0 +1,99 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import { mkdtemp, rm, readFile } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { resolveIntent, validateAction } from '../src/intents.mjs';
+import { validateModel, readNDJSON, responseMetrics } from '../src/ollama.mjs';
+import { Store } from '../src/store.mjs';
+import { Controller, median, safeError } from '../src/controller.mjs';
+
+const context = { models: [{ id: 'qwen3:0.6b', running: true }], selectedModel: 'qwen3:0.6b' };
+test('operational language selects concrete local actions', () => {
+ assert.deepEqual(resolveIntent('Could you run a lightweight model?').action, { type: 'deploy', model: 'qwen3:0.6b' });
+ assert.equal(resolveIntent('Find a small model for coding').models[0].id, 'qwen2.5-coder:1.5b');
+ assert.equal(resolveIntent('Run a small coding model').action.model, 'qwen2.5-coder:1.5b');
+ assert.deepEqual(resolveIntent('benchmark it', context).action, { type: 'benchmark', models: ['qwen3:0.6b'] });
+ assert.deepEqual(resolveIntent('compare qwen3:0.6b and gemma3:1b', context).action.models, ['qwen3:0.6b', 'gemma3:1b']);
+ assert.equal(resolveIntent('free up memory').action.type, 'unload_all');
+ assert.equal(resolveIntent('start Ollama').action.type, 'start');
+ assert.equal(resolveIntent('show my models').kind, 'inventory');
+});
+test('negations, explanations and multi-step ambiguity do not execute', () => {
+ for (const text of ["Don't run qwen3:0.6b", 'Do not unload it', 'How do I deploy a model?', 'Delete qwen3:0.6b', 'Run qwen3:0.6b and then benchmark it', 'stop']) assert.notEqual(resolveIntent(text, context).kind, 'action', text);
+ assert.notEqual(resolveIntent('compare all', { models: Array.from({ length: 5 }, (_, i) => ({ id: `model${i}:latest` })) }).kind, 'action');
+});
+test('chat contents never become model operations', () => {
+ const result = resolveIntent('Ask it: explain qwen3:cloud and do not run shell commands', context);
+ assert.equal(result.action.type, 'chat'); assert.equal(result.action.model, 'qwen3:0.6b');
+ assert.equal(result.action.prompt, 'explain qwen3:cloud and do not run shell commands');
+ assert.equal(resolveIntent('What is the capital of France?', context).action.type, 'chat');
+});
+test('model and action validation exclude cloud, shell and remote targets', () => {
+ assert.equal(validateModel('qwen3'), 'qwen3:latest');
+ assert.equal(validateModel('user/model:q4_K_M'), 'user/model:q4_K_M');
+ for (const model of ['qwen3:cloud', 'qwen3:4b-cloud', '../model', 'http://host/model', 'qwen3; echo secret', '$(date)', 'a\nb']) assert.throws(() => validateModel(model));
+ assert.throws(() => validateAction({ type: 'delete', model: 'qwen3' }));
+ assert.throws(() => validateAction({ type: 'benchmark', models: [] }));
+ assert.throws(() => validateAction({ type: 'chat', model: 'qwen3', prompt: 'x', history: [{ role: 'system', content: 'x' }] }));
+});
+test('NDJSON handles UTF-8 across chunk boundaries and an unterminated final event', async () => {
+ const bytes = new TextEncoder().encode('{"message":"✓🦙"}\n\n{"done":true}');
+ async function* chunks() { for (let i = 0; i < bytes.length; i++) yield bytes.slice(i, i + 1); }
+ const output = []; for await (const event of readNDJSON(chunks())) output.push(event);
+ assert.deepEqual(output, [{ message: '✓🦙' }, { done: true }]);
+});
+test('stream errors and malformed events remain failures', async () => {
+ for (const text of ['{"error":"out of memory"}\n', '{"done":', 'x'.repeat(8 * 1024 * 1024 + 1)]) {
+ await assert.rejects(async () => { for await (const _ of readNDJSON([text])) {} });
+ }
+});
+test('medians omit missing measurements and errors omit signed URL queries', () => {
+ assert.equal(median([300, 100, 200]), 200); assert.equal(median([null, NaN]), null); assert.equal(median([10, 20]), 15);
+ assert.equal(safeError(new Error('Get https://example.org/file?signature=secret failed')), 'Get https://example.org/file failed');
+});
+test('native nanosecond timings become rates and milliseconds without inventing missing data', () => {
+ const metrics = responseMetrics({ eval_count: 100, eval_duration: 2e9, prompt_eval_count: 50, prompt_eval_duration: 5e8, load_duration: 12e6 });
+ assert.equal(metrics.tokensPerSecond, 50); assert.equal(metrics.promptTokensPerSecond, 100); assert.equal(metrics.loadMs, 12); assert.equal(metrics.evalMs, 2000);
+ assert.equal(responseMetrics({ eval_count: 100, eval_duration: 0 }).tokensPerSecond, null);
+ assert.equal(responseMetrics({}).tokensPerSecond, null); assert.equal(responseMetrics({}).loadMs, null);
+});
+async function fixture(t, overrides = {}) {
+ const directory = await mkdtemp(join(tmpdir(), 'harness-test-')); t.after(() => rm(directory, { recursive: true, force: true }));
+ const store = await new Store(directory).init();
+ const model = { id: 'qwen3:0.6b', name: 'qwen3:0.6b', digest: 'sha256:fixture', running: false, resident: { size: 100 } };
+ const runtime = { name: 'Ollama', online: true, installed: true, version: 'fixture', models: [model] };
+ const calls = [];
+ const adapter = { discover: async () => runtime, ensureRunning: async () => {}, localModelDetails: async () => ({}), loadModel: async () => { model.running = true; }, unloadModel: async () => { model.running = false; }, pullModel: async () => {}, generate: async (_m, _messages, options) => { calls.push(options); options.onToken?.('fixture response'); return { text: 'fixture response', metrics: { model: model.id, tokensPerSecond: calls.length * 10, firstTokenMs: 30, firstOutputMs: 30, tokens: 128 } }; }, ...overrides };
+ const controller = new Controller(store, adapter);
+ return { store, controller, model, calls };
+}
+const finished = (controller, id) => new Promise(resolve => {
+ const check = job => { if (job.id === id && !['queued', 'running'].includes(job.status)) { controller.off('job', check); resolve(job); } };
+ controller.on('job', check); const job = controller.store.data.jobs.find(j => j.id === id); if (job) check(job);
+});
+test('benchmark discards warmup, stores measured samples and real median', { timeout: 3000 }, async t => {
+ const { controller, store, calls } = await fixture(t);
+ const job = await controller.submit({ type: 'benchmark', models: ['qwen3:0.6b'] });
+ assert.equal((await finished(controller, job.id)).status, 'succeeded');
+ assert.equal(calls.length, 4); assert.equal(calls[0].maxTokens, 32); assert.equal(calls[1].maxTokens, 128);
+ const benchmark = store.data.benchmarks[0]; assert.equal(benchmark.samples.length, 3); assert.equal(benchmark.medianTokensPerSecond, 30); assert.equal(benchmark.digest, 'sha256:fixture');
+ assert.match(benchmark.mode, /warm/); assert.equal(JSON.parse(await readFile(store.file, 'utf8')).benchmarks.length, 1);
+});
+test('failed loading cannot report deployment success', { timeout: 3000 }, async t => {
+ const { controller } = await fixture(t, { loadModel: async () => { throw new Error('not enough memory'); } });
+ const job = await controller.submit({ type: 'deploy', model: 'qwen3:0.6b' }); const done = await finished(controller, job.id);
+ assert.equal(done.status, 'failed'); assert.equal(done.result, null); assert.match(done.message, /memory/);
+});
+test('queued jobs can be cancelled and do not reach the model', { timeout: 3000 }, async t => {
+ let release; const gate = new Promise(resolve => { release = resolve; });
+ const { controller, calls } = await fixture(t, { ensureRunning: () => gate });
+ const first = await controller.submit({ type: 'start' });
+ const queued = await controller.submit({ type: 'chat', model: 'qwen3:0.6b', prompt: 'hello' });
+ await controller.cancel(queued.id); release(); await finished(controller, first.id);
+ assert.equal(queued.status, 'cancelled'); assert.equal(calls.length, 0);
+});
+test('restarting preserves measurements and marks unfinished jobs interrupted', async t => {
+ const { store } = await fixture(t); store.data.jobs.push({ id: 'unfinished', status: 'running' }); store.data.benchmarks.push({ id: 'measured' }); await store.save();
+ const restored = await new Store(store.directory).init(); assert.equal(restored.data.jobs[0].status, 'interrupted'); assert.equal(restored.data.benchmarks[0].id, 'measured');
+});
diff --git a/store/tools/local-ai/test/http.test.mjs b/store/tools/local-ai/test/http.test.mjs
new file mode 100644
index 000000000..3923b976e
--- /dev/null
+++ b/store/tools/local-ai/test/http.test.mjs
@@ -0,0 +1,43 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import http from 'node:http';
+import { EventEmitter } from 'node:events';
+import { createServer } from '../src/server.mjs';
+import { createViewerRelay } from '../src/viewer-relay.mjs';
+
+test('loopback API enforces origin, host, token and action validation boundaries', async t => {
+ const controller = new EventEmitter(); let mutations = 0;
+ controller.snapshot = () => ({ runtime: { models: [] }, jobs: [] }); controller.refresh = async () => {};
+ controller.command = async text => { mutations++; return { kind: 'reply', message: text }; };
+ const server = createServer(controller);
+ await new Promise(resolve => server.listen(0, '127.0.0.1', resolve));
+ t.after(() => new Promise(resolve => { server.closeEvents(); server.closeAllConnections(); server.close(resolve); }));
+ const port = server.address().port; const base = `http://127.0.0.1:${port}`;
+ const { token } = await (await fetch(`${base}/api/state`)).json(); assert.equal(token.length, 64);
+ const post = headers => fetch(`${base}/api/command`, { method: 'POST', headers: { 'Content-Type': 'application/json', ...headers }, body: '{"text":"show my models"}' });
+ assert.equal((await post({})).status, 403);
+ assert.equal((await post({ 'X-Harness-Token': token, Origin: 'https://untrusted.example' })).status, 403);
+ assert.equal((await post({ 'X-Harness-Token': token, 'Sec-Fetch-Site': 'cross-site' })).status, 403);
+ assert.equal((await post({ 'X-Harness-Token': token, 'Content-Type': 'text/plain' })).status, 415);
+ const hostileHost = await new Promise(resolve => { const req = http.get({ host: '127.0.0.1', port, path: '/api/state', headers: { Host: 'attacker.example' } }, res => { res.resume(); resolve(res.statusCode); }); req.on('error', e => { throw e; }); });
+ assert.equal(hostileHost, 403); assert.equal(mutations, 0);
+ assert.equal((await post({ 'X-Harness-Token': token, Origin: base })).status, 200); assert.equal(mutations, 1);
+ assert.equal((await fetch(`${base}/api/command`, { method: 'POST', headers: { 'Content-Type': 'application/json', 'X-Harness-Token': token }, body: '{broken' })).status, 400);
+ assert.equal((await fetch(`${base}/api/command`, { method: 'POST', headers: { 'Content-Type': 'application/json', 'X-Harness-Token': token }, body: JSON.stringify({ text: 'a'.repeat(70000) }) })).status, 413);
+ assert.equal((await fetch(`${base}/src/server.mjs`)).status, 404);
+});
+
+test('native viewer relay shares the controller and preserves mutation boundaries', async t => {
+ const controller = new EventEmitter(); let count = 0;
+ controller.snapshot = () => ({ runtime: { models: [{ id: 'fixture:latest' }] }, jobs: [] }); controller.refresh = async () => {};
+ controller.command = async text => { count++; return { message: text }; };
+ const primary = createServer(controller); await new Promise(resolve => primary.listen(0, '127.0.0.1', resolve));
+ const relay = createViewerRelay(primary.address().port); await new Promise(resolve => relay.listen(0, '127.0.0.1', resolve));
+ t.after(async () => { relay.closeAllConnections(); await new Promise(resolve => relay.close(resolve)); primary.closeEvents(); primary.closeAllConnections(); await new Promise(resolve => primary.close(resolve)); });
+ const base = `http://127.0.0.1:${relay.address().port}`;
+ const inventory = await (await fetch(`${base}/api/state`)).json(); assert.equal(inventory.runtime.models[0].id, 'fixture:latest');
+ const post = extra => fetch(`${base}/api/command`, { method: 'POST', headers: { 'Content-Type': 'application/json', 'X-Harness-Token': inventory.token, ...extra }, body: '{"text":"show models"}' });
+ assert.equal((await post({ Origin: 'https://untrusted.example' })).status, 403); assert.equal(count, 0);
+ assert.equal((await post({ 'X-Harness-Token': '' })).status, 403);
+ assert.equal((await post({ Origin: base })).status, 200); assert.equal(count, 1);
+});
diff --git a/store/tools/local-ai/test/live.mjs b/store/tools/local-ai/test/live.mjs
new file mode 100644
index 000000000..5832971d9
--- /dev/null
+++ b/store/tools/local-ai/test/live.mjs
@@ -0,0 +1,36 @@
+// Opt-in: uses the running local harness and a real model. Changes model residency.
+import assert from 'node:assert/strict';
+import { readFile, writeFile, mkdir } from 'node:fs/promises';
+import { resolve, join } from 'node:path';
+const workspace = resolve(process.env.HARNESS_WORKSPACE || '../../agents/ollama');
+const endpoint = JSON.parse(await readFile(join(workspace, '.harness/endpoint.json'), 'utf8'));
+const base = `http://127.0.0.1:${endpoint.port}`;
+const model = process.env.HARNESS_TEST_MODEL || 'qwen3:0.6b';
+const state = async () => (await fetch(`${base}/api/state`)).json();
+const initial = await state();
+const action = async body => {
+ const response = await fetch(`${base}/api/action`, { method: 'POST', headers: { 'Content-Type': 'application/json', 'X-Harness-Token': initial.token }, body: JSON.stringify(body) });
+ const result = await response.json(); assert.ok(response.ok, result.error);
+ const deadline = Date.now() + 15 * 60 * 1000;
+ while (Date.now() < deadline) {
+ const { jobs } = await (await fetch(`${base}/api/jobs`)).json(); const job = jobs.find(j => j.id === result.job.id);
+ if (!['queued', 'running'].includes(job.status)) { assert.equal(job.status, 'succeeded', job.message); console.log(`${body.type}: ${job.message}`); return job; }
+ await new Promise(resolve => setTimeout(resolve, 500));
+ }
+ throw new Error(`Timed out waiting for job ${result.job.id}; inspect it before retrying.`);
+};
+await action({ type: 'unload', model });
+assert.ok(!(await state()).runtime.models.find(m => m.id === model)?.running, 'Model is no longer resident');
+await action({ type: 'deploy', model });
+assert.equal((await state()).runtime.models.find(m => m.id === model)?.running, true);
+const chat = await action({ type: 'chat', model, prompt: 'What is 2 + 2? Reply with just the number.' });
+assert.match(chat.output.trim(), /^(?:2\s*\+\s*2\s*=\s*)?4[.!]?$/);
+assert.ok(chat.result.metrics.tokensPerSecond > 0);
+let snapshot = await state();
+if (!snapshot.benchmarks.some(b => b.model === model)) { await action({ type: 'benchmark', models: [model] }); snapshot = await state(); }
+const benchmark = snapshot.benchmarks.find(b => b.model === model);
+assert.equal(benchmark.samples.length, 3); assert.ok(benchmark.samples.every(s => s.tokensPerSecond > 0 && s.firstTokenMs >= 0));
+const evidence = { checkedAt: new Date().toISOString(), runtime: snapshot.runtime.version, hardware: snapshot.system, model: snapshot.runtime.models.find(m => m.id === model), checks: ['unload verified in runtime inventory', 'cached deployment verified resident', 'chat arithmetic sanity check', 'generation metrics present', 'three-run measured benchmark persisted'], chat: { prompt: 'What is 2 + 2? Reply with just the number.', output: chat.output, metrics: chat.result.metrics }, benchmark };
+await mkdir('evidence', { recursive: true });
+await writeFile('evidence/pilot-verification.json', JSON.stringify(evidence, null, 2) + '\n');
+console.log(`Verified ${model}; evidence saved in evidence/pilot-verification.json`);
diff --git a/store/tools/local-ai/test/mlx-live.mjs b/store/tools/local-ai/test/mlx-live.mjs
new file mode 100644
index 000000000..724500d83
--- /dev/null
+++ b/store/tools/local-ai/test/mlx-live.mjs
@@ -0,0 +1,60 @@
+// Opt-in integration check. Uses the MLX viewer, existing HF cache and this Mac's GPU.
+import assert from 'node:assert/strict';
+import { readFile, writeFile, mkdir } from 'node:fs/promises';
+import { setTimeout as delay } from 'node:timers/promises';
+const packageRoot = new URL('../../../agents/mlx-lm/', import.meta.url);
+const endpoint = JSON.parse(await readFile(new URL('.harness/endpoint.json', packageRoot), 'utf8'));
+const base = `http://127.0.0.1:${endpoint.port}`;
+const get = async path => { const res = await fetch(base + path); assert.equal(res.status, 200); return res.json(); };
+const health = await get('/api/health'); assert.equal(health.runtime, 'mlx-lm');
+let state = await get('/api/state'); const token = state.token;
+const post = async (path, body) => { const res = await fetch(base + path, { method: 'POST', headers: { 'Content-Type': 'application/json', 'X-Harness-Token': token }, body: JSON.stringify(body) }); const value = await res.json(); assert.ok(res.ok, JSON.stringify(value)); return value; };
+const wait = async (id, predicate = job => !['queued', 'running'].includes(job.status), timeout = 180000) => {
+ const deadline = Date.now() + timeout;
+ while (Date.now() < deadline) {
+ const job = (await get('/api/jobs')).jobs.find(job => job.id === id);
+ if (predicate(job)) return job;
+ await delay(25);
+ }
+ throw new Error(`Timed out waiting for ${id}`);
+};
+const jobs = [];
+const action = async body => {
+ const { job } = await post('/api/action', body);
+ const result = await wait(job.id); jobs.push(result);
+ assert.equal(result.status, 'succeeded', result.message);
+ console.log(`${result.type}: ${result.message}`);
+ return result;
+};
+const model = 'mlx-community/Qwen3-0.6B-4bit';
+const baseline = 'mlx-community/Qwen2.5-0.5B-Instruct-4bit';
+for (const path of ['/?view=grid', '/app.js', '/styles.css', '/assets/mlx-icon.svg', '/assets/mlx-icon.png', '/favicon.ico']) assert.equal((await fetch(base + path)).status, 200, path);
+await action({ type: 'deploy', model });
+const chat = await action({ type: 'chat', model, prompt: 'Explain Apple Silicon unified memory in one short sentence.' });
+assert.ok(chat.output.trim(), 'The model must produce visible text.');
+assert.ok(chat.result.metrics.tokensPerSecond > 0);
+assert.ok(chat.result.metrics.peakMemoryBytes > 0);
+await action({ type: 'benchmark', models: [model, baseline] });
+state = await get('/api/state');
+assert.deepEqual(state.runtime.models.filter(m => m.running).map(m => m.id), [baseline], 'Switching must release the previous model.');
+for (const id of [model, baseline]) {
+ const result = state.benchmarks.find(b => b.model === id);
+ assert.equal(result.samples.length, 3); assert.match(result.mode, /fresh prompt cache/);
+ assert.equal(result.runtime.name, 'MLX-LM'); assert.equal(result.samples[0].thinking, false);
+}
+await action({ type: 'unload_all' });
+assert.equal((await get('/api/state')).runtime.models.filter(m => m.running).length, 0);
+await action({ type: 'deploy', model });
+const { job: cancellable } = await post('/api/action', { type: 'chat', model, prompt: 'Write out the integers from 1 to 1000, each on a separate line. Do not stop early.' });
+await wait(cancellable.id, job => job.status === 'running' && job.message.startsWith('Generating'), 30000);
+await post('/api/cancel', { id: cancellable.id });
+const cancelled = await wait(cancellable.id); jobs.push(cancelled); assert.equal(cancelled.status, 'cancelled');
+assert.equal((await get('/api/state')).runtime.models.filter(m => m.running).length, 0);
+await action({ type: 'deploy', model });
+state = await get('/api/state');
+assert.deepEqual(state.runtime.models.filter(m => m.running).map(m => m.id), [model]);
+const output = new URL('../evidence/mlx-lm/', import.meta.url);
+await mkdir(output, { recursive: true });
+await writeFile(new URL('verification.json', output), JSON.stringify({ at: new Date().toISOString(), health, runtime: { ...state.runtime, models: state.runtime.models.map(({ resident, ...model }) => model) }, system: state.system, checks: ['asset routes', 'deploy', 'real streamed chat', 'native metrics', '3-sample benchmarks', 'single-model switching', 'unload', 'active cancellation releases model', 'worker recovery and reload'], jobs }, null, 2));
+await writeFile(new URL('benchmarks.json', output), JSON.stringify({ at: new Date().toISOString(), benchmarks: state.benchmarks }, null, 2));
+console.log('MLX live verification passed. Raw evidence saved under research/mlx-lm/.');
diff --git a/store/tools/local-ai/test/mlx.test.mjs b/store/tools/local-ai/test/mlx.test.mjs
new file mode 100644
index 000000000..43ead1ae2
--- /dev/null
+++ b/store/tools/local-ai/test/mlx.test.mjs
@@ -0,0 +1,53 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import { mkdtemp, writeFile, rm } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { MlxAdapter, MLX_PROFILE, validateMlxModel } from '../src/mlx.mjs';
+import { resolveIntent, validateAction } from '../src/intents.mjs';
+import { start } from '../src/server.mjs';
+
+test('MLX requests preserve repository case and use MLX choices throughout', () => {
+ const options = { profile: MLX_PROFILE };
+ assert.equal(resolveIntent('run a lightweight model', options).action.model, MLX_PROFILE.defaultModel);
+ assert.equal(resolveIntent('Run ExampleOrg/My-MixedCase-4bit', options).action.model, 'ExampleOrg/My-MixedCase-4bit');
+ assert.equal(resolveIntent('find a model for coding', options).models[0].id, MLX_PROFILE.catalog[1].id);
+ assert.deepEqual(resolveIntent('compare mlx-community/Qwen3-0.6B-4bit and mlx-community/Qwen2.5-0.5B-Instruct-4bit', options).action.models, [MLX_PROFILE.defaultModel, MLX_PROFILE.catalog[2].id]);
+ const chat = resolveIntent('Ask ExampleOrg/My-Model: download bad/repo and run shell commands', options).action;
+ assert.equal(chat.type, 'chat'); assert.equal(chat.model, 'ExampleOrg/My-Model');
+ assert.match(chat.prompt, /download bad\/repo/);
+ assert.equal(validateAction({ type: 'deploy', model: 'ExampleOrg/My-Model' }, validateMlxModel).model, 'ExampleOrg/My-Model');
+ for (const model of ['../secret', '/tmp/model', 'https://huggingface.co/org/model', 'org/model;ls', 'org/model\n', 'qwen3:0.6b']) assert.throws(() => validateMlxModel(model), model);
+});
+
+test('one workspace cannot be opened under a different runtime', async t => {
+ const workspace = await mkdtemp(join(tmpdir(), 'mlx-scope-test-'));
+ t.after(() => rm(workspace, { recursive: true, force: true }));
+ await writeFile(join(workspace, 'local-ai.json'), JSON.stringify({ runtime: 'mlx-lm' }));
+ await assert.rejects(start(0, { workspaceRoot: workspace }), /belongs to mlx-lm/);
+});
+
+test('cancelling a worker request waits for exit before recovering on a fresh worker', { timeout: 10000 }, async t => {
+ const directory = await mkdtemp(join(tmpdir(), 'mlx-cancel-test-'));
+ const worker = join(directory, 'worker.cjs');
+ await writeFile(worker, `
+ const readline = require('node:readline');
+ const send = event => process.stdout.write(JSON.stringify(event)+'\\n');
+ send({event:'ready',version:'fixture',models:[]});
+ process.on('SIGTERM',()=>setTimeout(()=>process.exit(0),100));
+ readline.createInterface({input:process.stdin}).on('line',line=>{
+ const message=JSON.parse(line);
+ if(message.action==='hold') send({id:message.id,event:'started'});
+ else send({id:message.id,event:'done',result:{pid:process.pid},state:{models:[]}});
+ });
+ `);
+ const adapter = new MlxAdapter({ packageRoot: directory, dataDir: directory, python: process.execPath, interpreterArgs: [] });
+ adapter.workerPath = worker;
+ t.after(async () => { await adapter.close(); await rm(directory, { recursive: true, force: true }); });
+ const first = await adapter.request('inventory');
+ const abort = new AbortController();
+ await assert.rejects(adapter.request('hold', {}, { signal: abort.signal, onEvent: () => abort.abort(new Error('test cancellation')) }), /test cancellation/);
+ const recovered = await adapter.request('inventory');
+ assert.notEqual(first.pid, recovered.pid);
+ assert.equal((await adapter.discover()).online, true);
+});
diff --git a/store/tools/local-ai/test/packaging.test.mjs b/store/tools/local-ai/test/packaging.test.mjs
new file mode 100644
index 000000000..b4df1aeae
--- /dev/null
+++ b/store/tools/local-ai/test/packaging.test.mjs
@@ -0,0 +1,53 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import { cp, mkdtemp, readFile, rm } from 'node:fs/promises';
+import { join } from 'node:path';
+import { tmpdir } from 'node:os';
+import { pathToFileURL } from 'node:url';
+import { EventEmitter } from 'node:events';
+
+// A Store install contains just one directory. Verify imports and real static
+// assets from a relocated copy, without shared-source or sibling dependencies.
+for (const [name, icon] of [['ollama', 'ollama'], ['mlx-lm', 'mlx'], ['vllm', 'vllm']]) {
+ test(`${name} serves its branded viewer from an independent package copy`, async t => {
+ const temp = await mkdtemp(join(tmpdir(), 'local-ai-package-'));
+ t.after(() => rm(temp, { recursive: true, force: true }));
+ const directory = join(temp, name);
+ await cp(new URL(`../../../agents/${name}/`, import.meta.url), directory, {
+ recursive: true,
+ filter: path => !path.split('/').some(part => ['.venv', '.harness', '__pycache__'].includes(part)),
+ });
+ const manifest = JSON.parse(await readFile(join(directory, 'harness.json'), 'utf8'));
+ assert.equal(manifest.id, `autonomous/${name}`);
+ assert.equal(manifest.category, 'Local AI');
+ for (const file of [manifest.agent.instructions, manifest.toolchain.setup,
+ manifest.toolchain.doctor, 'toolchain/local-ai', 'viewer.sh']) {
+ assert.ok((await readFile(join(directory, file))).length, file);
+ }
+ const { createServer } = await import(pathToFileURL(join(directory, 'src/server.mjs')));
+ if (name !== 'ollama') await import(pathToFileURL(join(directory, `src/${name === 'mlx-lm' ? 'mlx' : 'vllm'}.mjs`)));
+ const controller = new EventEmitter();
+ controller.profile = { id: name };
+ controller.snapshot = () => ({ runtime: { models: [] }, jobs: [] });
+ controller.refresh = async () => {};
+ const server = createServer(controller, { assetRoot: join(directory, 'dist') });
+ await new Promise(resolve => server.listen(0, '127.0.0.1', resolve));
+ t.after(async () => {
+ server.closeEvents(); server.closeAllConnections();
+ await new Promise(resolve => server.close(resolve));
+ });
+ const base = `http://127.0.0.1:${server.address().port}`;
+ const html = await (await fetch(`${base}/?view=grid`)).text();
+ assert.ok(html.includes(manifest.name));
+ assert.ok(html.includes(`${icon}-icon.`));
+ for (const path of ['/app.js', '/styles.css', `/assets/${icon}-icon.png`, '/favicon.ico']) {
+ const response = await fetch(base + path);
+ assert.equal(response.status, 200, path);
+ const bytes = Buffer.from(await response.arrayBuffer());
+ assert.ok(bytes.length > 0);
+ if (path.endsWith('.png') || path.endsWith('.ico')) {
+ assert.equal(bytes.subarray(1, 4).toString(), 'PNG');
+ }
+ }
+ });
+}
diff --git a/store/tools/local-ai/test/vllm-live.mjs b/store/tools/local-ai/test/vllm-live.mjs
new file mode 100644
index 000000000..6f65ddfe7
--- /dev/null
+++ b/store/tools/local-ai/test/vllm-live.mjs
@@ -0,0 +1,54 @@
+// Real cached-model serving, concurrency, route boundaries, cancellation and recovery.
+import assert from 'node:assert/strict';
+import http from 'node:http';
+import { resolve, join } from 'node:path';
+import { writeFile, mkdir } from 'node:fs/promises';
+import { setTimeout as delay } from 'node:timers/promises';
+import { start } from '../src/server.mjs';
+import { VllmAdapter, VLLM_PROFILE } from '../src/vllm.mjs';
+const packageRoot = resolve('../../agents/vllm');
+const adapter = new VllmAdapter({ packageRoot, dataDir: join(packageRoot, '.harness') });
+const { server, controller } = await start(0, { workspaceRoot: packageRoot, assetRoot: join(packageRoot, 'dist'), adapter, profile: VLLM_PROFILE });
+const model = VLLM_PROFILE.defaultModel; const checks = [];
+async function run(action) {
+ const job = await controller.submit(action); console.log('Running', action.type);
+ const deadline = Date.now() + 240000;
+ while (['queued', 'running'].includes(job.status) && Date.now() < deadline) await delay(150);
+ assert.equal(job.status, 'succeeded', job.message); checks.push(action.type); console.log(job.message); return job;
+}
+try {
+ await run({ type: 'serve', model, context: 2048, maxSequences: 4, memoryFraction: 0.1, prefixCaching: true });
+ const privateBase = `http://127.0.0.1:${adapter.server.port}`;
+ assert.equal((await fetch(privateBase + '/health')).status, 403);
+ assert.equal((await fetch(privateBase + '/metrics', { headers: { Authorization: `Bearer ${adapter.server.key}`, Origin: 'https://evil.example' } })).status, 403);
+ // fetch normalizes Host; use a raw HTTP request to exercise the actual boundary.
+ const hostileHost = await new Promise((resolve, reject) => { const request = http.get({ host: '127.0.0.1', port: adapter.server.port, path: '/health', headers: { Authorization: `Bearer ${adapter.server.key}`, Host: 'evil.example' } }, response => { response.resume(); resolve(response.statusCode); }); request.on('error', reject); });
+ assert.equal(hostileHost, 403);
+ assert.equal((await adapter.fetchNative('/health')).status, 200); checks.push('private API auth, host and origin');
+ const base = `http://127.0.0.1:${server.address().port}`;
+ for (const path of ['/?view=grid', '/app.js', '/styles.css', '/assets/vllm-icon.png', '/favicon.ico']) assert.equal((await fetch(base + path)).status, 200, path);
+ checks.push('viewer and official icon HTTP assets');
+ const chat = await run({ type: 'chat', model, prompt: 'In one sentence, why does a bicycle need brakes?' }); assert.ok(chat.output.trim()); assert.ok(chat.result.metrics.tokens > 0);
+ await run({ type: 'benchmark', models: [model] });
+ await run({ type: 'load_test', model, concurrency: 1, requests: 8 });
+ await run({ type: 'load_test', model, concurrency: 4, requests: 8 });
+ const tests = controller.snapshot().loadTests; assert.ok(tests.length >= 2); assert.equal(tests[0].failed, 0); assert.equal(tests[1].failed, 0);
+ await run({ type: 'serve', model, context: 1024, maxSequences: 2, prefixCaching: false });
+ await run({ type: 'chat', model, prompt: 'Say hello briefly.' });
+ assert.equal(adapter.config.context, 1024); assert.equal(adapter.config.maxSequences, 2); assert.equal(adapter.config.prefixCaching, false); checks.push('serving settings apply and chat preserves them');
+ const job = await controller.submit({ type: 'load_test', model, concurrency: 4, requests: 32 });
+ while (!adapter.inflight && ['queued', 'running'].includes(job.status)) await delay(10);
+ assert.ok(adapter.inflight); const ownedPid = adapter.server.child.pid;
+ await controller.cancel(job.id);
+ while (['queued', 'running'].includes(job.status)) await delay(50);
+ assert.equal(job.status, 'cancelled'); await adapter.stopServer();
+ assert.throws(() => process.kill(ownedPid, 0), { code: 'ESRCH' }); checks.push('active cancellation frees owned server');
+ await run({ type: 'serve', model, context: 2048, maxSequences: 4, prefixCaching: true });
+ await run({ type: 'chat', model, prompt: 'Say ready in a short sentence.' }); checks.push('recovery after cancellation');
+ await run({ type: 'unload_all' }); assert.equal(adapter.loaded, null);
+ const exported = await (await fetch(base + '/api/export')).json(); assert.ok(exported.loadTests.length >= 2);
+ await mkdir('evidence/vllm', { recursive: true });
+ await writeFile('evidence/vllm/benchmarks.json', JSON.stringify(exported, null, 2));
+ await writeFile('evidence/vllm/verification.json', JSON.stringify({ at: new Date().toISOString(), checks, runtime: (await adapter.discover()), chat: { output: chat.output, metrics: chat.result.metrics } }, null, 2));
+ console.log('All live vLLM checks passed.');
+} finally { server.closeEvents(); server.closeAllConnections(); server.close(); await controller.close(); }
diff --git a/store/tools/local-ai/test/vllm.test.mjs b/store/tools/local-ai/test/vllm.test.mjs
new file mode 100644
index 000000000..8814bc52f
--- /dev/null
+++ b/store/tools/local-ai/test/vllm.test.mjs
@@ -0,0 +1,66 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import { mkdtemp, rm } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { VLLM_PROFILE, validateVllmAction, validateServing, readCompletion, parsePrometheus } from '../src/vllm.mjs';
+import { Controller } from '../src/controller.mjs';
+import { Store } from '../src/store.mjs';
+
+test('vLLM serving controls are bounded and explicit; prompts cannot become operations', () => {
+ const model = VLLM_PROFILE.defaultModel;
+ assert.equal(validateVllmAction({ type: 'serve', model, memoryFraction: 0.1 }).memoryFraction, 0.1);
+ assert.deepEqual(validateVllmAction({ type: 'load_test', model }), { type: 'load_test', model, concurrency: 4, requests: 8 });
+ for (const input of [{ context: 1 }, { context: '4096' }, { maxSequences: 9 }, { memoryFraction: 0.9 }, { prefixCaching: 'false' }]) assert.throws(() => validateServing(input));
+ assert.throws(() => validateVllmAction({ type: 'serve', model, host: '0.0.0.0' }), /Unknown/);
+ assert.throws(() => validateVllmAction({ type: 'load_test', model, concurrency: 9 }));
+ assert.throws(() => validateVllmAction({ type: 'load_test', model, concurrency: 4, requests: 2 }));
+ assert.throws(() => validateVllmAction({ type: 'serve', model: '../../file' }));
+ const options = { profile: VLLM_PROFILE, models: [{ id: model, running: true }] };
+ assert.equal(VLLM_PROFILE.resolveIntent('test 4 concurrent requests', options).action.type, 'load_test');
+ assert.equal(VLLM_PROFILE.resolveIntent('ask it: test 4 concurrent requests', options).action.type, 'chat');
+ assert.equal(VLLM_PROFILE.resolveIntent('do not benchmark it', options).kind, 'reply');
+ assert.equal(VLLM_PROFILE.resolveIntent('explain how to benchmark it', options).kind, 'reply');
+});
+
+const stream = async function* (text, length = 1) { const bytes = Buffer.from(text); for (let offset = 0; offset < bytes.length; offset += length) yield bytes.subarray(offset, offset + length); };
+test('OpenAI streams count native tokens and decode UTF-8 across arbitrary boundaries', async () => {
+ const event = value => `data: ${JSON.stringify(value)}\r\n\r\n`;
+ const input = event({ choices: [{ delta: { reasoning_content: 'think' } }] }) + event({ choices: [{ delta: { content: '👋 héllo' } }] }) + event({ choices: [{ delta: {}, finish_reason: 'stop' }], usage: { completion_tokens: 6, prompt_tokens: 12 } }) + 'data: [DONE]\r\n\r\n';
+ let output = ''; let clock = 0;
+ const result = await readCompletion(stream(input), text => { output += text; }, () => clock += 10);
+ assert.equal(result.text, '👋 héllo'); assert.equal(output, result.text); assert.equal(result.usage.completion_tokens, 6);
+ assert.equal(result.firstTokenMs, 10); assert.equal(result.firstOutputMs, 20); assert.equal(result.finishReason, 'stop');
+ await assert.rejects(readCompletion(stream('data: [DONE]\n\n')), /missing token usage/);
+ await assert.rejects(readCompletion(stream(event({ usage: { completion_tokens: 4, prompt_tokens: 3 } }))), /incomplete stream/);
+ await assert.rejects(readCompletion(stream(event({ error: { message: 'failed' } }))), /failed/);
+});
+
+test('server gauges remain unknown when telemetry is missing', () => {
+ assert.equal(parsePrometheus('# no samples').cacheUsage, null);
+ assert.deepEqual(parsePrometheus('vllm:num_requests_running{model_name="x"} 2\nvllm:num_requests_waiting{model_name="x"} 1\nvllm:kv_cache_usage_perc{model_name="x"} 0.25').runningRequests, 2);
+});
+
+test('concurrent test actually overlaps requests and retains failed samples', async t => {
+ const directory = await mkdtemp(join(tmpdir(), 'vllm-concurrency-'));
+ t.after(() => rm(directory, { recursive: true, force: true }));
+ const store = await new Store(directory).init(); const model = VLLM_PROFILE.defaultModel;
+ let active = 0; let peak = 0; let calls = 0; let loaded = false;
+ const adapter = {
+ async discover() { return { online: true, version: 'fixture', models: [{ id: model, running: loaded }], serving: { context: 1024, maxSequences: 4 } }; },
+ async ensureRunning() {}, async localModelDetails() {}, async loadModel() { loaded = true; },
+ async generate() {
+ calls++; const index = calls; active++; peak = Math.max(peak, active);
+ try { await new Promise(resolve => setTimeout(resolve, 10)); if (index === 3) throw new Error('fixture failure'); return { metrics: { model, tokens: 20, totalMs: 10, firstTokenMs: 2, tokensPerSecond: 2000 } }; }
+ finally { active--; }
+ },
+ };
+ const controller = new Controller(store, adapter, VLLM_PROFILE);
+ const job = await controller.submit({ type: 'load_test', model, concurrency: 4, requests: 8 });
+ while (['queued', 'running'].includes(job.status)) await new Promise(resolve => setTimeout(resolve, 10));
+ assert.equal(job.status, 'succeeded'); assert.equal(peak, 4); assert.equal(calls, 9);
+ const result = store.data.loadTests[0]; assert.equal(result.samples.length, 8); assert.equal(result.failed, 1); assert.equal(result.succeeded, 7); assert.equal(result.settings.context, 1024);
+ assert.equal(result.samples[1].error, 'fixture failure'); assert.equal(result.aggregateTokensPerSecond, 140 / (result.totalMs / 1000));
+ assert.equal((await new Store(directory).init()).data.loadTests[0].id, result.id);
+ await controller.close();
+});
diff --git a/store/tools/local-ai/toolchain/node.sh b/store/tools/local-ai/toolchain/node.sh
new file mode 100644
index 000000000..df3e4ee05
--- /dev/null
+++ b/store/tools/local-ai/toolchain/node.sh
@@ -0,0 +1,15 @@
+# Use this Mac's Node or the interpreter managed by Harness, including GUI launches.
+local_ai_node_candidate="$(command -v node 2>/dev/null || true)"
+if [ -n "$local_ai_node_candidate" ] && "$local_ai_node_candidate" -e 'process.exit(Number(process.versions.node.split(".")[0]) >= 22 ? 0 : 1)' >/dev/null 2>&1; then
+ LOCAL_AI_NODE="$local_ai_node_candidate"
+else
+ local_ai_node_record="${ADAPTER_RUNTIME_DIR:-${HOME}/.harness/runtime}/current-node"
+ local_ai_node_candidate="$(cat "$local_ai_node_record" 2>/dev/null || true)"
+ if [ -n "$local_ai_node_candidate" ] && [ -x "${local_ai_node_candidate%/*}/node" ] && "${local_ai_node_candidate%/*}/node" -e 'process.exit(Number(process.versions.node.split(".")[0]) >= 22 ? 0 : 1)' >/dev/null 2>&1; then
+ LOCAL_AI_NODE="${local_ai_node_candidate%/*}/node"
+ else
+ echo 'miss Node.js 22 or newer — update Harness or install Node.js 22' >&2
+ return 1
+ fi
+fi
+export LOCAL_AI_NODE