Skip to content

Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors

Rust

Revert jev-reranker-adapter docs (#873) for the 2.4.1 patch line

AkitaOnRailse45db18a0eb5
diff --git a/docs/examples/jev-reranker-adapter/jev_rerank_shim.py b/docs/examples/jev-reranker-adapter/jev_rerank_shim.py
deleted file mode 100644
index 98acb5a2..00000000
--- a/docs/examples/jev-reranker-adapter/jev_rerank_shim.py
+++ /dev/null
@@ -1,197 +0,0 @@
-#!/usr/bin/env python3
-"""OpenAI-compat -> Jev `/v1/systemone` reranker adapter for ai-memory.
-
-ai-memory's LLM reranker (`AI_MEMORY_RERANKER=llm`) sends one structured chat
-request per `memory_query`: a fixed system prompt plus a user JSON payload
-`{"query", "candidates":[{"candidate","title","text"}]}`, and expects the
-first balanced JSON object in the reply content to be
-`{"scores":[{"candidate","relevance"}]}` (1-based indices, relevance in
-[0,1]; a timeout, error, or invalid score set preserves the server's own
-order).
-
-This adapter recognises exactly that request (by the system-prompt prefix),
-scores every candidate with one batched Jev `score` question whose rubric
-mirrors the reranker prompt's own 1.0 / 0.7 / 0.3 / 0.0 guidance, and returns
-the judgement as plain chat content. Everything else — consolidation, lint,
-bootstrap — is reverse-proxied unchanged to the configured upstream, so only
-reranking rides the Jev endpoint.
-
-Why: a hosted reranker model that answers in tens of seconds dwarfs the rest
-of a `memory_query`. A judge endpoint that scores a fixed rubric answers in
-well under a second, which keeps `AI_MEMORY_RERANKER=llm` usable in
-interactive sessions. On a 102-query golden set (see
-docs/jev-reranker-adapter.md) this adapter matched the hosted reranker's
-hit@1 / MRR / NDCG@10 while cutting mean rerank latency from ~20s to ~0.2s.
-
-Env:
- JEV_URL Jev systemone endpoint (default http://127.0.0.1:18095/v1/systemone)
- JEV_MODEL model name sent to Jev (default jev-latest)
- UPSTREAM OpenAI-compat upstream base (default http://127.0.0.1:8000)
- LISTEN host:port to bind (default 127.0.0.1:18097)
-
-Stdlib only. No secrets are stored: Authorization headers are forwarded
-verbatim to the upstream.
-"""
-import json
-import os
-import sys
-import time
-import urllib.error
-import urllib.request
-from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
-
-JEV_URL = os.environ.get("JEV_URL", "http://127.0.0.1:18095/v1/systemone").rstrip("/")
-JEV_MODEL = os.environ.get("JEV_MODEL", "jev-latest")
-UPSTREAM = os.environ.get("UPSTREAM", "http://127.0.0.1:8000").rstrip("/")
-LISTEN = os.environ.get("LISTEN", "127.0.0.1:18097")
-
-# Fixed prefix of the reranker's system prompt; see
-# crates/ai-memory-llm/src/reranker.rs in the ai-memory tree.
-RERANK_SYSTEM_PREFIX = "You are a retrieval reranker for a software project's memory wiki."
-
-# Mirrors the four grades of the reranker prompt (1.0 / 0.7 / 0.3 / 0.0).
-RUBRIC = [
- "0.0 unrelated — does not address the query at all",
- "0.3 tangential — loosely related to the query",
- "0.7 same topic — same subsystem or topic, useful supporting context",
- "1.0 direct answer — directly answers the query",
-]
-
-
-def log(msg):
- sys.stderr.write(f"[jev-rerank] {time.strftime('%H:%M:%S')} {msg}\n")
- sys.stderr.flush()
-
-
-def jev_rerank(payload):
- """Translate a reranker chat request into one batched Jev score call."""
- user = next((m.get("content", "") for m in payload.get("messages", [])
- if m.get("role") == "user"), "")
- data = json.loads(user)
- query, cands = data["query"], data["candidates"]
- lines = [f"User query: {query}", "", "Candidate documents:"]
- questions = {}
- for c in cands:
- n = c["candidate"]
- lines.append(f"{n}. {c['title']} — {c['text']}")
- questions[f"c{n}"] = {
- "type": "score",
- "instructions": (
- f"Apply the relevance rubric to candidate {n}: how well "
- "does it answer the user query?"
- ),
- "criteria": RUBRIC,
- }
- body = {"model": JEV_MODEL, "state": "\n".join(lines), "questions": questions}
- t0 = time.perf_counter()
- req = urllib.request.Request(JEV_URL, data=json.dumps(body).encode(),
- headers={"Content-Type": "application/json"})
- with urllib.request.urlopen(req, timeout=25) as resp:
- out = json.loads(resp.read())
- dt = time.perf_counter() - t0
- scores = []
- for c in cands:
- ans = (out.get("answers") or {}).get(f"c{c['candidate']}") or {}
- raw = ans.get("score")
- # criteria index 0..3 -> relevance 0.0 / 0.3 / 0.7 / 1.0
- relevance = max(0.0, min(1.0, raw / (len(RUBRIC) - 1))) \
- if isinstance(raw, (int, float)) else 0.0
- scores.append({"candidate": c["candidate"], "relevance": round(relevance, 4)})
- log(f"jev {len(cands)} candidates in {dt:.3f}s query={query[:60]!r}")
- return {"scores": scores}
-
-
-class Handler(BaseHTTPRequestHandler):
- protocol_version = "HTTP/1.1"
-
- def log_message(self, fmt, *args):
- pass
-
- def _reply(self, status, body=b"", content_type="application/json"):
- self.send_response(status)
- if content_type:
- self.send_header("Content-Type", content_type)
- self.send_header("Content-Length", str(len(body)))
- self.end_headers()
- if body:
- self.wfile.write(body)
-
- def _proxy(self, raw):
- """Reverse-proxy this request unchanged to the upstream."""
- req = urllib.request.Request(UPSTREAM + self.path, data=raw, method=self.command)
- for k, v in self.headers.items():
- if k.lower() in ("host", "content-length", "connection", "accept-encoding"):
- continue
- req.add_header(k, v)
- try:
- with urllib.request.urlopen(req, timeout=900) as resp:
- body = resp.read()
- self.send_response(resp.status)
- for k, v in resp.headers.items():
- if k.lower() in ("transfer-encoding", "connection", "content-length"):
- continue
- self.send_header(k, v)
- self._reply_finish(body)
- except urllib.error.HTTPError as e:
- self._reply(e.code, e.read())
- except Exception as e: # transport failure upstream
- log(f"forward error {self.command} {self.path}: {e}")
- self._reply(502, json.dumps({"error": str(e)[:200]}).encode())
-
- def _reply_finish(self, body):
- # headers besides Content-Length were already sent by the caller
- self.send_header("Content-Length", str(len(body)))
- self.end_headers()
- self.wfile.write(body)
-
- def do_POST(self):
- length = int(self.headers.get("Content-Length") or 0)
- raw = self.rfile.read(length) if length else b""
- if not self.path.rstrip("/").endswith("/chat/completions"):
- self._proxy(raw)
- return
- try:
- payload = json.loads(raw or b"{}")
- except json.JSONDecodeError:
- self._reply(400)
- return
- system = next((m.get("content", "") for m in payload.get("messages", [])
- if m.get("role") == "system"), "")
- if not system.startswith(RERANK_SYSTEM_PREFIX):
- self._proxy(raw) # consolidation / lint / bootstrap traffic
- return
- try:
- result = jev_rerank(payload)
- out = {
- "id": "jev-rerank",
- "object": "chat.completion",
- "model": payload.get("model", JEV_MODEL),
- "choices": [{
- "index": 0,
- "message": {"role": "assistant", "content": json.dumps(result)},
- "finish_reason": "stop",
- }],
- "usage": {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0},
- }
- self._reply(200, json.dumps(out).encode())
- except Exception as e:
- # The server treats any failure as "keep the original order".
- log(f"jev rerank failed: {e}")
- self._reply(500)
-
- def do_GET(self):
- self._proxy(None)
-
-
-def main():
- host, port = LISTEN.rsplit(":", 1)
- srv = ThreadingHTTPServer((host, int(port)), Handler)
- log(f"listening {LISTEN} jev={JEV_URL} upstream={UPSTREAM}")
- try:
- srv.serve_forever()
- except KeyboardInterrupt:
- pass
-
-
-if __name__ == "__main__":
- main()
diff --git a/docs/jev-reranker-adapter.md b/docs/jev-reranker-adapter.md
deleted file mode 100644
index e22bafbb..00000000
--- a/docs/jev-reranker-adapter.md
+++ /dev/null
@@ -1,143 +0,0 @@
-# Jev Reranker Adapter
-
-> Route ai-memory's LLM reranker to a Jev judge endpoint while keeping the
-> hosted LLM for everything else. Stdlib-only Python, no Rust changes.
-
-[`AI_MEMORY_RERANKER=llm`](llm-providers.md) reorders `memory_query`
-candidates with the configured chat provider. That works until the provider
-is a hosted reasoning model: the reranker prompt is long, a graded judgement
-over up to 30 candidates, and a slow model answers in tens of seconds —
-longer than one `memory_query` should ever take, and long enough to trip the
-server's completion timeout, which turns the reranker into a dead feature
-that stalls every query before falling back to the original order.
-
-The reranker prompt, though, is already a scoring rubric: grade each
-candidate 1.0 (direct answer) / 0.7 (same topic) / 0.3 (tangential) /
-0.0 (unrelated). A judge endpoint that scores a fixed rubric answers the
-same question without autoregressive decoding. The adapter in
-[`docs/examples/jev-reranker-adapter/jev_rerank_shim.py`](examples/jev-reranker-adapter/jev_rerank_shim.py)
-sits between ai-memory and the provider, translates exactly that request
-into one batched Jev `score` call, and proxies everything else unchanged.
-
-## How it works
-
-ai-memory 2.4.0 has a single chat-provider configuration, so the reranker
-and consolidation share one base URL. The adapter does not try to split
-configuration; it splits traffic by request shape:
-
-1. The reranker's system prompt starts with the fixed sentence
- `You are a retrieval reranker for a software project's memory wiki.`
- (see `crates/ai-memory-llm/src/reranker.rs`). Requests whose system
- message has that prefix are reranker requests.
-2. The user message is `{"query", "candidates": [{"candidate", "title",
- "text"}]}` with 1-based indices. The adapter renders one `state` block
- (query + numbered candidates) and one `score` question per candidate
- whose `criteria` mirror the reranker prompt's four grades, then makes a
- single POST to the Jev `/v1/systemone` endpoint.
-3. Each answer's rubric index maps back to `relevance` 0.0 / 0.3 / 0.7 /
- 1.0 (index / 3), and the adapter returns
- `{"scores": [{"candidate": n, "relevance": f}]}` as plain chat-completion
- content — the exact shape the reranker's tolerant parser expects.
-4. Every other request — consolidation, lint, bootstrap, plain chats — is
- reverse-proxied to the real upstream byte-for-byte, `Authorization`
- forwarded verbatim. The adapter stores no secrets.
-
-Failure semantics match the server contract: if the Jev call fails, the
-adapter answers HTTP 500 and ai-memory keeps its own candidate order, the
-same as any provider outage. A judge endpoint that is down degrades to
-"no reranking", never to "no search".
-
-One cosmetic note: the server logs still show the provider's configured
-model name for reranking (it comes from provider config, not from the
-adapter's reply), so the reranker leg will be attributed to your hosted
-model in `docker logs` even while the adapter serves it.
-
-## Deploy
-
-The adapter is stdlib-only Python 3. Configure with environment variables
-(all defaults are loopback examples):
-
-| Env | Meaning | Default |
-|---|---|---|
-| `JEV_URL` | Jev `/v1/systemone` endpoint | `http://127.0.0.1:18095/v1/systemone` |
-| `JEV_MODEL` | model name sent to Jev | `jev-latest` |
-| `UPSTREAM` | real OpenAI-compat base URL | `http://127.0.0.1:8000` |
-| `LISTEN` | adapter bind address | `127.0.0.1:18097` |
-
-Run it next to the server (systemd unit adapted to your paths):
-
-```ini
-[Unit]
-Description=ai-memory Jev reranker adapter
-After=network-online.target
-
-[Service]
-ExecStart=/usr/bin/python3 /opt/ai-memory/ops/jev_rerank_shim.py
-Environment=JEV_URL=http://127.0.0.1:18095/v1/systemone
-Environment=UPSTREAM=http://127.0.0.1:8000
-Environment=LISTEN=127.0.0.1:18097
-Restart=always
-RestartSec=3
-
-[Install]
-WantedBy=multi-user.target
-```
-
-Then point the provider at the adapter instead of the upstream:
-
-```console
-AI_MEMORY_LLM_PROVIDER=openai-compat
-AI_MEMORY_LLM_BASE_URL=http://127.0.0.1:18097/v1
-AI_MEMORY_RERANKER=llm
-```
-
-Verify the split with `journalctl -u <unit>`: reranker requests log
-`jev N candidates in X.XXXs`, everything else is silent (proxied).
-
-## Benchmarks
-
-Offline A/B, same candidate pool per query, 102 queries / 99 scored against
-a hand-curated golden set for a real production wiki (FTS5 + vector + graph
-hybrid, `explain=true` score details, identical server build):
-
-| Reranker | hit@1 | MRR | NDCG@10 | mean rerank latency |
-|---|---|---|---|---|
-| none (server order) | 0.495 | 0.651 | 0.739 | — |
-| hosted model A (reasoning, max) | 0.778 | 0.840 | 0.875 | **20.2 s** |
-| hosted model B (reasoning) | 0.768 | 0.834 | 0.870 | 13.9 s |
-| **Jev via this adapter** | **0.778** | **0.838** | **0.873** | **0.205 s** |
-
-The judge endpoint matches the hosted reasoning models' quality at
-~100× lower latency. Hosted model A's 20.2 s mean sat exactly on the
-server's 20 s completion timeout, so in production every rerank call timed
-out and the feature was effectively off (queries stalled 20 s, then fell
-back to the original order).
-
-Live end-to-end (production server, `AI_MEMORY_RERANKER=llm` through the
-adapter, full golden set): hit@1 0.657 / MRR 0.788 / NDCG@10 0.843,
-0 errors in 102 queries, mean `memory_query` latency 2.2 s (p50 2.21 s,
-max 3.07 s), 110/110 rerank translations served, 0 adapter failures. The
-gap to the offline arm is candidate-pool shape, not scoring: the live
-server over-fetches 15–30 candidates with its own bounded snippets (the
-offline arm rescored a fixed top-10 pool), and in 32 of 34 non-top-1
-queries the expected page was still returned — mostly at rank 2 — with
-hit@5 at 0.949. Versus the no-reranker baseline that is +16.2 points
-hit@1 and +10.4 points NDCG@10 end to end.
-
-Judge latency scales with the candidate count in the batch (the rubric
-question is asked once per candidate): ~0.2 s for 10 candidates, ~1.6 s
-mean for the 15–30-candidate batches the live server sends, still an order
-of magnitude under any hosted reasoning model.
-
-## Caveats
-
-- The adapter keys on the exact system-prompt prefix above. If the
- reranker prompt wording changes in a future release, detection (not
- scoring) breaks first: reranker requests would be proxied to the hosted
- model, which is the pre-adapter behavior, not an outage.
-- Scoring uses Jev's calibrated rubric grades. Do not substitute
- per-candidate choice probabilities: those are normalized across options
- (winner ≈ 0.99, rest ≈ 0.001) and do not fit the reranker's 0–1
- relevance semantics.
-- Run the adapter on loopback or a trusted private network. It forwards
- `Authorization` headers verbatim and adds no authentication of its own.
diff --git a/docs/llm-providers.md b/docs/llm-providers.md
index 69391e24..3d0e69e5 100644
--- a/docs/llm-providers.md
+++ b/docs/llm-providers.md
@@ -191,11 +191,8 @@ request sends the query plus at most 30 bounded page titles and search snippets
to the configured provider; all values are JSON-encoded and treated as
untrusted data. A timeout, provider error, or incomplete/invalid score set
preserves the normal order. `global=true` and supplemental global-preference
-at four; saturated queries keep their local ranking without waiting. If the
-configured provider is too slow for interactive reranking, a judge-endpoint
-adapter can serve the reranker leg in sub-second time while consolidation
-keeps the hosted model — see
-[`docs/jev-reranker-adapter.md`](jev-reranker-adapter.md).
+hits keep their existing non-RRF ranking. Concurrent provider calls are capped
+at four; saturated queries keep their local ranking without waiting.
Embeddings are optional and separate from the LLM provider. Set
`AI_MEMORY_EMBEDDING_PROVIDER=openai`, `voyage`, `google`/`gemini`,