Skip to content

https://arxiv.org/abs/2606.28344. The end of web parsing. The beginning of scalable pixel-native search. link: https://pixelrag.ai/

fix(agent): an unreachable index must not end a chat as success (#147)

Zhifei Lif5d96e95e4f5
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index a39edbb..b34d49c 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -49,6 +49,9 @@ jobs:
- name: Install dependencies
run: npm ci
+ - name: Test
+ run: npm test
+
- name: Build
run: npm run build
diff --git a/deploy/README.md b/deploy/README.md
index b5df1b7..a78f91e 100644
--- a/deploy/README.md
+++ b/deploy/README.md
@@ -39,6 +39,25 @@ index or model with zero downtime:
This is preferred over restarting a slot in place, which reloads the (large) FAISS index and would
mean minutes of downtime.
+**Enable the slot you switch to.** `systemctl start` on a slot survives until the next reboot and
+no further — a slot that is running but not enabled comes back as a 502 the first time the box
+restarts, long after whoever started it has stopped watching. Both slot units and the agent should
+be `enabled`, and `systemctl is-enabled` on each of them belongs in the post-cutover check.
+
+## Monitoring
+The agent exposes two different questions, and monitoring wants the second one:
+
+- `GET /health` — **liveness**. The process is up. It answers 200 even when the search API behind
+ it is gone.
+- `GET /ready` — **readiness**. The process is up *and* the search API answers. 200 when the agent
+ can actually retrieve, 503 with the reason when it cannot.
+
+The distinction is not academic. With the index unreachable, the agent still accepts chats and the
+model still writes fluent answers — from its own memory rather than from retrieved tiles. Liveness
+stays green throughout, so a monitor watching only `/health` reports a healthy service that is
+silently answering off-corpus. Those turns now end as degraded rather than done and log as
+`chat DEGRADED`, and the frontend marks the reply as not grounded.
+
## Files
- `deploy.sh` — CD restart logic (invoked by the Deploy workflow)
- `api-switch.sh` — blue-green cutover + rollback
diff --git a/web/agent-server.mjs b/web/agent-server.mjs
index e0c9c68..2835871 100644
--- a/web/agent-server.mjs
+++ b/web/agent-server.mjs
@@ -22,6 +22,18 @@ import http from "node:http"
import { query, tool, createSdkMcpServer } from "@anthropic-ai/claude-agent-sdk"
import { z } from "zod"
+import {
+ RETRIEVAL_OK,
+ RETRIEVAL_BACKEND_DOWN,
+ classifyStatus,
+ classifyThrow,
+ createRetrievalHealth,
+ recordRetrieval,
+ isUngrounded,
+ backendDownInstruction,
+ UNGROUNDED_CLIENT_MESSAGE,
+} from "./lib/retrieval-health.mjs"
+
const PORT = parseInt(process.env.AGENT_PORT || "30010", 10)
const SEARCH_URL = process.env.PIXELRAG_SEARCH_URL || "https://api.pixelrag.ai"
const MAX_BUDGET = parseFloat(process.env.CHAT_MAX_BUDGET_USD || "2.00")
@@ -73,7 +85,16 @@ function log(...args) {
console.log(new Date().toISOString(), ...args)
}
-function createTools(onEvent, uploadedImage) {
+// An unreachable index is not information for the model to work around: it
+// ends the turn. Recorded on the turn's health so the stream can close as
+// degraded instead of done.
+function retrievalDown(health, onEvent, label, detail) {
+ recordRetrieval(health, RETRIEVAL_BACKEND_DOWN, detail)
+ onEvent("search_unavailable", { query: label, reason: detail })
+ return { content: [{ type: "text", text: backendDownInstruction(detail) }], isError: true }
+}
+
+function createTools(onEvent, uploadedImage, health) {
const searchTool = tool(
"pixelrag_search",
"Search the visual Wikipedia index by text, by the user's uploaded image, or BOTH combined. When the user uploaded an image, you MUST set use_uploaded_image=true AND provide a text query to get joint image+text retrieval — this gives the best results. Returns ranked results with article URLs, tile positions, and `pages` — the article's valid tile:chunk ranges (e.g. '0:0-7,1:0-4' = tile 0 has chunks 0-7, tile 1 has chunks 0-4). Use this first, then pixelrag_tile to view tiles.",
@@ -96,15 +117,26 @@ function createTools(onEvent, uploadedImage) {
if (args.query) queryObj.text = args.query
const label = searchByImage && args.query ? `${args.query} + uploaded image` : args.query || "uploaded image"
onEvent("searching", { query: label })
- const resp = await fetch(`${SEARCH_URL}/search`, {
- method: "POST",
- headers: { "Content-Type": "application/json", "X-Source": "chat" },
- body: JSON.stringify({ queries: [queryObj], n_docs: args.n_results ?? 5, articles_only: true }),
- signal: AbortSignal.timeout(30000),
- })
- if (!resp.ok) {
+ let resp
+ try {
+ resp = await fetch(`${SEARCH_URL}/search`, {
+ method: "POST",
+ headers: { "Content-Type": "application/json", "X-Source": "chat" },
+ body: JSON.stringify({ queries: [queryObj], n_docs: args.n_results ?? 5, articles_only: true }),
+ signal: AbortSignal.timeout(30000),
+ })
+ } catch (err) {
+ return retrievalDown(health, onEvent, label, String(err))
+ }
+ const outcome = classifyStatus(resp.status)
+ if (outcome === RETRIEVAL_BACKEND_DOWN) {
+ return retrievalDown(health, onEvent, label, `HTTP ${resp.status}`)
+ }
+ if (outcome !== RETRIEVAL_OK) {
+ recordRetrieval(health, outcome, `HTTP ${resp.status}`)
return { content: [{ type: "text", text: `Search API error: ${resp.status}` }] }
}
+ recordRetrieval(health, RETRIEVAL_OK)
const data = await resp.json()
const hits = data.results?.[0]?.hits ?? []
const results = hits.map((h) => {
@@ -141,13 +173,19 @@ function createTools(onEvent, uploadedImage) {
// The agent pages through articles by guessing chunk coordinates, so
// 404s are normal exploration — only surface tiles that actually load,
// otherwise the chat gallery renders broken images.
- if (!resp.ok) return { content: [{ type: "text", text: `Tile not found: ${resp.status}` }] }
+ if (!resp.ok) {
+ if (classifyStatus(resp.status) === RETRIEVAL_BACKEND_DOWN) {
+ recordRetrieval(health, RETRIEVAL_BACKEND_DOWN, `tile HTTP ${resp.status}`)
+ }
+ return { content: [{ type: "text", text: `Tile not found: ${resp.status}` }] }
+ }
onEvent("viewing_tile", { article_id: args.article_id, tile_index: args.tile_index, chunk_index: args.chunk_index })
const buffer = await resp.arrayBuffer()
const base64 = Buffer.from(buffer).toString("base64")
const mimeType = resp.headers.get("content-type") || "image/png"
return { content: [{ type: "image", data: base64, mimeType }] }
} catch (err) {
+ recordRetrieval(health, classifyThrow(err), `tile ${err}`)
return { content: [{ type: "text", text: `Failed to fetch tile: ${err}` }] }
}
}
@@ -173,6 +211,24 @@ const server = http.createServer(async (req, res) => {
return
}
+ // Liveness (/health) says this process is up. Readiness says it can actually
+ // do its job: with the index unreachable the agent still answers chats, just
+ // from model memory instead of tiles. Monitoring needs to see that.
+ if (req.method === "GET" && req.url === "/ready") {
+ let searchOk = false
+ let detail = null
+ try {
+ const probe = await fetch(`${SEARCH_URL}/health`, { signal: AbortSignal.timeout(5000) })
+ searchOk = probe.ok
+ if (!probe.ok) detail = `search HTTP ${probe.status}`
+ } catch (err) {
+ detail = `search unreachable: ${err}`
+ }
+ res.writeHead(searchOk ? 200 : 503, { "Content-Type": "application/json" })
+ res.end(JSON.stringify({ status: searchOk ? "ready" : "degraded", search: searchOk ? "ok" : detail }))
+ return
+ }
+
if (req.method !== "POST" || !req.url.startsWith("/chat")) {
res.writeHead(404); res.end("Not found"); return
}
@@ -246,7 +302,8 @@ const server = http.createServer(async (req, res) => {
const send = (event, data) => res.write(sse(event, data))
const uploadedImage = last?.image && typeof last.image === "string" ? last.image : null
- const tools = createTools(send, uploadedImage)
+ const health = createRetrievalHealth()
+ const tools = createTools(send, uploadedImage, health)
const mcpServer = createSdkMcpServer({ name: "pixelrag", version: "1.0.0", tools })
inFlight++
@@ -282,8 +339,15 @@ const server = http.createServer(async (req, res) => {
send("text", { text: message.result })
}
}
- send("done", {})
- log(`chat done in ${((Date.now() - t0) / 1000).toFixed(1)}s`)
+ const elapsed = ((Date.now() - t0) / 1000).toFixed(1)
+ if (isUngrounded(health)) {
+ send("error", { message: UNGROUNDED_CLIENT_MESSAGE })
+ send("done", { degraded: true })
+ log(`chat DEGRADED in ${elapsed}s — retrieval unavailable (${health.lastError})`)
+ } else {
+ send("done", {})
+ log(`chat done in ${elapsed}s`)
+ }
} catch (err) {
log("chat error:", String(err))
send("error", { message: String(err) })
diff --git a/web/app/api/chat/route.ts b/web/app/api/chat/route.ts
index 9c70e15..bd04b86 100644
--- a/web/app/api/chat/route.ts
+++ b/web/app/api/chat/route.ts
@@ -6,6 +6,19 @@ import {
} from "@anthropic-ai/claude-agent-sdk"
import { z } from "zod"
+import {
+ RETRIEVAL_OK,
+ RETRIEVAL_BACKEND_DOWN,
+ classifyStatus,
+ classifyThrow,
+ createRetrievalHealth,
+ recordRetrieval,
+ isUngrounded,
+ backendDownInstruction,
+ UNGROUNDED_CLIENT_MESSAGE,
+ type RetrievalHealth,
+} from "@/lib/retrieval-health.mjs"
+
const SEARCH_URL =
process.env.PIXELRAG_SEARCH_URL || "https://api.pixelrag.ai"
@@ -41,9 +54,27 @@ function sseEvent(event: string, data: unknown): string {
return `event: ${event}\ndata: ${JSON.stringify(data)}\n\n`
}
+// An unreachable index is not information for the model to work around: it
+// ends the turn. Recorded on the turn's health so the stream can close as
+// degraded instead of done.
+function retrievalDown(
+ health: RetrievalHealth,
+ onEvent: (event: string, data: unknown) => void,
+ label: string,
+ detail: string
+) {
+ recordRetrieval(health, RETRIEVAL_BACKEND_DOWN, detail)
+ onEvent("search_unavailable", { query: label, reason: detail })
+ return {
+ content: [{ type: "text" as const, text: backendDownInstruction(detail) }],
+ isError: true,
+ }
+}
+
function createTools(
onEvent: (event: string, data: unknown) => void,
- uploadedImage: string | null
+ uploadedImage: string | null,
+ health: RetrievalHealth
) {
const searchTool = tool(
"pixelrag_search",
@@ -97,17 +128,27 @@ function createTools(
: args.query || "uploaded image"
onEvent("searching", { query: label })
- const resp = await fetch(`${SEARCH_URL}/search`, {
- method: "POST",
- headers: { "Content-Type": "application/json", "X-Source": "chat" },
- body: JSON.stringify({
- queries: [queryObj],
- n_docs: args.n_results ?? 5,
- articles_only: true,
- }),
- signal: AbortSignal.timeout(30000),
- })
- if (!resp.ok) {
+ let resp: Response
+ try {
+ resp = await fetch(`${SEARCH_URL}/search`, {
+ method: "POST",
+ headers: { "Content-Type": "application/json", "X-Source": "chat" },
+ body: JSON.stringify({
+ queries: [queryObj],
+ n_docs: args.n_results ?? 5,
+ articles_only: true,
+ }),
+ signal: AbortSignal.timeout(30000),
+ })
+ } catch (err) {
+ return retrievalDown(health, onEvent, label, String(err))
+ }
+ const outcome = classifyStatus(resp.status)
+ if (outcome === RETRIEVAL_BACKEND_DOWN) {
+ return retrievalDown(health, onEvent, label, `HTTP ${resp.status}`)
+ }
+ if (outcome !== RETRIEVAL_OK) {
+ recordRetrieval(health, outcome, `HTTP ${resp.status}`)
return {
content: [
{
@@ -117,6 +158,7 @@ function createTools(
],
}
}
+ recordRetrieval(health, RETRIEVAL_OK)
const data = await resp.json()
const hits: SearchHit[] = data.results?.[0]?.hits ?? []
const results = hits.map((h: SearchHit) => {
@@ -176,6 +218,9 @@ function createTools(
})
}
if (!resp.ok) {
+ if (classifyStatus(resp.status) === RETRIEVAL_BACKEND_DOWN) {
+ recordRetrieval(health, RETRIEVAL_BACKEND_DOWN, `tile HTTP ${resp.status}`)
+ }
return {
content: [
{
@@ -200,6 +245,7 @@ function createTools(
],
}
} catch (err) {
+ recordRetrieval(health, classifyThrow(err), `tile ${err}`)
return {
content: [
{
@@ -298,7 +344,8 @@ export async function POST(req: Request) {
controller.enqueue(encoder.encode(sseEvent(event, data)))
}
- const tools = createTools(send, uploadedImage)
+ const health = createRetrievalHealth()
+ const tools = createTools(send, uploadedImage, health)
const mcpServer = createSdkMcpServer({
name: "pixelrag",
version: "1.0.0",
@@ -351,7 +398,12 @@ export async function POST(req: Request) {
}
}
- send("done", {})
+ if (isUngrounded(health)) {
+ send("error", { message: UNGROUNDED_CLIENT_MESSAGE })
+ send("done", { degraded: true })
+ } else {
+ send("done", {})
+ }
} catch (err) {
send("error", { message: String(err) })
} finally {
diff --git a/web/app/chat/page.tsx b/web/app/chat/page.tsx
index cc66824..b0792cb 100644
--- a/web/app/chat/page.tsx
+++ b/web/app/chat/page.tsx
@@ -15,6 +15,7 @@ import {
ImagePlus,
X,
Clock,
+ AlertTriangle,
} from "lucide-react"
import { tileUrl } from "@/lib/api"
import { getHistory, addHistory, clearHistory } from "@/lib/history"
@@ -55,6 +56,10 @@ interface ChatMessage {
searching?: string
tiles?: TileView[]
viewingTile?: boolean
+ // The retrieval backend was unreachable, so whatever the model said is not
+ // grounded in Wikipedia tiles. Marked so the answer cannot be mistaken for
+ // an ordinary one.
+ degraded?: boolean
}
const EXAMPLES = [
@@ -173,7 +178,8 @@ function ChatPageInner() {
case "searching": return { ...m, searching: data.query as string }
case "search_results": return { ...m, searching: undefined, searches: [...(m.searches || []), { query: data.query as string, hits: data.hits as SearchResult["hits"] }] }
case "viewing_tile": return { ...m, viewingTile: true, tiles: [...(m.tiles || []), { article_id: data.article_id as number, tile_index: data.tile_index as number, chunk_index: data.chunk_index as number }] }
- case "done": return { ...m, searching: undefined, viewingTile: false }
+ case "search_unavailable": return { ...m, searching: undefined, degraded: true }
+ case "done": return { ...m, searching: undefined, viewingTile: false, degraded: m.degraded || Boolean(data.degraded) }
case "error": return { ...m, content: m.content || `Error: ${data.message}`, searching: undefined }
default: return m
}
@@ -447,6 +453,15 @@ function AssistantMessage({ message, isStreaming, onTileClick }: { message: Chat
</motion.div>
)}
+ {message.degraded && (
+ <div className="mb-4 flex items-start gap-2.5 rounded-lg border border-amber-500/30 bg-amber-500/10 px-3.5 py-2.5">
+ <AlertTriangle className="mt-0.5 h-3.5 w-3.5 shrink-0 text-amber-500" />
+ <span className="text-[12px] leading-[1.6] text-[var(--chat-secondary)]">
+ Visual search was unavailable, so this reply was not read off Wikipedia screenshots. Try again in a moment.
+ </span>
+ </div>
+ )}
+
{message.content && (
<div className="chat-prose prose prose-sm max-w-none text-[14px] leading-[1.8] text-[var(--chat-secondary)] prose-headings:font-display prose-headings:font-normal prose-headings:tracking-tight prose-headings:text-[var(--chat-fg)] prose-h2:mt-6 prose-h2:text-lg prose-h3:mt-4 prose-h3:text-[15px] prose-p:text-[var(--chat-secondary)] prose-a:text-[var(--chat-accent)] prose-a:no-underline hover:prose-a:underline prose-strong:font-semibold prose-strong:text-[var(--chat-fg)] prose-code:rounded-md prose-code:bg-[var(--chat-card)] prose-code:px-1.5 prose-code:py-0.5 prose-code:text-[12px] prose-code:text-[var(--chat-secondary)] prose-code:before:content-none prose-code:after:content-none prose-pre:border prose-pre:border-[var(--chat-border)] prose-pre:bg-[var(--chat-card)] prose-pre:rounded-xl prose-li:marker:text-[var(--chat-muted)] prose-hr:border-[var(--chat-border)] prose-blockquote:border-l-[var(--chat-accent-dim)] prose-blockquote:text-[var(--chat-muted)] prose-th:font-mono prose-th:text-[11px] prose-th:font-medium prose-th:uppercase prose-th:tracking-wider prose-th:text-[var(--chat-muted)] prose-td:text-[var(--chat-secondary)] prose-table:text-[13px]">
<Markdown remarkPlugins={[remarkGfm]} components={{ a: ({ href, children }) => (<a href={href} target="_blank" rel="noopener noreferrer">{href?.includes("wikipedia.org") ? decodeURIComponent(href.split("/wiki/").pop() || String(children)).replace(/_/g, " ") : children}</a>) }}>
diff --git a/web/lib/retrieval-health.d.mts b/web/lib/retrieval-health.d.mts
new file mode 100644
index 0000000..9991d86
--- /dev/null
+++ b/web/lib/retrieval-health.d.mts
@@ -0,0 +1,28 @@
+// Type surface for the shared classifier. The implementation stays plain
+// `.mjs` because `web/agent-server.mjs` imports it directly under node, with
+// no build step between them.
+
+export type RetrievalOutcome = "ok" | "backend_down" | "bad_request"
+
+export interface RetrievalHealth {
+ ok: number
+ backendDown: number
+ badRequest: number
+ lastError: string | null
+}
+
+export declare const RETRIEVAL_OK: "ok"
+export declare const RETRIEVAL_BACKEND_DOWN: "backend_down"
+export declare const RETRIEVAL_BAD_REQUEST: "bad_request"
+export declare const UNGROUNDED_CLIENT_MESSAGE: string
+
+export declare function classifyStatus(status: number): RetrievalOutcome
+export declare function classifyThrow(err?: unknown): RetrievalOutcome
+export declare function createRetrievalHealth(): RetrievalHealth
+export declare function recordRetrieval(
+ health: RetrievalHealth,
+ outcome: RetrievalOutcome,
+ detail?: string | null
+): RetrievalHealth
+export declare function isUngrounded(health: RetrievalHealth): boolean
+export declare function backendDownInstruction(reason: string): string
diff --git a/web/lib/retrieval-health.mjs b/web/lib/retrieval-health.mjs
new file mode 100644
index 0000000..6c4f5e9
--- /dev/null
+++ b/web/lib/retrieval-health.mjs
@@ -0,0 +1,75 @@
+// Retrieval failure classification, shared by the two chat backends
+// (`web/agent-server.mjs` in production, the inline SDK path in
+// `web/app/api/chat/route.ts` for local dev) so the contract cannot drift
+// between them.
+//
+// The distinction that matters is NOT "did the search return hits" but
+// "was the index reached at all":
+//
+// - reached, no hits -> a real answer. The model should say so.
+// - never reached -> there is nothing to ground an answer in.
+//
+// The second case used to be handed to the model as an ordinary tool result
+// ("Search API error: 502"), which reads as information rather than as a
+// failure. The model would then answer from its own memory and the turn would
+// end as a success, so a dead retrieval backend produced confident, ungrounded
+// answers and a clean log line. PixelRAG's entire contract is that answers come
+// from retrieved tiles, so an unreachable index has to end the turn as
+// degraded, not as done.
+
+export const RETRIEVAL_OK = "ok"
+export const RETRIEVAL_BACKEND_DOWN = "backend_down"
+export const RETRIEVAL_BAD_REQUEST = "bad_request"
+
+// 5xx and 429 are the backend failing or shedding load; the request itself was
+// fine, so retrying with different arguments cannot help. 4xx means this call
+// was malformed — the model can usefully correct that one itself.
+export function classifyStatus(status) {
+ if (status >= 200 && status < 300) return RETRIEVAL_OK
+ if (status >= 500 || status === 429) return RETRIEVAL_BACKEND_DOWN
+ return RETRIEVAL_BAD_REQUEST
+}
+
+// A throw from fetch is a connection refused, DNS failure, or the abort signal
+// firing — the index was never reached in any of those cases.
+export function classifyThrow() {
+ return RETRIEVAL_BACKEND_DOWN
+}
+
+export function createRetrievalHealth() {
+ return { ok: 0, backendDown: 0, badRequest: 0, lastError: null }
+}
+
+export function recordRetrieval(health, outcome, detail = null) {
+ if (outcome === RETRIEVAL_OK) health.ok += 1
+ else if (outcome === RETRIEVAL_BACKEND_DOWN) {
+ health.backendDown += 1
+ health.lastError = detail
+ } else {
+ health.badRequest += 1
+ health.lastError = detail
+ }
+ return health
+}
+
+// True when the turn produced no grounded retrieval at all: the backend failed
+// and nothing ever came back from the index. A turn that failed once and then
+// succeeded is still grounded, so it is not degraded.
+export function isUngrounded(health) {
+ return health.backendDown > 0 && health.ok === 0
+}
+
+// What the model is told when the index is unreachable. It has to be an
+// instruction, not a status line: the failure mode being fixed is the model
+// treating an error string as context and answering anyway.
+export function backendDownInstruction(reason) {
+ return [
+ `RETRIEVAL BACKEND UNAVAILABLE (${reason}).`,
+ "The visual Wikipedia index cannot be reached, so there are no tiles to read.",
+ "Do NOT answer from your own knowledge, and do NOT retry the search.",
+ "Tell the user that PixelRAG's search backend is temporarily unavailable, then stop.",
+ ].join(" ")
+}
+
+export const UNGROUNDED_CLIENT_MESSAGE =
+ "PixelRAG's retrieval backend is unavailable, so no Wikipedia tiles could be read for this answer."
diff --git a/web/lib/retrieval-health.test.mjs b/web/lib/retrieval-health.test.mjs
new file mode 100644
index 0000000..ee51087
--- /dev/null
+++ b/web/lib/retrieval-health.test.mjs
@@ -0,0 +1,73 @@
+import test from "node:test"
+import assert from "node:assert/strict"
+
+import {
+ RETRIEVAL_OK,
+ RETRIEVAL_BACKEND_DOWN,
+ RETRIEVAL_BAD_REQUEST,
+ classifyStatus,
+ classifyThrow,
+ createRetrievalHealth,
+ recordRetrieval,
+ isUngrounded,
+ backendDownInstruction,
+} from "./retrieval-health.mjs"
+
+test("2xx is a reached index", () => {
+ assert.equal(classifyStatus(200), RETRIEVAL_OK)
+ assert.equal(classifyStatus(204), RETRIEVAL_OK)
+})
+
+test("5xx and 429 mean the index was not reached", () => {
+ for (const status of [500, 502, 503, 504, 429]) {
+ assert.equal(classifyStatus(status), RETRIEVAL_BACKEND_DOWN, `status ${status}`)
+ }
+})
+
+test("4xx is the call's own fault, not the backend's", () => {
+ for (const status of [400, 404, 422]) {
+ assert.equal(classifyStatus(status), RETRIEVAL_BAD_REQUEST, `status ${status}`)
+ }
+})
+
+test("a fetch throw is always an unreached index", () => {
+ assert.equal(classifyThrow(new TypeError("fetch failed")), RETRIEVAL_BACKEND_DOWN)
+})
+
+test("a turn whose only search hit a dead backend is ungrounded", () => {
+ const health = createRetrievalHealth()
+ recordRetrieval(health, RETRIEVAL_BACKEND_DOWN, "connection refused")
+ assert.equal(isUngrounded(health), true)
+ assert.equal(health.lastError, "connection refused")
+})
+
+test("empty results from a live backend are grounded, not degraded", () => {
+ // The regression this guards: "no hits" must stay an ordinary answer.
+ const health = createRetrievalHealth()
+ recordRetrieval(health, RETRIEVAL_OK)
+ assert.equal(isUngrounded(health), false)
+})
+
+test("a failure followed by a success is still grounded", () => {
+ const health = createRetrievalHealth()
+ recordRetrieval(health, RETRIEVAL_BACKEND_DOWN, "502")
+ recordRetrieval(health, RETRIEVAL_OK)
+ assert.equal(isUngrounded(health), false)
+})
+
+test("a malformed call alone does not mark the turn ungrounded", () => {
+ const health = createRetrievalHealth()
+ recordRetrieval(health, RETRIEVAL_BAD_REQUEST, "400")
+ assert.equal(isUngrounded(health), false)
+})
+
+test("a clean turn is grounded", () => {
+ assert.equal(isUngrounded(createRetrievalHealth()), false)
+})
+
+test("the model is instructed to stop, not merely informed", () => {
+ const text = backendDownInstruction("connection refused")
+ assert.match(text, /connection refused/)
+ assert.match(text, /Do NOT answer from your own knowledge/)
+ assert.match(text, /temporarily unavailable/)
+})
diff --git a/web/package.json b/web/package.json
index 2655330..74aeea2 100644
--- a/web/package.json
+++ b/web/package.json
@@ -9,7 +9,8 @@
"start": "next start",
"lint": "eslint",
"format": "prettier --write \"**/*.{ts,tsx}\"",
- "typecheck": "tsc --noEmit"
+ "typecheck": "tsc --noEmit",
+ "test": "node --test lib/*.test.mjs"
},
"dependencies": {
"@anthropic-ai/claude-agent-sdk": "^0.3.156",