Understand any codebase instantly. System intelligence for codebases, built for humans and AI.
TypeScript aiai-memoryclaude-codeclicode-analysiscode-mappingcode-memorycodebase-understandingcodexdeveloper-toolsllmopenclawpersistent-memoryprogram-analysissoftware-architecturesystem-intelligencesystem-mapping
fix(backend): pin Arango, use the current vector-index flag, and diagnose the boot loop (#653)
KageBinary3ced3b155fec
diff --git a/docker-compose.standalone.yml b/docker-compose.standalone.ymlindex 942b8ff2..5f55c7be 100644--- a/docker-compose.standalone.yml+++ b/docker-compose.standalone.yml@@ -7,14 +7,27 @@services:arangodb:- image: arangodb:3.12+ # PINNED to an exact patch, deliberately. The rolling `3.12` tag let a+ # `docker pull` move the engine under a live data volume, and ArangoDB's+ # vector index leaves a one-way footprint: once `VectorIndex` is in the+ # RocksDB MANIFEST, any arangod that does not register that column family+ # refuses to open the database at all --+ # FATAL [fe3df] unable to initialize RocksDB engine:+ # Invalid argument: Column families not opened: VectorIndex+ # -- which with `restart: unless-stopped` is a fatal loop, not a degraded+ # start, and takes memory-layer down with it via `service_healthy`. Ix#614.+ image: arangodb:3.12.11networks:- backendports:- "127.0.0.1:8529:8529"environment:ARANGO_NO_AUTH: "1"- command: ["arangod", "--server.endpoint", "tcp://0.0.0.0:8529", "--experimental-vector-index"]+ # `--vector-index` is the current name; it was `--experimental-vector-index`+ # before 3.12.11, which still accepts the old spelling but only as a renamed+ # alias and warns. Pinning the image above is what makes it safe to state+ # one spelling here.+ command: ["arangod", "--server.endpoint", "tcp://0.0.0.0:8529", "--vector-index", "true"]volumes:- arangodb-data:/var/lib/arangodb3healthcheck:diff --git a/ix-cli/src/cli/__tests__/backend-stack-diagnosis.test.ts b/ix-cli/src/cli/__tests__/backend-stack-diagnosis.test.tsnew file mode 100644index 00000000..938d3448--- /dev/null+++ b/ix-cli/src/cli/__tests__/backend-stack-diagnosis.test.ts@@ -0,0 +1,115 @@+import { describe, expect, it, vi, afterEach } from "vitest";++/**+ * Ix#614: a fatal Arango boot loop is invisible from the CLI. The container+ * restarts forever, memory-layer never leaves `Created` behind+ * `service_healthy`, and `ix doctor` reports only that the backend is+ * unreachable — the same thing it says when nothing was started at all.+ *+ * These pin the classification, not docker. The log lines are verbatim from a+ * real 3.12.11 container booted against a volume carrying the VectorIndex+ * column family.+ */+const FATAL_VECTOR_INDEX =+ "2026-09-08T18:35:45.330761Z [1-1] FATAL [fe3df] {startup} unable to initialize " ++ "RocksDB engine: Invalid argument: Column families not opened: VectorIndex";++const SEP = "|::|";++function mockDocker(handler: (args: string[]) => string) {+ vi.doMock("node:child_process", () => ({+ execFileSync: (_cmd: string, args: string[]) => handler(args),+ spawnSync: (_cmd: string, args: string[]) => ({ stdout: handler(args), stderr: "" }),+ }));+}++afterEach(() => { vi.resetModules(); vi.doUnmock("node:child_process"); });++describe("diagnoseBackendStack", () => {+ it("classifies the VectorIndex boot loop and gives the remedy", async () => {+ mockDocker((args) => {+ if (args[0] === "ps") return ["abc123", "arangodb:3.12", "restarting", "arangodb"].join(SEP);+ if (args[0] === "logs") return `some earlier line\n${FATAL_VECTOR_INDEX}`;+ return "";+ });+ const { diagnoseBackendStack } = await import("../backend-status.js");+ const f = diagnoseBackendStack();+ expect(f).not.toBeNull();+ expect(f!.service).toBe("arangodb");+ expect(f!.state).toBe("restarting");+ expect(f!.lastError).toContain("Column families not opened: VectorIndex");+ expect(f!.remedy).toContain("--vector-index true");+ });++ it("ignores a running arango", async () => {+ mockDocker((args) =>+ args[0] === "ps" ? ["abc123", "arangodb:3.12", "running", "arangodb"].join(SEP) : "");+ const { diagnoseBackendStack } = await import("../backend-status.js");+ expect(diagnoseBackendStack()).toBeNull();+ });++ it("reports an unrecognised failure without inventing a remedy", async () => {+ mockDocker((args) => {+ if (args[0] === "ps") return ["d1", "arangodb:3.12", "exited", "arangodb"].join(SEP);+ if (args[0] === "logs") return "FATAL [xxxxx] {startup} disk quota exceeded";+ return "";+ });+ const { diagnoseBackendStack } = await import("../backend-status.js");+ const f = diagnoseBackendStack();+ expect(f!.lastError).toContain("disk quota exceeded");+ expect(f!.remedy).toBeNull();+ });++ it("finds the fatal line when it arrives on stderr", async () => {+ vi.doMock("node:child_process", () => ({+ execFileSync: (_c: string, args: string[]) =>+ args[0] === "ps" ? ["e1", "arangodb:3.12", "exited", "arangodb"].join(SEP) : "",+ spawnSync: () => ({ stdout: "", stderr: FATAL_VECTOR_INDEX }),+ }));+ const { diagnoseBackendStack } = await import("../backend-status.js");+ expect(diagnoseBackendStack()!.remedy).toContain("--vector-index true");+ });++ it("returns null when no arango container exists at all", async () => {+ mockDocker((args) =>+ args[0] === "ps" ? ["z1", "ghcr.io/ix-infrastructure/ix-memory-layer:latest", "exited", "memory-layer"].join(SEP) : "");+ const { diagnoseBackendStack } = await import("../backend-status.js");+ expect(diagnoseBackendStack()).toBeNull();+ });+});++describe("candidate ranking", () => {+ it("prefers a restarting container over a stale exited one", async () => {+ // The real shape this guards: a healthy install plus a long-dead arango+ // from an old experiment. Reporting the corpse's error as the current+ // outage is worse than saying nothing.+ mockDocker((args) => {+ if (args[0] === "ps") {+ return [+ ["old1", "arangodb:3.12", "exited", ""].join(SEP),+ ["live1", "arangodb:3.12", "restarting", "arangodb"].join(SEP),+ ].join("\n");+ }+ if (args[0] === "logs" && args.includes("live1")) return FATAL_VECTOR_INDEX;+ return "FATAL [zzzzz] something ancient and irrelevant";+ });+ const { diagnoseBackendStack } = await import("../backend-status.js");+ const f = diagnoseBackendStack();+ expect(f!.containerId).toBe("live1");+ expect(f!.remedy).toContain("--vector-index true");+ });++ it("prefers a compose-managed container among equals", async () => {+ mockDocker((args) => {+ if (args[0] === "ps") {+ return [+ ["hand", "arangodb:3.12", "exited", ""].join(SEP),+ ["composed", "arangodb:3.12", "exited", "arangodb"].join(SEP),+ ].join("\n");+ }+ return FATAL_VECTOR_INDEX;+ });+ const { diagnoseBackendStack } = await import("../backend-status.js");+ expect(diagnoseBackendStack()!.containerId).toBe("composed");+ });+});diff --git a/ix-cli/src/cli/backend-status.ts b/ix-cli/src/cli/backend-status.tsindex 73e4d40f..af2a2857 100644--- a/ix-cli/src/cli/backend-status.ts+++ b/ix-cli/src/cli/backend-status.ts@@ -12,7 +12,7 @@// reported schema_version against what this CLI expects, so `ix doctor` and// `ix upgrade` can surface both instead of looking mysteriously broken.-import { execFileSync } from "node:child_process";+import { execFileSync, spawnSync } from "node:child_process";import { join } from "node:path";import { homedir } from "node:os";import type { IxClient } from "../client/api.js";@@ -129,6 +129,112 @@ export function inspectBackendContainer(): BackendContainer | null {return publishers[0] ?? null;}+/**+ * A container in the backend stack that is down, with whatever it said on the+ * way out.+ *+ * Exists because a fatal Arango boot loop is invisible from the CLI: the+ * failure is inside a container that keeps restarting, so every command --+ * `ix doctor` included -- sees nothing but a backend that will not answer, and+ * "backend not started" is indistinguishable from "backend cannot start and+ * never will". Ix#614.+ */+export interface StackFailure {+ /** Compose service name where known, else the image reference. */+ service: string;+ containerId: string;+ /** Docker's own state string: `restarting`, `exited`, ... */+ state: string;+ /** The most recent fatal-looking log line, trimmed. */+ lastError: string | null;+ /** Set only for a failure whose remedy we actually know. */+ remedy: string | null;+}++/** `docker logs` with stdout and stderr merged; "" on any failure. */+function dockerLogsMerged(containerId: string, tail = "80"): string {+ const r = spawnSync("docker", ["logs", "--tail", tail, containerId], {+ encoding: "utf-8",+ timeout: 10000,+ });+ return `${r.stdout ?? ""}\n${r.stderr ?? ""}`;+}++/** Last line that looks like a hard failure, searching newest-first. */+function lastFatalLine(logs: string): string | null {+ const lines = logs.split("\n").map((l) => l.trim()).filter(Boolean);+ for (let i = lines.length - 1; i >= 0; i--) {+ if (/\bFATAL\b|\bunable to initialize\b|\bInvalid argument\b/i.test(lines[i]!)) return lines[i]!;+ }+ return lines.length > 0 ? lines[lines.length - 1]! : null;+}++/**+ * Recognise failures whose fix is known, so the user gets an instruction and+ * not just a transcript.+ */+function remedyFor(line: string | null): string | null {+ if (!line) return null;+ if (/Column families not opened:\s*VectorIndex/i.test(line)) {+ // One-way door: the column family is in the RocksDB MANIFEST for good, so+ // the only non-destructive fix is to keep registering it. Ix#614.+ return "ArangoDB's data directory has the VectorIndex column family, so arangod must be started with it enabled. " ++ "Add `--vector-index true` to the arangodb `command:` in ~/.ix/backend/docker-compose.yml, then `ix docker restart`. " ++ "(The option is spelled `--experimental-vector-index` before ArangoDB 3.12.11.) No data is lost.";+ }+ return null;+}++/**+ * Look for a container in the backend stack that is not running.+ *+ * Deliberately image-based rather than compose-label-based: the label is absent+ * for a hand-run container, and this is called precisely when the stack is in a+ * state nobody planned. Returns the highest-ranked non-running Arango -- an+ * actively restarting, compose-managed one first, since that is what holds+ * memory-layer's `service_healthy` gate shut.+ */+export function diagnoseBackendStack(): StackFailure | null {+ const SEP = "|::|";+ const listed = docker(["ps", "-a", "--format", `{{.ID}}${SEP}{{.Image}}${SEP}{{.State}}${SEP}{{.Label "com.docker.compose.service"}}`]);+ if (!listed) return null;++ // Rank before picking. A boot LOOP presents as `restarting`, and that is the+ // one holding memory-layer's `service_healthy` gate shut; a long-dead+ // `exited` container from an old experiment is the likeliest false positive,+ // and reporting its error as the current outage would be worse than saying+ // nothing. A compose-managed container outranks a hand-run one for the same+ // reason.+ const rank = (state: string, service: string): number =>+ (state === "restarting" ? 0 : 2) + (service ? 0 : 1);++ const candidates = listed.split("\n")+ .map((row) => {+ const [id = "", image = "", state = "", service = ""] = row.split(SEP);+ return { id, image, state, service };+ })+ .filter((c) => c.id && /arangodb/i.test(c.image) && c.state !== "running")+ .sort((a, b) => rank(a.state, a.service) - rank(b.state, b.service));++ for (const { id, image, state, service } of candidates) {+ // --tail keeps this bounded; a boot loop can produce a very large log.+ // Both streams: `docker logs` keeps the container's stdout/stderr apart,+ // and which one carries the fatal line is the container's choice, not+ // ours. The arangodb image logs it on stdout; a config that sends it to+ // stderr would otherwise make this whole diagnostic silently find nothing.+ const logs = dockerLogsMerged(id);+ const lastError = lastFatalLine(logs);+ return {+ service: service || image,+ containerId: id,+ state: state || "unknown",+ lastError,+ remedy: remedyFor(lastError),+ };+ }+ return null;+}+export type BackendImageStatus =| { kind: "ok"; container: BackendContainer }| { kind: "local-build"; container: BackendContainer }diff --git a/ix-cli/src/cli/commands/doctor.ts b/ix-cli/src/cli/commands/doctor.tsindex e4bd29fd..c222ccc8 100644--- a/ix-cli/src/cli/commands/doctor.ts+++ b/ix-cli/src/cli/commands/doctor.ts@@ -13,6 +13,8 @@ import {checkBackendImage,checkBackendSchema,isNonStandardBackend,+ dockerAvailable,+ diagnoseBackendStack,} from "../backend-status.js";import { backendCeiling, isNewer, readBackendHealth } from "./upgrade.js";import { loadIngestBaseline } from "../ingest-baseline.js";@@ -180,7 +182,18 @@ export function registerDoctorCommand(program: Command): void {const h = await readBackendHealth(client);return { ok: h.status === "ok", detail: `${endpoint} → ${h.status}` };} catch (e: any) {- return { ok: false, detail: e.message ?? "unreachable" };+ // An unreachable backend is where a fatal Arango boot loop hides:+ // the container restarts forever, memory-layer never leaves+ // `Created` behind `service_healthy`, and from out here that is+ // indistinguishable from "not started yet". Say what the stack+ // actually reported. Ix#614.+ const base = e.message ?? "unreachable";+ const failure = dockerAvailable() ? diagnoseBackendStack() : null;+ if (!failure) return { ok: false, detail: base };+ const parts = [`${base} — ${failure.service} is ${failure.state}`];+ if (failure.lastError) parts.push(` last log: ${failure.lastError}`);+ if (failure.remedy) parts.push(` fix: ${failure.remedy}`);+ return { ok: false, detail: parts.join("\n") };}},},