Skip to content

armgit / Ix

Public Imported from GitHub
0

Understand any codebase instantly. System intelligence for codebases, built for humans and AI.

fix(backend): pin Arango, use the current vector-index flag, and diagnose the boot loop (#653)

KageBinary3ced3b155fec
diff --git a/docker-compose.standalone.yml b/docker-compose.standalone.yml
index 942b8ff2..5f55c7be 100644
--- a/docker-compose.standalone.yml
+++ b/docker-compose.standalone.yml
@@ -7,14 +7,27 @@
services:
arangodb:
- image: arangodb:3.12
+ # PINNED to an exact patch, deliberately. The rolling `3.12` tag let a
+ # `docker pull` move the engine under a live data volume, and ArangoDB's
+ # vector index leaves a one-way footprint: once `VectorIndex` is in the
+ # RocksDB MANIFEST, any arangod that does not register that column family
+ # refuses to open the database at all --
+ # FATAL [fe3df] unable to initialize RocksDB engine:
+ # Invalid argument: Column families not opened: VectorIndex
+ # -- which with `restart: unless-stopped` is a fatal loop, not a degraded
+ # start, and takes memory-layer down with it via `service_healthy`. Ix#614.
+ image: arangodb:3.12.11
networks:
- backend
ports:
- "127.0.0.1:8529:8529"
environment:
ARANGO_NO_AUTH: "1"
- command: ["arangod", "--server.endpoint", "tcp://0.0.0.0:8529", "--experimental-vector-index"]
+ # `--vector-index` is the current name; it was `--experimental-vector-index`
+ # before 3.12.11, which still accepts the old spelling but only as a renamed
+ # alias and warns. Pinning the image above is what makes it safe to state
+ # one spelling here.
+ command: ["arangod", "--server.endpoint", "tcp://0.0.0.0:8529", "--vector-index", "true"]
volumes:
- arangodb-data:/var/lib/arangodb3
healthcheck:
diff --git a/ix-cli/src/cli/__tests__/backend-stack-diagnosis.test.ts b/ix-cli/src/cli/__tests__/backend-stack-diagnosis.test.ts
new file mode 100644
index 00000000..938d3448
--- /dev/null
+++ b/ix-cli/src/cli/__tests__/backend-stack-diagnosis.test.ts
@@ -0,0 +1,115 @@
+import { describe, expect, it, vi, afterEach } from "vitest";
+
+/**
+ * Ix#614: a fatal Arango boot loop is invisible from the CLI. The container
+ * restarts forever, memory-layer never leaves `Created` behind
+ * `service_healthy`, and `ix doctor` reports only that the backend is
+ * unreachable — the same thing it says when nothing was started at all.
+ *
+ * These pin the classification, not docker. The log lines are verbatim from a
+ * real 3.12.11 container booted against a volume carrying the VectorIndex
+ * column family.
+ */
+const FATAL_VECTOR_INDEX =
+ "2026-09-08T18:35:45.330761Z [1-1] FATAL [fe3df] {startup} unable to initialize " +
+ "RocksDB engine: Invalid argument: Column families not opened: VectorIndex";
+
+const SEP = "|::|";
+
+function mockDocker(handler: (args: string[]) => string) {
+ vi.doMock("node:child_process", () => ({
+ execFileSync: (_cmd: string, args: string[]) => handler(args),
+ spawnSync: (_cmd: string, args: string[]) => ({ stdout: handler(args), stderr: "" }),
+ }));
+}
+
+afterEach(() => { vi.resetModules(); vi.doUnmock("node:child_process"); });
+
+describe("diagnoseBackendStack", () => {
+ it("classifies the VectorIndex boot loop and gives the remedy", async () => {
+ mockDocker((args) => {
+ if (args[0] === "ps") return ["abc123", "arangodb:3.12", "restarting", "arangodb"].join(SEP);
+ if (args[0] === "logs") return `some earlier line\n${FATAL_VECTOR_INDEX}`;
+ return "";
+ });
+ const { diagnoseBackendStack } = await import("../backend-status.js");
+ const f = diagnoseBackendStack();
+ expect(f).not.toBeNull();
+ expect(f!.service).toBe("arangodb");
+ expect(f!.state).toBe("restarting");
+ expect(f!.lastError).toContain("Column families not opened: VectorIndex");
+ expect(f!.remedy).toContain("--vector-index true");
+ });
+
+ it("ignores a running arango", async () => {
+ mockDocker((args) =>
+ args[0] === "ps" ? ["abc123", "arangodb:3.12", "running", "arangodb"].join(SEP) : "");
+ const { diagnoseBackendStack } = await import("../backend-status.js");
+ expect(diagnoseBackendStack()).toBeNull();
+ });
+
+ it("reports an unrecognised failure without inventing a remedy", async () => {
+ mockDocker((args) => {
+ if (args[0] === "ps") return ["d1", "arangodb:3.12", "exited", "arangodb"].join(SEP);
+ if (args[0] === "logs") return "FATAL [xxxxx] {startup} disk quota exceeded";
+ return "";
+ });
+ const { diagnoseBackendStack } = await import("../backend-status.js");
+ const f = diagnoseBackendStack();
+ expect(f!.lastError).toContain("disk quota exceeded");
+ expect(f!.remedy).toBeNull();
+ });
+
+ it("finds the fatal line when it arrives on stderr", async () => {
+ vi.doMock("node:child_process", () => ({
+ execFileSync: (_c: string, args: string[]) =>
+ args[0] === "ps" ? ["e1", "arangodb:3.12", "exited", "arangodb"].join(SEP) : "",
+ spawnSync: () => ({ stdout: "", stderr: FATAL_VECTOR_INDEX }),
+ }));
+ const { diagnoseBackendStack } = await import("../backend-status.js");
+ expect(diagnoseBackendStack()!.remedy).toContain("--vector-index true");
+ });
+
+ it("returns null when no arango container exists at all", async () => {
+ mockDocker((args) =>
+ args[0] === "ps" ? ["z1", "ghcr.io/ix-infrastructure/ix-memory-layer:latest", "exited", "memory-layer"].join(SEP) : "");
+ const { diagnoseBackendStack } = await import("../backend-status.js");
+ expect(diagnoseBackendStack()).toBeNull();
+ });
+});
+
+describe("candidate ranking", () => {
+ it("prefers a restarting container over a stale exited one", async () => {
+ // The real shape this guards: a healthy install plus a long-dead arango
+ // from an old experiment. Reporting the corpse's error as the current
+ // outage is worse than saying nothing.
+ mockDocker((args) => {
+ if (args[0] === "ps") {
+ return [
+ ["old1", "arangodb:3.12", "exited", ""].join(SEP),
+ ["live1", "arangodb:3.12", "restarting", "arangodb"].join(SEP),
+ ].join("\n");
+ }
+ if (args[0] === "logs" && args.includes("live1")) return FATAL_VECTOR_INDEX;
+ return "FATAL [zzzzz] something ancient and irrelevant";
+ });
+ const { diagnoseBackendStack } = await import("../backend-status.js");
+ const f = diagnoseBackendStack();
+ expect(f!.containerId).toBe("live1");
+ expect(f!.remedy).toContain("--vector-index true");
+ });
+
+ it("prefers a compose-managed container among equals", async () => {
+ mockDocker((args) => {
+ if (args[0] === "ps") {
+ return [
+ ["hand", "arangodb:3.12", "exited", ""].join(SEP),
+ ["composed", "arangodb:3.12", "exited", "arangodb"].join(SEP),
+ ].join("\n");
+ }
+ return FATAL_VECTOR_INDEX;
+ });
+ const { diagnoseBackendStack } = await import("../backend-status.js");
+ expect(diagnoseBackendStack()!.containerId).toBe("composed");
+ });
+});
diff --git a/ix-cli/src/cli/backend-status.ts b/ix-cli/src/cli/backend-status.ts
index 73e4d40f..af2a2857 100644
--- a/ix-cli/src/cli/backend-status.ts
+++ b/ix-cli/src/cli/backend-status.ts
@@ -12,7 +12,7 @@
// reported schema_version against what this CLI expects, so `ix doctor` and
// `ix upgrade` can surface both instead of looking mysteriously broken.
-import { execFileSync } from "node:child_process";
+import { execFileSync, spawnSync } from "node:child_process";
import { join } from "node:path";
import { homedir } from "node:os";
import type { IxClient } from "../client/api.js";
@@ -129,6 +129,112 @@ export function inspectBackendContainer(): BackendContainer | null {
return publishers[0] ?? null;
}
+/**
+ * A container in the backend stack that is down, with whatever it said on the
+ * way out.
+ *
+ * Exists because a fatal Arango boot loop is invisible from the CLI: the
+ * failure is inside a container that keeps restarting, so every command --
+ * `ix doctor` included -- sees nothing but a backend that will not answer, and
+ * "backend not started" is indistinguishable from "backend cannot start and
+ * never will". Ix#614.
+ */
+export interface StackFailure {
+ /** Compose service name where known, else the image reference. */
+ service: string;
+ containerId: string;
+ /** Docker's own state string: `restarting`, `exited`, ... */
+ state: string;
+ /** The most recent fatal-looking log line, trimmed. */
+ lastError: string | null;
+ /** Set only for a failure whose remedy we actually know. */
+ remedy: string | null;
+}
+
+/** `docker logs` with stdout and stderr merged; "" on any failure. */
+function dockerLogsMerged(containerId: string, tail = "80"): string {
+ const r = spawnSync("docker", ["logs", "--tail", tail, containerId], {
+ encoding: "utf-8",
+ timeout: 10000,
+ });
+ return `${r.stdout ?? ""}\n${r.stderr ?? ""}`;
+}
+
+/** Last line that looks like a hard failure, searching newest-first. */
+function lastFatalLine(logs: string): string | null {
+ const lines = logs.split("\n").map((l) => l.trim()).filter(Boolean);
+ for (let i = lines.length - 1; i >= 0; i--) {
+ if (/\bFATAL\b|\bunable to initialize\b|\bInvalid argument\b/i.test(lines[i]!)) return lines[i]!;
+ }
+ return lines.length > 0 ? lines[lines.length - 1]! : null;
+}
+
+/**
+ * Recognise failures whose fix is known, so the user gets an instruction and
+ * not just a transcript.
+ */
+function remedyFor(line: string | null): string | null {
+ if (!line) return null;
+ if (/Column families not opened:\s*VectorIndex/i.test(line)) {
+ // One-way door: the column family is in the RocksDB MANIFEST for good, so
+ // the only non-destructive fix is to keep registering it. Ix#614.
+ return "ArangoDB's data directory has the VectorIndex column family, so arangod must be started with it enabled. " +
+ "Add `--vector-index true` to the arangodb `command:` in ~/.ix/backend/docker-compose.yml, then `ix docker restart`. " +
+ "(The option is spelled `--experimental-vector-index` before ArangoDB 3.12.11.) No data is lost.";
+ }
+ return null;
+}
+
+/**
+ * Look for a container in the backend stack that is not running.
+ *
+ * Deliberately image-based rather than compose-label-based: the label is absent
+ * for a hand-run container, and this is called precisely when the stack is in a
+ * state nobody planned. Returns the highest-ranked non-running Arango -- an
+ * actively restarting, compose-managed one first, since that is what holds
+ * memory-layer's `service_healthy` gate shut.
+ */
+export function diagnoseBackendStack(): StackFailure | null {
+ const SEP = "|::|";
+ const listed = docker(["ps", "-a", "--format", `{{.ID}}${SEP}{{.Image}}${SEP}{{.State}}${SEP}{{.Label "com.docker.compose.service"}}`]);
+ if (!listed) return null;
+
+ // Rank before picking. A boot LOOP presents as `restarting`, and that is the
+ // one holding memory-layer's `service_healthy` gate shut; a long-dead
+ // `exited` container from an old experiment is the likeliest false positive,
+ // and reporting its error as the current outage would be worse than saying
+ // nothing. A compose-managed container outranks a hand-run one for the same
+ // reason.
+ const rank = (state: string, service: string): number =>
+ (state === "restarting" ? 0 : 2) + (service ? 0 : 1);
+
+ const candidates = listed.split("\n")
+ .map((row) => {
+ const [id = "", image = "", state = "", service = ""] = row.split(SEP);
+ return { id, image, state, service };
+ })
+ .filter((c) => c.id && /arangodb/i.test(c.image) && c.state !== "running")
+ .sort((a, b) => rank(a.state, a.service) - rank(b.state, b.service));
+
+ for (const { id, image, state, service } of candidates) {
+ // --tail keeps this bounded; a boot loop can produce a very large log.
+ // Both streams: `docker logs` keeps the container's stdout/stderr apart,
+ // and which one carries the fatal line is the container's choice, not
+ // ours. The arangodb image logs it on stdout; a config that sends it to
+ // stderr would otherwise make this whole diagnostic silently find nothing.
+ const logs = dockerLogsMerged(id);
+ const lastError = lastFatalLine(logs);
+ return {
+ service: service || image,
+ containerId: id,
+ state: state || "unknown",
+ lastError,
+ remedy: remedyFor(lastError),
+ };
+ }
+ return null;
+}
+
export type BackendImageStatus =
| { kind: "ok"; container: BackendContainer }
| { kind: "local-build"; container: BackendContainer }
diff --git a/ix-cli/src/cli/commands/doctor.ts b/ix-cli/src/cli/commands/doctor.ts
index e4bd29fd..c222ccc8 100644
--- a/ix-cli/src/cli/commands/doctor.ts
+++ b/ix-cli/src/cli/commands/doctor.ts
@@ -13,6 +13,8 @@ import {
checkBackendImage,
checkBackendSchema,
isNonStandardBackend,
+ dockerAvailable,
+ diagnoseBackendStack,
} from "../backend-status.js";
import { backendCeiling, isNewer, readBackendHealth } from "./upgrade.js";
import { loadIngestBaseline } from "../ingest-baseline.js";
@@ -180,7 +182,18 @@ export function registerDoctorCommand(program: Command): void {
const h = await readBackendHealth(client);
return { ok: h.status === "ok", detail: `${endpoint} → ${h.status}` };
} catch (e: any) {
- return { ok: false, detail: e.message ?? "unreachable" };
+ // An unreachable backend is where a fatal Arango boot loop hides:
+ // the container restarts forever, memory-layer never leaves
+ // `Created` behind `service_healthy`, and from out here that is
+ // indistinguishable from "not started yet". Say what the stack
+ // actually reported. Ix#614.
+ const base = e.message ?? "unreachable";
+ const failure = dockerAvailable() ? diagnoseBackendStack() : null;
+ if (!failure) return { ok: false, detail: base };
+ const parts = [`${base} — ${failure.service} is ${failure.state}`];
+ if (failure.lastError) parts.push(` last log: ${failure.lastError}`);
+ if (failure.remedy) parts.push(` fix: ${failure.remedy}`);
+ return { ok: false, detail: parts.join("\n") };
}
},
},