Skip to content

armgit / Ix

Public Imported from GitHub
0

Understand any codebase instantly. System intelligence for codebases, built for humans and AI.

13.9 KB
Raw Blame
// Copyright 2026 Ix Infrastructure Inc.

// Backend freshness helpers (Ix#270, Ix#271).
//
// Two distinct staleness problems can make a healthy-looking backend serve
// wrong/empty results:
//   1. The running CONTAINER is not the released image (a local dev build, or an
//      older digest), yet `ix upgrade` thinks it is current because it trusts the
//      ~/.ix/.backend-version stamp instead of the running image (Ix#270).
//   2. The persisted GRAPH predates the running engine's on-disk format, so
//      scoped reads silently return empty until the user re-maps (Ix#271).
//
// This module inspects the actual running container and compares the backend's
// reported schema_version against what this CLI expects, so `ix doctor` and
// `ix upgrade` can surface both instead of looking mysteriously broken.

import { execFileSync, spawnSync } from "node:child_process";
import { join } from "node:path";
import type { IxClient } from "../client/api.js";
import { fetchBackendHealth, isLocalEndpoint } from "./backend-version.js";
import { getEndpoint } from "./config.js";
import { ixHome } from "./ix-home.js";

export const BACKEND_IMAGE = "ghcr.io/ix-infrastructure/ix-memory-layer";

// On-disk graph format this CLI expects. MUST stay in sync with the backend's
// reported schema_version; a mismatch forces a full re-ingest (see ingest.ts,
// which imports this constant). Bump when the node-identity/format changes.
export const CLIENT_EXPECTED_SCHEMA_VERSION = 3;

function docker(args: string[], timeout = 10000): string | null {
  try {
    return execFileSync("docker", args, { encoding: "utf-8", timeout }).trim();
  } catch {
    return null;
  }
}

export function dockerAvailable(): boolean {
  return docker(["info"]) !== null;
}

export interface BackendContainer {
  containerId: string;
  /** Image reference the container was created from (e.g. ghcr.io/...:latest). */
  imageRef: string;
  /** Resolved image content id (sha256:...) the container is actually running. */
  imageId: string;
  /** Registry digests of that image; empty for a never-pushed local build. */
  repoDigests: string[];
  composeProject: string | null;
  composeConfigFiles: string | null;
}

function inspectContainer(containerId: string): BackendContainer | null {
  // A unique separator keeps Go-template parsing robust against odd image refs.
  const SEP = "|::|";
  const fmt =
    `{{.Image}}${SEP}{{.Config.Image}}` +
    `${SEP}{{index .Config.Labels "com.docker.compose.project"}}` +
    `${SEP}{{index .Config.Labels "com.docker.compose.project.config_files"}}`;
  const inspected = docker(["inspect", containerId, "--format", fmt]);
  if (!inspected) return null;
  const [imageId = "", imageRef = "", project = "", configFiles = ""] = inspected.split(SEP);

  let repoDigests: string[] = [];
  const digestsJson = docker(["image", "inspect", imageId || imageRef, "--format", "{{json .RepoDigests}}"]);
  if (digestsJson) {
    try {
      const parsed = JSON.parse(digestsJson);
      if (Array.isArray(parsed)) repoDigests = parsed;
    } catch {
      /* leave empty */
    }
  }

  return {
    containerId,
    imageRef,
    imageId,
    repoDigests,
    composeProject: project || null,
    composeConfigFiles: configFiles || null,
  };
}

function containerIds(output: string | null): string[] {
  return output?.split("\n").map((id) => id.trim()).filter(Boolean) ?? [];
}

function isReleasedBackendRef(imageRef: string): boolean {
  return imageRef === BACKEND_IMAGE ||
    imageRef.startsWith(`${BACKEND_IMAGE}:`) ||
    imageRef.startsWith(`${BACKEND_IMAGE}@`);
}

/**
 * The host port a container publishes to serve `endpoint`, or null when the
 * endpoint is not on this machine and no local container can be serving it.
 * Taken from the configured endpoint, not a fixed 8090: with IX_ENDPOINT on
 * another port, a fixed 8090 inspected some other stack on the same host.
 */
export function backendPortFor(endpoint: string): string | null {
  if (!isLocalEndpoint(endpoint)) return null;
  try {
    const url = new URL(endpoint);
    return url.port || (url.protocol === "https:" ? "443" : "80");
  } catch {
    return null;
  }
}

/** Inspect the backend reached through the container publishing the endpoint's port. */
export function inspectBackendContainer(endpoint: string = getEndpoint()): BackendContainer | null {
  const port = backendPortFor(endpoint);
  if (!port) return null;
  const publisherIds = containerIds(
    docker(["ps", "--filter", `publish=${port}`, "--format", "{{.ID}}"]),
  );
  if (publisherIds.length === 0) return null;

  const publishers = publisherIds
    .map((id) => inspectContainer(id))
    .filter((container): container is BackendContainer => container !== null);
  const directBackend = publishers.find((container) => isReleasedBackendRef(container.imageRef));
  if (directBackend) return directBackend;

  // A hardened compose may publish the port through nginx while the memory layer
  // stays on an internal network. Follow the publisher's compose project to
  // the service whose role is the backend instead of comparing nginx to GHCR.
  for (const publisher of publishers) {
    if (!publisher.composeProject) continue;
    const backendIds = containerIds(docker([
      "ps",
      "--filter", `label=com.docker.compose.project=${publisher.composeProject}`,
      "--filter", "label=com.docker.compose.service=memory-layer",
      "--format", "{{.ID}}",
    ]));
    for (const id of backendIds) {
      const backend = inspectContainer(id);
      if (backend) return backend;
    }
  }

  // Preserve local-development detection for a directly-published backend
  // whose image does not use the released repository name.
  return publishers[0] ?? null;
}

/**
 * A container in the backend stack that is down, with whatever it said on the
 * way out.
 *
 * Exists because a fatal Arango boot loop is invisible from the CLI: the
 * failure is inside a container that keeps restarting, so every command --
 * `ix doctor` included -- sees nothing but a backend that will not answer, and
 * "backend not started" is indistinguishable from "backend cannot start and
 * never will". Ix#614.
 */
export interface StackFailure {
  /** Compose service name where known, else the image reference. */
  service: string;
  containerId: string;
  /** Docker's own state string: `restarting`, `exited`, ... */
  state: string;
  /** The most recent fatal-looking log line, trimmed. */
  lastError: string | null;
  /** Set only for a failure whose remedy we actually know. */
  remedy: string | null;
}

/** `docker logs` with stdout and stderr merged; "" on any failure. */
function dockerLogsMerged(containerId: string, tail = "80"): string {
  const r = spawnSync("docker", ["logs", "--tail", tail, containerId], {
    encoding: "utf-8",
    timeout: 10000,
  });
  return `${r.stdout ?? ""}\n${r.stderr ?? ""}`;
}

/** Last line that looks like a hard failure, searching newest-first. */
function lastFatalLine(logs: string): string | null {
  const lines = logs.split("\n").map((l) => l.trim()).filter(Boolean);
  for (let i = lines.length - 1; i >= 0; i--) {
    if (/\bFATAL\b|\bunable to initialize\b|\bInvalid argument\b/i.test(lines[i]!)) return lines[i]!;
  }
  return lines.length > 0 ? lines[lines.length - 1]! : null;
}

/**
 * Recognise failures whose fix is known, so the user gets an instruction and
 * not just a transcript.
 */
function remedyFor(line: string | null): string | null {
  if (!line) return null;
  if (/Column families not opened:\s*VectorIndex/i.test(line)) {
    // One-way door: the column family is in the RocksDB MANIFEST for good, so
    // the only non-destructive fix is to keep registering it. Ix#614.
    return "ArangoDB's data directory has the VectorIndex column family, so arangod must be started with it enabled. " +
      "Add `--vector-index true` to the arangodb `command:` in ~/.ix/backend/docker-compose.yml, then `ix docker restart`. " +
      "(The option is spelled `--experimental-vector-index` before ArangoDB 3.12.11.) No data is lost.";
  }
  return null;
}

/**
 * Look for a container in the backend stack that is not running.
 *
 * Deliberately image-based rather than compose-label-based: the label is absent
 * for a hand-run container, and this is called precisely when the stack is in a
 * state nobody planned. Returns the highest-ranked non-running Arango -- an
 * actively restarting, compose-managed one first, since that is what holds
 * memory-layer's `service_healthy` gate shut.
 */
export function diagnoseBackendStack(): StackFailure | null {
  const SEP = "|::|";
  const listed = docker(["ps", "-a", "--format", `{{.ID}}${SEP}{{.Image}}${SEP}{{.State}}${SEP}{{.Label "com.docker.compose.service"}}`]);
  if (!listed) return null;

  // Rank before picking. A boot LOOP presents as `restarting`, and that is the
  // one holding memory-layer's `service_healthy` gate shut; a long-dead
  // `exited` container from an old experiment is the likeliest false positive,
  // and reporting its error as the current outage would be worse than saying
  // nothing. A compose-managed container outranks a hand-run one for the same
  // reason.
  const rank = (state: string, service: string): number =>
    (state === "restarting" ? 0 : 2) + (service ? 0 : 1);

  const candidates = listed.split("\n")
    .map((row) => {
      const [id = "", image = "", state = "", service = ""] = row.split(SEP);
      return { id, image, state, service };
    })
    .filter((c) => c.id && /arangodb/i.test(c.image) && c.state !== "running")
    .sort((a, b) => rank(a.state, a.service) - rank(b.state, b.service));

  for (const { id, image, state, service } of candidates) {
    // --tail keeps this bounded; a boot loop can produce a very large log.
    // Both streams: `docker logs` keeps the container's stdout/stderr apart,
    // and which one carries the fatal line is the container's choice, not
    // ours. The arangodb image logs it on stdout; a config that sends it to
    // stderr would otherwise make this whole diagnostic silently find nothing.
    const logs = dockerLogsMerged(id);
    const lastError = lastFatalLine(logs);
    return {
      service: service || image,
      containerId: id,
      state: state || "unknown",
      lastError,
      remedy: remedyFor(lastError),
    };
  }
  return null;
}

export type BackendImageStatus =
  | { kind: "ok"; container: BackendContainer }
  | { kind: "local-build"; container: BackendContainer }
  | { kind: "digest-mismatch"; container: BackendContainer; latestImageId: string }
  | { kind: "latest-not-pulled"; container: BackendContainer }
  | { kind: "not-running" }
  | { kind: "remote" }
  | { kind: "docker-unavailable" };

/**
 * Compare the running backend container against the locally-pulled released
 * `:latest` image. Conclusions only when we can prove a mismatch; an
 * inconclusive state (latest not pulled, docker down) never reports a problem.
 */
export function checkBackendImage(endpoint: string = getEndpoint()): BackendImageStatus {
  // A remote endpoint is served by no container on this machine.
  if (!backendPortFor(endpoint)) return { kind: "remote" };
  if (!dockerAvailable()) return { kind: "docker-unavailable" };
  const container = inspectBackendContainer(endpoint);
  if (!container) return { kind: "not-running" };

  const latestImageId = docker(["image", "inspect", `${BACKEND_IMAGE}:latest`, "--format", "{{.Id}}"]);
  if (!latestImageId) return { kind: "latest-not-pulled", container };

  if (container.imageId === latestImageId) return { kind: "ok", container };

  // Different from the released image. A container with no registry digests (or
  // one not built from the released repo) is a local build; otherwise it is an
  // older/divergent pulled digest.
  const isReleasedRepo = container.repoDigests.some((d) => d.startsWith(BACKEND_IMAGE + "@"));
  if (container.repoDigests.length === 0 || !isReleasedRepo) {
    return { kind: "local-build", container };
  }
  return { kind: "digest-mismatch", container, latestImageId };
}

/** True when the running backend uses a compose project outside ~/.ix/backend. */
export function isNonStandardBackend(container: BackendContainer): boolean {
  const cfg = container.composeConfigFiles;
  if (!cfg) return false;
  const standardBackendDir = join(ixHome(), "backend");
  return !cfg.split(",").some((p) => p.trim().startsWith(standardBackendDir));
}

export interface BackendSchemaStatus {
  reachable: boolean;
  serverVersion: number | null;
  expected: number;
  matches: boolean;
}

/**
 * Read the backend's reported schema_version and compare to what we expect.
 *
 * Also RECORDS the release the backend reports, as a side effect of going
 * through the shared health fetch — the same as every other health call site.
 */
export async function checkBackendSchema(
  client: IxClient,
  // REQUIRED, not defaulted. Defaults of `null` + `() => false` turn off the
  // ceiling AND the no-ceiling fallback at once, so a caller that omitted them
  // would record any version-shaped claim a local container made — the exact
  // 99.0.0 case recordBackendRelease exists to refuse. This is a published
  // export, so "a call site can get it wrong" has to be impossible, not
  // discouraged.
  knownLatest: string | null,
  isNewer: (a: string, b: string) => boolean,
): Promise<BackendSchemaStatus> {
  const expected = CLIENT_EXPECTED_SCHEMA_VERSION;
  try {
    // Through the chokepoint, so this records like every other health fetch —
    // there is no cycle: backend-version.ts imports only node builtins and two
    // erased `import type`s.
    const health = await fetchBackendHealth(client, knownLatest, isNewer);
    const serverVersion = typeof health.schema_version === "number" ? health.schema_version : null;
    // No reported version (older backend) is treated as a match: we can't prove
    // staleness, and the ingest path already forces a re-ingest when it can.
    const matches = serverVersion === null || serverVersion === expected;
    return { reachable: true, serverVersion, expected, matches };
  } catch {
    return { reachable: false, serverVersion: null, expected, matches: true };
  }
}