Skip to content

A desktop coding agent with free access to frontier models.

TypeScript
44.4 KB
Raw Blame
/**
 * Kubernetes Den-stack harness for eval environments.
 *
 * This is the kind-backed Den placement: the same eval scenarios target the
 * same OMNIRUSH_EVAL_DEN_* URLs, but the control plane runs through the Helm
 * chart in a local Kubernetes cluster. Endpoints are exposed with kubectl
 * port-forward instead of kind node port mappings so an existing warm cluster
 * can be reused across profiles without recreating the node config.
 */
import { spawn } from "node:child_process";
import { openSync } from "node:fs";
import { access, mkdir, readFile, rm, writeFile } from "node:fs/promises";
import { dirname, join, resolve } from "node:path";
import { fileURLToPath } from "node:url";
import { resolveChromeBinary } from "@omnirush/hosts";

const MODULE_DIR = dirname(fileURLToPath(import.meta.url));
const REPO_ROOT = resolve(MODULE_DIR, "..", "..", "..", "..");
const DEFAULT_STATE_DIR = resolve(MODULE_DIR, "..", "..", "..", "results", ".kube-stack");
const DEFAULT_ELECTRON_USERDATA = process.env.OMNIRUSH_EVAL_KUBE_ELECTRON_USERDATA?.trim()
  || join(DEFAULT_STATE_DIR, "electron-user-data");

export const KUBE_CLUSTER_NAME = "omnirush-kube-lab";
export const KUBE_CONTEXT = `kind-${KUBE_CLUSTER_NAME}`;
export const KUBE_RELEASE_NAME = "omnirush-ee";
export const KUBE_NAMESPACE = "default";
export const KUBE_CHART_PATH = "packaging/helm/omnirush-ee";
const KUBE_FIXTURE_DIR = "evals/fixtures/kube";
const KUBE_MYSQL_MANIFEST = `${KUBE_FIXTURE_DIR}/mysql.yaml`;
const KUBE_EGRESS_KIND_CONFIG = `${KUBE_FIXTURE_DIR}/kind-config-egress.yaml`;
const KUBE_NETPOL_DIR = `${KUBE_FIXTURE_DIR}/netpol`;
const KUBE_DEFAULT_DENY_EGRESS_MANIFEST = `${KUBE_NETPOL_DIR}/default-deny-egress.yaml`;
const KUBE_ALLOW_CORE_EGRESS_MANIFEST = `${KUBE_NETPOL_DIR}/allow-core.yaml`;
const KUBE_ALLOW_EXTERNAL_EGRESS_TEMPLATE = `${KUBE_NETPOL_DIR}/allow-external.template.yaml`;
const CALICO_MANIFEST_URL = "https://raw.githubusercontent.com/projectcalico/calico/v3.28.2/manifests/calico.yaml";
const DEN_API_SERVICE = `${KUBE_RELEASE_NAME}-den-api`;
const DEN_WEB_SERVICE = `${KUBE_RELEASE_NAME}-den-web`;
const MYSQL_DEPLOYMENT = "omnirush-mysql";
const DEN_API_PORT = Number(process.env.OMNIRUSH_EVAL_DEN_PORT ?? 8790);
const DEN_WEB_PORT = Number(process.env.OMNIRUSH_EVAL_DEN_WEB_PORT ?? 3005);
const DEN_API_URL = `http://127.0.0.1:${DEN_API_PORT}`;
const DEN_WEB_URL = `http://127.0.0.1:${DEN_WEB_PORT}`;
const DEN_BASE_URL = `http://localhost:${DEN_API_PORT}`;
const DEMO_EMAIL = process.env.DEN_DEMO_OWNER_EMAIL ?? "[email protected]";
export const DEMO_PASSWORD = process.env.DEN_DEMO_OWNER_PASSWORD ?? "OmniRushDemo123!";
const LOCAL_IMAGE_TAG = process.env.OMNIRUSH_EVAL_KUBE_LOCAL_IMAGE_TAG?.trim() || "kube-lab";
const PUBLISHED_IMAGE_TAG = process.env.OMNIRUSH_EVAL_KUBE_IMAGE_TAG?.trim() || "latest";
const PUBLISHED_DEN_API_REPOSITORY = "ghcr.io/omnirush-ai/omnirush-den-api";
const PUBLISHED_DEN_WEB_REPOSITORY = "ghcr.io/omnirush-ai/omnirush-den-web";
const LOCAL_DEN_API_REPOSITORY = "omnirush-den-api";
const LOCAL_DEN_WEB_REPOSITORY = "omnirush-den-web";

type DenOrgMode = "single_org" | "multi_org";
export type KubeProfile = "single-org" | "multi-org";
export type KubeImageMode = "published" | "local";
export type KubeEgressMode = "allowlist";

export interface KubeExecResult {
  stdout: string;
  stderr: string;
  code: number;
}

export interface KubeExecOptions {
  input?: string;
  timeoutMs?: number;
  cwd?: string;
  env?: NodeJS.ProcessEnv;
}

export type KubeExec = (
  command: string,
  args: string[],
  options?: KubeExecOptions,
) => Promise<KubeExecResult>;

export interface KubeSpawnOptions {
  logName: string;
  cwd?: string;
  env?: NodeJS.ProcessEnv;
  stateDir: string;
}

export type KubeSpawnDetached = (
  command: string,
  args: string[],
  options: KubeSpawnOptions,
) => number;

export interface KubeProfileConfig {
  profile: KubeProfile;
  orgMode: DenOrgMode;
  valuesPath: string;
}

export interface KubeImagePlan {
  mode: KubeImageMode;
  denApiRepository: string;
  denWebRepository: string;
  tag: string;
  pullPolicy: string;
  reason: string;
}

interface KubePlatform {
  os: string;
  architecture: string;
}

interface KubeRuntime {
  run: KubeExec;
  spawnDetached: KubeSpawnDetached;
  stateDir: string;
  log(message: string): void;
  killProcess(pid: number, signal?: NodeJS.Signals): void;
  sleep(ms: number): Promise<void>;
}

interface KubeLayerOptions {
  exec?: KubeExec;
  spawnDetached?: KubeSpawnDetached;
  stateDir?: string;
  log?: (message: string) => void;
  killProcess?: (pid: number, signal?: NodeJS.Signals) => void;
  sleep?: (ms: number) => Promise<void>;
}

export interface EnsureKubeStackOptions extends KubeLayerOptions {
  cdpCandidates: string[];
  skipApp?: boolean;
  profile?: KubeProfile;
  images?: KubeImageMode;
  egress?: KubeEgressMode;
}

export interface KubeStackDownOptions extends KubeLayerOptions {
  deleteCluster?: boolean;
}

export interface KindEndpoints {
  apiUrl: string;
  webUrl: string;
  token: string;
  adminEmail: string;
  stop(): Promise<void>;
}

function defaultExec(command: string, args: string[], options: KubeExecOptions = {}): Promise<KubeExecResult> {
  return new Promise((resolveExec, reject) => {
    const child = spawn(command, args, {
      cwd: options.cwd ?? REPO_ROOT,
      env: { ...process.env, ...options.env },
      stdio: ["pipe", "pipe", "pipe"],
    });
    let stdout = "";
    let stderr = "";
    let timedOut = false;
    const timer = options.timeoutMs
      ? globalThis.setTimeout(() => {
        timedOut = true;
        child.kill("SIGTERM");
      }, options.timeoutMs)
      : null;

    child.stdout.setEncoding("utf8");
    child.stderr.setEncoding("utf8");
    child.stdout.on("data", (chunk) => {
      stdout += String(chunk);
    });
    child.stderr.on("data", (chunk) => {
      stderr += String(chunk);
    });
    child.on("error", (error) => {
      if (timer) clearTimeout(timer);
      reject(error);
    });
    child.on("close", (code) => {
      if (timer) clearTimeout(timer);
      resolveExec({
        stdout,
        stderr: timedOut ? `${stderr}\nTimed out after ${options.timeoutMs}ms.` : stderr,
        code: timedOut ? 124 : code ?? 1,
      });
    });
    child.stdin.end(options.input ?? "");
  });
}

function defaultSpawnDetached(command: string, args: string[], options: KubeSpawnOptions): number {
  const logFd = openSync(join(options.stateDir, `${options.logName}.log`), "a");
  const child = spawn(command, args, {
    cwd: options.cwd ?? REPO_ROOT,
    env: { ...process.env, ...options.env },
    detached: true,
    stdio: ["ignore", logFd, logFd],
  });
  child.unref();
  if (!child.pid) throw new Error(`Could not spawn ${command}.`);
  return child.pid;
}

function createRuntime(options: KubeLayerOptions = {}): KubeRuntime {
  return {
    run: options.exec ?? defaultExec,
    spawnDetached: options.spawnDetached ?? defaultSpawnDetached,
    stateDir: options.stateDir ?? DEFAULT_STATE_DIR,
    log: options.log ?? (() => undefined),
    killProcess: options.killProcess ?? ((pid, signal) => process.kill(pid, signal)),
    sleep: options.sleep ?? ((ms) => new Promise((resolveSleep) => setTimeout(resolveSleep, ms))),
  };
}

function isRecord(value: unknown): value is Record<string, unknown> {
  return typeof value === "object" && value !== null && !Array.isArray(value);
}

function errorText(error: unknown): string {
  return error instanceof Error ? error.message : String(error);
}

function formatDuration(ms: number): string {
  return `${(ms / 1000).toFixed(1)}s`;
}

async function phase<T>(runtime: KubeRuntime, name: string, action: () => Promise<T>): Promise<T> {
  const startedAt = Date.now();
  runtime.log(`kube:${name} starting`);
  try {
    const result = await action();
    runtime.log(`kube:${name} finished in ${formatDuration(Date.now() - startedAt)}`);
    return result;
  } catch (error) {
    runtime.log(`kube:${name} failed after ${formatDuration(Date.now() - startedAt)}`);
    throw error;
  }
}

async function checkedExec(runtime: KubeRuntime, command: string, args: string[], context: string, options: KubeExecOptions = {}): Promise<KubeExecResult> {
  const result = await runtime.run(command, args, options);
  if (result.code !== 0) {
    const detail = [result.stderr.trim(), result.stdout.trim()].filter(Boolean).join("\n");
    throw new Error(`${context} failed with exit ${result.code}${detail ? `:\n${detail}` : ""}`);
  }
  return result;
}

function pidStatePath(runtime: KubeRuntime, name: string): string {
  return join(runtime.stateDir, name);
}

async function writePidState(runtime: KubeRuntime, name: string, value: unknown): Promise<void> {
  await mkdir(runtime.stateDir, { recursive: true });
  await writeFile(pidStatePath(runtime, name), String(value));
}

async function readPidState(runtime: KubeRuntime, name: string): Promise<string | null> {
  try {
    return (await readFile(pidStatePath(runtime, name), "utf8")).trim();
  } catch {
    return null;
  }
}

function processIsAlive(pid: number): boolean {
  try {
    process.kill(pid, 0);
    return true;
  } catch (error) {
    return isRecord(error) && error.code === "EPERM";
  }
}

async function waitForPidGone(pid: number, runtime: KubeRuntime, timeoutMs: number): Promise<boolean> {
  const startedAt = Date.now();
  while (Date.now() - startedAt < timeoutMs) {
    if (!processIsAlive(pid)) return true;
    await runtime.sleep(250);
  }
  return !processIsAlive(pid);
}

async function stopProcess(runtime: KubeRuntime, pid: number, label: string): Promise<void> {
  try {
    runtime.killProcess(-pid, "SIGINT");
  } catch {
    // Process group may already be gone or unsupported.
  }
  try {
    runtime.killProcess(pid, "SIGINT");
    runtime.log(`Stopped ${label} (pid ${pid})`);
  } catch {
    // Already gone.
  }
  if (await waitForPidGone(pid, runtime, 5_000)) return;
  try {
    runtime.killProcess(-pid, "SIGKILL");
  } catch {
    // Already gone.
  }
  try {
    runtime.killProcess(pid, "SIGKILL");
  } catch {
    // Already gone.
  }
  await waitForPidGone(pid, runtime, 1_000);
}

async function stopRecordedProcess(runtime: KubeRuntime, stateName: string, label: string): Promise<void> {
  const rawPid = await readPidState(runtime, stateName);
  if (!rawPid) return;
  const pid = Number(rawPid);
  if (!Number.isInteger(pid) || pid <= 0) return;
  await stopProcess(runtime, pid, label);
}

async function httpOk(url: string, timeoutMs = 2_500): Promise<boolean> {
  try {
    const response = await fetch(url, { signal: AbortSignal.timeout(timeoutMs) });
    return response.ok;
  } catch {
    return false;
  }
}

async function demoOwnerTokenIsValid(token: string): Promise<boolean> {
  try {
    const response = await fetch(`${DEN_API_URL}/v1/me`, {
      headers: { authorization: `Bearer ${token}` },
      signal: AbortSignal.timeout(2_500),
    });
    return response.ok;
  } catch {
    return false;
  }
}

async function signInDemoOwner(runtime: KubeRuntime): Promise<{ token: string; email: string }> {
  const existingToken = await mysqlQuery(
    runtime,
    `SELECT s.token FROM \`session\` s JOIN \`user\` u ON u.id=s.user_id WHERE u.email='${DEMO_EMAIL.toLowerCase()}' AND s.expires_at > UTC_TIMESTAMP(3) ORDER BY s.expires_at DESC LIMIT 1;`,
  );
  if (existingToken && await demoOwnerTokenIsValid(existingToken)) {
    runtime.log(`Reusing a valid demo-owner session (${DEMO_EMAIL})`);
    return { token: existingToken, email: DEMO_EMAIL };
  }

  try {
    const response = await fetch(`${DEN_API_URL}/api/auth/sign-in/email`, {
      method: "POST",
      headers: {
        "content-type": "application/json",
        origin: DEN_BASE_URL,
        "x-forwarded-for": "127.0.0.1",
      },
      body: JSON.stringify({ email: DEMO_EMAIL, password: DEMO_PASSWORD }),
    });
    if (!response.ok) {
      throw new Error(`HTTP ${response.status}: ${(await response.text()).slice(0, 500)}`);
    }
    const payload: unknown = await response.json();
    if (!isRecord(payload) || typeof payload.token !== "string" || !payload.token) {
      throw new Error("the response did not contain a session token");
    }
    return { token: payload.token, email: DEMO_EMAIL };
  } catch (error) {
    throw new Error(`Could not obtain a demo-owner session from the kube Den API: ${errorText(error)}`);
  }
}

export function kubeProfileConfig(profile: KubeProfile): KubeProfileConfig {
  if (profile === "single-org") {
    return {
      profile,
      orgMode: "single_org",
      valuesPath: `${KUBE_FIXTURE_DIR}/values/single-org.yaml`,
    };
  }
  return {
    profile,
    orgMode: "multi_org",
    valuesPath: `${KUBE_FIXTURE_DIR}/values/multi-org.yaml`,
  };
}

function parseImageMode(value: string | undefined): KubeImageMode | null {
  const trimmed = value?.trim();
  if (!trimmed) return null;
  if (trimmed === "published" || trimmed === "local") return trimmed;
  throw new Error(`Unknown kube image mode: ${trimmed}. Supported: published, local.`);
}

function dockerArchitecture(): string {
  if (process.arch === "x64") return "amd64";
  return process.arch;
}

export function currentDockerPlatform(): KubePlatform {
  return { os: "linux", architecture: dockerArchitecture() };
}

export function manifestSupportsPlatform(manifest: unknown, platform: KubePlatform): boolean {
  if (!isRecord(manifest)) return false;
  if (typeof manifest.os === "string" && typeof manifest.architecture === "string") {
    return manifest.os === platform.os && manifest.architecture === platform.architecture;
  }
  if (!Array.isArray(manifest.manifests)) return false;
  return manifest.manifests.some((entry) => {
    if (!isRecord(entry) || !isRecord(entry.platform)) return false;
    return entry.platform.os === platform.os && entry.platform.architecture === platform.architecture;
  });
}

function publishedImagePlan(reason: string): KubeImagePlan {
  return {
    mode: "published",
    denApiRepository: PUBLISHED_DEN_API_REPOSITORY,
    denWebRepository: PUBLISHED_DEN_WEB_REPOSITORY,
    tag: PUBLISHED_IMAGE_TAG,
    pullPolicy: "IfNotPresent",
    reason,
  };
}

function localImagePlan(reason: string): KubeImagePlan {
  return {
    mode: "local",
    denApiRepository: LOCAL_DEN_API_REPOSITORY,
    denWebRepository: LOCAL_DEN_WEB_REPOSITORY,
    tag: LOCAL_IMAGE_TAG,
    pullPolicy: "IfNotPresent",
    reason,
  };
}

async function inspectManifest(runtime: KubeRuntime, image: string): Promise<unknown> {
  const result = await checkedExec(runtime, "docker", ["manifest", "inspect", image], `docker manifest inspect ${image}`, { timeoutMs: 120_000 });
  const parsed: unknown = JSON.parse(result.stdout);
  return parsed;
}

async function publishedImagesSupportPlatform(runtime: KubeRuntime, platform: KubePlatform): Promise<boolean> {
  const apiManifest = await inspectManifest(runtime, `${PUBLISHED_DEN_API_REPOSITORY}:${PUBLISHED_IMAGE_TAG}`);
  const webManifest = await inspectManifest(runtime, `${PUBLISHED_DEN_WEB_REPOSITORY}:${PUBLISHED_IMAGE_TAG}`);
  return manifestSupportsPlatform(apiManifest, platform) && manifestSupportsPlatform(webManifest, platform);
}

export async function resolveKubeImagePlan(options: { exec?: KubeExec; images?: KubeImageMode; log?: (message: string) => void } = {}): Promise<KubeImagePlan> {
  const runtime = createRuntime({ exec: options.exec, log: options.log });
  const envMode = parseImageMode(process.env.OMNIRUSH_EVAL_KUBE_IMAGES);
  const requested = options.images ?? envMode;
  const platform = currentDockerPlatform();
  if (requested === "local") {
    return localImagePlan("--images local selected; building and loading local images into kind");
  }
  if (requested === "published") {
    if (!(await publishedImagesSupportPlatform(runtime, platform))) {
      throw new Error(`Published Den images do not advertise ${platform.os}/${platform.architecture}; rerun with --images local.`);
    }
    return publishedImagePlan(`--images published selected; ghcr images support ${platform.os}/${platform.architecture}`);
  }

  try {
    if (await publishedImagesSupportPlatform(runtime, platform)) {
      return publishedImagePlan(`auto-selected published images; ghcr manifests include ${platform.os}/${platform.architecture}`);
    }
    return localImagePlan(`auto-selected local images; ghcr manifests do not include ${platform.os}/${platform.architecture}`);
  } catch (error) {
    runtime.log(`Could not inspect published image manifests; falling back to local images: ${errorText(error)}`);
    return localImagePlan("auto-selected local images because published manifest inspection failed");
  }
}

export function helmImageSetArgs(plan: KubeImagePlan): string[] {
  const args = [
    "--set", `image.tag=${plan.tag}`,
    "--set", `image.pullPolicy=${plan.pullPolicy}`,
  ];
  if (plan.mode === "local") {
    args.push(
      "--set", `denApi.image.repository=${plan.denApiRepository}`,
      "--set", `denWeb.image.repository=${plan.denWebRepository}`,
    );
  }
  return args;
}

export function helmUpgradeArgs(profile: KubeProfileConfig, plan: KubeImagePlan, egress?: KubeEgressMode): string[] {
  const args = [
    "upgrade",
    "--install",
    KUBE_RELEASE_NAME,
    KUBE_CHART_PATH,
    "-f",
    profile.valuesPath,
    ...helmImageSetArgs(plan),
    "--namespace",
    KUBE_NAMESPACE,
    "--create-namespace",
    "--kube-context",
    KUBE_CONTEXT,
    "--timeout",
    "10m",
  ];
  if (egress === "allowlist") {
    args.push(
      "--set", "config.omnirushDevMode=0",
      "--set", "config.public.allowPrivateMcpUrls=1",
    );
  }
  return args;
}

export function helmTestArgs(): string[] {
  return ["test", KUBE_RELEASE_NAME, "--namespace", KUBE_NAMESPACE, "--kube-context", KUBE_CONTEXT, "--timeout", "5m"];
}

export function kubectlArgs(args: string[]): string[] {
  return ["--context", KUBE_CONTEXT, ...args];
}

export function rolloutStatusArgs(deployment: string, timeout: string): string[] {
  return kubectlArgs(["rollout", "status", `deployment/${deployment}`, `--timeout=${timeout}`]);
}

export function portForwardArgs(service: string, localPort: number, remotePort: number): string[] {
  return kubectlArgs(["port-forward", `service/${service}`, `${localPort}:${remotePort}`]);
}

async function collectTroubleshooting(runtime: KubeRuntime, labelSelector = `app.kubernetes.io/instance=${KUBE_RELEASE_NAME}`): Promise<string> {
  const probes = [
    { label: "pod status", args: kubectlArgs(["get", "pods", "-l", labelSelector, "-o", "wide"]) },
    { label: "pod descriptions", args: kubectlArgs(["describe", "pods", "-l", labelSelector]) },
    { label: "recent pod logs", args: kubectlArgs(["logs", "-l", labelSelector, "--all-containers", "--tail=160", "--prefix"]) },
  ];
  const sections: string[] = [];
  for (const probe of probes) {
    const result = await runtime.run("kubectl", probe.args, { timeoutMs: 90_000 });
    const body = [result.stdout.trim(), result.stderr.trim()].filter(Boolean).join("\n");
    sections.push(`## ${probe.label}\n$ kubectl ${probe.args.join(" ")}\n${body || `(exit ${result.code}, no output)`}`);
  }
  return sections.join("\n\n");
}

async function ensureRollout(runtime: KubeRuntime, deployment: string, timeout: string): Promise<void> {
  const args = rolloutStatusArgs(deployment, timeout);
  const result = await runtime.run("kubectl", args, { timeoutMs: 420_000 });
  if (result.code === 0) return;
  const diagnostics = await collectTroubleshooting(runtime);
  const detail = [result.stderr.trim(), result.stdout.trim(), diagnostics].filter(Boolean).join("\n\n");
  throw new Error(`kubectl rollout status deployment/${deployment} failed with exit ${result.code}:\n${detail}`);
}

async function waitForCalico(runtime: KubeRuntime): Promise<void> {
  await checkedExec(
    runtime,
    "kubectl",
    kubectlArgs(["rollout", "status", "daemonset/calico-node", "--namespace", "kube-system", "--timeout=300s"]),
    "waiting for calico-node DaemonSet",
    { timeoutMs: 360_000 },
  );
  await checkedExec(
    runtime,
    "kubectl",
    kubectlArgs(["wait", "--for=condition=Ready", "nodes", "--all", "--timeout=300s"]),
    "waiting for kind nodes to become Ready",
    { timeoutMs: 360_000 },
  );
}

async function kindClusterExists(runtime: KubeRuntime): Promise<boolean> {
  const clusters = await runtime.run("kind", ["get", "clusters"], { timeoutMs: 60_000 });
  return clusters.code === 0
    && clusters.stdout.split(/\r?\n/).map((line) => line.trim()).includes(KUBE_CLUSTER_NAME);
}

export async function ensureCluster(options: KubeLayerOptions & { egress?: KubeEgressMode } = {}): Promise<void> {
  const runtime = createRuntime(options);
  const exists = await kindClusterExists(runtime);
  if (exists) {
    if (options.egress === "allowlist") {
      const calico = await runtime.run(
        "kubectl",
        kubectlArgs(["get", "daemonset/calico-node", "--namespace", "kube-system"]),
        { timeoutMs: 60_000 },
      );
      if (calico.code !== 0) {
        throw new Error(
          `Existing kind cluster ${KUBE_CLUSTER_NAME} does not have Calico, so it cannot enforce --kube-egress allowlist. Run --stack-down --delete-cluster first.`,
        );
      }
    }
    runtime.log(`Reusing kind cluster ${KUBE_CLUSTER_NAME}`);
  } else {
    runtime.log(`Creating kind cluster ${KUBE_CLUSTER_NAME} (endpoints use kubectl port-forward)`);
    const args = ["create", "cluster", "--name", KUBE_CLUSTER_NAME];
    if (options.egress === "allowlist") args.push("--config", KUBE_EGRESS_KIND_CONFIG);
    await checkedExec(runtime, "kind", args, `kind create cluster --name ${KUBE_CLUSTER_NAME}`, { timeoutMs: 300_000 });
  }
  await checkedExec(runtime, "kubectl", kubectlArgs(["cluster-info"]), `kubectl cluster-info --context ${KUBE_CONTEXT}`, { timeoutMs: 60_000 });
  if (options.egress === "allowlist") {
    if (!exists) {
      runtime.log("Installing Calico v3.28.2");
      await checkedExec(
        runtime,
        "kubectl",
        kubectlArgs(["create", "-f", CALICO_MANIFEST_URL]),
        "installing Calico",
        { timeoutMs: 180_000 },
      );
    }
    await waitForCalico(runtime);
  }
}

const PREPARE_KIND_STACK_COMMAND = "ensureKubeStack({ cdpCandidates: [], skipApp: true })";

function missingKindStackError(subject: string): Error {
  return new Error(`${subject}. Prepare the shared kind stack first with ${PREPARE_KIND_STACK_COMMAND}.`);
}

async function requireKindStack(runtime: KubeRuntime): Promise<void> {
  if (!(await kindClusterExists(runtime))) {
    throw missingKindStackError(`Kind cluster ${KUBE_CLUSTER_NAME} is not running`);
  }
  const release = await runtime.run("helm", [
    "status",
    KUBE_RELEASE_NAME,
    "--namespace",
    KUBE_NAMESPACE,
    "--kube-context",
    KUBE_CONTEXT,
  ], { timeoutMs: 60_000 });
  if (release.code !== 0) {
    throw missingKindStackError(`Helm release ${KUBE_RELEASE_NAME} is not installed in ${KUBE_CONTEXT}`);
  }
}

export async function ensureImages(plan: KubeImagePlan, options: KubeLayerOptions = {}): Promise<void> {
  const runtime = createRuntime(options);
  runtime.log(`Image mode: ${plan.mode} (${plan.reason})`);
  if (plan.mode === "published") return;

  const apiImage = `${plan.denApiRepository}:${plan.tag}`;
  const webImage = `${plan.denWebRepository}:${plan.tag}`;
  await checkedExec(runtime, "docker", ["build", "-f", "packaging/docker/Dockerfile.den", "-t", apiImage, "."], `docker build ${apiImage}`, { timeoutMs: 20 * 60_000 });
  await checkedExec(runtime, "docker", ["build", "-f", "packaging/docker/Dockerfile.den-web", "-t", webImage, "."], `docker build ${webImage}`, { timeoutMs: 20 * 60_000 });
  await checkedExec(runtime, "kind", ["load", "docker-image", apiImage, "--name", KUBE_CLUSTER_NAME], `kind load docker-image ${apiImage}`, { timeoutMs: 300_000 });
  await checkedExec(runtime, "kind", ["load", "docker-image", webImage, "--name", KUBE_CLUSTER_NAME], `kind load docker-image ${webImage}`, { timeoutMs: 300_000 });
}

export async function ensureDatabase(options: KubeLayerOptions = {}): Promise<void> {
  const runtime = createRuntime(options);
  await checkedExec(runtime, "kubectl", kubectlArgs(["apply", "-f", KUBE_MYSQL_MANIFEST]), `kubectl apply -f ${KUBE_MYSQL_MANIFEST}`, { timeoutMs: 60_000 });
  await ensureRollout(runtime, MYSQL_DEPLOYMENT, "180s");
}

export async function ensureRelease(profile: KubeProfileConfig, plan: KubeImagePlan, options: KubeLayerOptions & { egress?: KubeEgressMode } = {}): Promise<void> {
  const runtime = createRuntime(options);
  const helmArgs = helmUpgradeArgs(profile, plan, options.egress);
  const result = await runtime.run("helm", helmArgs, { timeoutMs: 12 * 60_000 });
  if (result.code !== 0) {
    const diagnostics = await collectTroubleshooting(runtime);
    const detail = [result.stderr.trim(), result.stdout.trim(), diagnostics].filter(Boolean).join("\n\n");
    throw new Error(`helm ${helmArgs.join(" ")} failed with exit ${result.code}:\n${detail}`);
  }
  await ensureRollout(runtime, `${KUBE_RELEASE_NAME}-den-api`, "300s");
  await ensureRollout(runtime, `${KUBE_RELEASE_NAME}-den-web`, "300s");
}

function parseIpv4(output: string): string | null {
  for (const token of output.split(/\s+/)) {
    const parts = token.split(".");
    if (parts.length !== 4) continue;
    if (parts.every((part) => /^\d{1,3}$/.test(part) && Number(part) <= 255)) return token;
  }
  return null;
}

async function resolveAllowedHostIp(runtime: KubeRuntime): Promise<string> {
  const nodeLookup = await runtime.run("docker", [
    "exec",
    `${KUBE_CLUSTER_NAME}-control-plane`,
    "sh",
    "-lc",
    "getent hosts host.docker.internal | awk '{print $1}'",
  ], { timeoutMs: 30_000 });
  const nodeIp = nodeLookup.code === 0 ? parseIpv4(nodeLookup.stdout) : null;
  if (nodeIp) return nodeIp;

  const gateway = await checkedExec(
    runtime,
    "docker",
    ["network", "inspect", "kind", "--format", "{{range .IPAM.Config}}{{println .Gateway}}{{end}}"],
    "resolving the kind Docker network gateway",
    { timeoutMs: 30_000 },
  );
  const gatewayIp = parseIpv4(gateway.stdout);
  if (!gatewayIp) throw new Error(`Could not resolve an IPv4 host gateway for kind. docker network inspect returned: ${gateway.stdout.trim() || "(empty)"}`);
  return gatewayIp;
}

function mockPort(envName: string, defaultPort: number): number {
  const raw = process.env[envName]?.trim();
  if (!raw) return defaultPort;
  if (!/^\d+$/.test(raw)) throw new Error(`${envName} must be an integer port, got ${raw}.`);
  const port = Number(raw);
  if (!Number.isInteger(port) || port < 1 || port > 65_535) throw new Error(`${envName} must be between 1 and 65535, got ${raw}.`);
  return port;
}

async function ensureHostMockMasquerade(runtime: KubeRuntime, hostIp: string, port: number): Promise<void> {
  const rule = [
    "-t", "nat",
    "-C", "POSTROUTING",
    "-s", "192.168.0.0/16",
    "-d", `${hostIp}/32`,
    "-p", "tcp",
    "--dport", String(port),
    "-j", "MASQUERADE",
  ];
  const check = await runtime.run(
    "docker",
    ["exec", `${KUBE_CLUSTER_NAME}-control-plane`, "iptables", ...rule],
    { timeoutMs: 30_000 },
  );
  if (check.code === 0) return;
  rule[2] = "-I";
  rule.splice(4, 0, "1");
  await checkedExec(
    runtime,
    "docker",
    ["exec", `${KUBE_CLUSTER_NAME}-control-plane`, "iptables", ...rule],
    "installing kind host-mock masquerade rule",
    { timeoutMs: 30_000 },
  );
}

export async function ensureEgressAllowlist(options: KubeLayerOptions = {}): Promise<void> {
  const runtime = createRuntime(options);
  const allowedHostIp = await resolveAllowedHostIp(runtime);
  const allowedMockPort = mockPort("OMNIRUSH_EVAL_KUBE_ALLOWED_MOCK_PORT", 4791);
  const deniedMockPort = mockPort("OMNIRUSH_EVAL_KUBE_DENIED_MOCK_PORT", 4792);
  const template = await readFile(resolve(REPO_ROOT, KUBE_ALLOW_EXTERNAL_EGRESS_TEMPLATE), "utf8");
  const externalManifest = template
    .replaceAll("${OMNIRUSH_EVAL_KUBE_ALLOWED_HOST_IP}", allowedHostIp)
    .replaceAll("${OMNIRUSH_EVAL_KUBE_ALLOWED_MOCK_PORT}", String(allowedMockPort));
  if (externalManifest.includes("${OMNIRUSH_EVAL_")) {
    throw new Error(`Unresolved placeholder in ${KUBE_ALLOW_EXTERNAL_EGRESS_TEMPLATE}.`);
  }
  await ensureHostMockMasquerade(runtime, allowedHostIp, allowedMockPort);

  await checkedExec(
    runtime,
    "kubectl",
    kubectlArgs(["apply", "-f", KUBE_DEFAULT_DENY_EGRESS_MANIFEST]),
    `kubectl apply -f ${KUBE_DEFAULT_DENY_EGRESS_MANIFEST}`,
    { timeoutMs: 60_000 },
  );
  await checkedExec(
    runtime,
    "kubectl",
    kubectlArgs(["apply", "-f", KUBE_ALLOW_CORE_EGRESS_MANIFEST]),
    `kubectl apply -f ${KUBE_ALLOW_CORE_EGRESS_MANIFEST}`,
    { timeoutMs: 60_000 },
  );
  await checkedExec(
    runtime,
    "kubectl",
    kubectlArgs(["apply", "-f", "-"]),
    `rendering and applying ${KUBE_ALLOW_EXTERNAL_EGRESS_TEMPLATE}`,
    { input: externalManifest, timeoutMs: 60_000 },
  );

  process.env.OMNIRUSH_EVAL_KUBE_EGRESS_TEST = "1";
  process.env.OMNIRUSH_EVAL_KUBE_ALLOWED_HOST_IP = allowedHostIp;
  process.env.OMNIRUSH_EVAL_KUBE_ALLOWED_MOCK_PORT = String(allowedMockPort);
  process.env.OMNIRUSH_EVAL_KUBE_DENIED_MOCK_PORT = String(deniedMockPort);
  runtime.log("Kube egress allowlist enforced for den-api and den-web");
  runtime.log(`export OMNIRUSH_EVAL_KUBE_EGRESS_TEST=1 OMNIRUSH_EVAL_KUBE_ALLOWED_HOST_IP=${allowedHostIp} OMNIRUSH_EVAL_KUBE_ALLOWED_MOCK_PORT=${allowedMockPort} OMNIRUSH_EVAL_KUBE_DENIED_MOCK_PORT=${deniedMockPort}`);
}

export async function kubeStackTest(options: KubeLayerOptions = {}): Promise<void> {
  const runtime = createRuntime(options);
  const args = helmTestArgs();
  const result = await runtime.run("helm", args, { timeoutMs: 6 * 60_000 });
  if (result.code !== 0) {
    const logs = await runtime.run("kubectl", kubectlArgs(["logs", `job/${KUBE_RELEASE_NAME}-env-probe`, "--all-containers", "--tail=160"]), { timeoutMs: 60_000 });
    const detail = [result.stderr.trim(), result.stdout.trim(), logs.stdout.trim(), logs.stderr.trim()].filter(Boolean).join("\n");
    throw new Error(`helm ${args.join(" ")} failed with exit ${result.code}:\n${detail}`);
  }
  const output = [result.stdout.trim(), result.stderr.trim()].filter(Boolean).join("\n");
  if (output) runtime.log(output);
}

async function mysqlQuery(runtime: KubeRuntime, sql: string): Promise<string> {
  const result = await checkedExec(runtime, "kubectl", kubectlArgs([
    "exec",
    `deployment/${MYSQL_DEPLOYMENT}`,
    "--",
    "mysql",
    "-uomnirush",
    "-pomnirush",
    "omnirush_den",
    "-N",
    "-e",
    sql,
  ]), "kubectl exec mysql query", { timeoutMs: 60_000 });
  return result.stdout.trim();
}

export async function ensureSchema(options: KubeLayerOptions = {}): Promise<void> {
  const runtime = createRuntime(options);
  const schema = await mysqlQuery(runtime, "SHOW TABLES LIKE 'organization'; SHOW TABLES LIKE 'desktop_connect_grant'; SHOW TABLES LIKE 'scim_group'; SHOW COLUMNS FROM scim_provider LIKE 'group_mapping_mode';");
  if (
    schema.includes("organization")
    && schema.includes("desktop_connect_grant")
    && schema.includes("scim_group")
    && schema.includes("group_mapping_mode")
  ) {
    runtime.log("Schema present after Helm migration job");
    return;
  }
  throw new Error(`Helm migration did not leave the expected Den schema. mysql output:\n${schema}`);
}

export async function ensureSeed(options: KubeLayerOptions = {}): Promise<void> {
  const runtime = createRuntime(options);
  const existing = await mysqlQuery(runtime, `SELECT id FROM \`user\` WHERE email='${DEMO_EMAIL.toLowerCase()}' LIMIT 1;`);
  if (existing) {
    runtime.log(`Demo org present (${DEMO_EMAIL})`);
    return;
  }
  const result = await runtime.run("kubectl", kubectlArgs([
    "exec",
    `deployment/${KUBE_RELEASE_NAME}-den-api`,
    "--",
    "sh",
    "-lc",
    "cd /app/ee/apps/den-api && OMNIRUSH_DEV_MODE=1 DEN_DEMO_SEED_ALLOW_NONLOCAL=1 DEN_DEMO_SEED_FETCH_GITHUB=0 node --conditions=development --import tsx scripts/seed-demo-org.ts",
  ]), { timeoutMs: 10 * 60_000 });
  if (result.code !== 0) {
    const detail = [result.stderr.trim(), result.stdout.trim()].filter(Boolean).join("\n");
    throw new Error(`Demo seed failed with exit ${result.code}:\n${detail}`);
  }
  const seeded = await mysqlQuery(runtime, `SELECT id FROM \`user\` WHERE email='${DEMO_EMAIL.toLowerCase()}' LIMIT 1;`);
  if (!seeded) throw new Error("Seed completed but the demo owner was not found in MySQL.");
  runtime.log("Demo org seeded");
}

export async function ensureKindDenReady(options: KubeLayerOptions = {}): Promise<void> {
  const runtime = createRuntime(options);
  await phase(runtime, "existing-stack", () => requireKindStack(runtime));
  await phase(runtime, "den-api-rollout", () => ensureRollout(runtime, `${KUBE_RELEASE_NAME}-den-api`, "60s"));
  await phase(runtime, "den-web-rollout", () => ensureRollout(runtime, `${KUBE_RELEASE_NAME}-den-web`, "60s"));
  await phase(runtime, "schema", () => ensureSchema(options));
  await phase(runtime, "seed", () => ensureSeed(options));
}

async function portForwardHealthy(kind: "api" | "web"): Promise<boolean> {
  return kind === "api"
    ? await httpOk(`${DEN_API_URL}/health`)
    : await httpOk(`${DEN_WEB_URL}/api/ready`);
}

async function recordedLivePid(runtime: KubeRuntime, stateName: string): Promise<number | null> {
  const rawPid = await readPidState(runtime, stateName);
  if (!rawPid) return null;
  const pid = Number(rawPid);
  return Number.isInteger(pid) && pid > 0 && processIsAlive(pid) ? pid : null;
}

async function freeLocalPort(runtime: KubeRuntime, port: number): Promise<void> {
  const result = await runtime.run("lsof", ["-nP", `-iTCP:${port}`, "-sTCP:LISTEN", "-t"], { timeoutMs: 10_000 });
  if (result.code !== 0) return;
  const pids = result.stdout.split("\n").map((line) => line.trim()).filter(Boolean);
  for (const rawPid of pids) {
    const pid = Number(rawPid);
    if (!Number.isInteger(pid) || pid <= 0) continue;
    try {
      runtime.killProcess(pid, "SIGKILL");
      runtime.log(`Cleared stale process ${pid} holding :${port}`);
    } catch {
      // Already gone.
    }
  }
}

async function ensurePortForward(runtime: KubeRuntime, kind: "api" | "web", service: string, localPort: number, remotePort: number): Promise<number> {
  const stateName = `${kind}-port-forward.pid`;
  const recordedPid = await recordedLivePid(runtime, stateName);
  if (recordedPid !== null && await portForwardHealthy(kind)) {
    runtime.log(`${kind} port-forward already healthy on :${localPort}`);
    return recordedPid;
  }
  if (await portForwardHealthy(kind)) {
    runtime.log(`${kind} endpoint on :${localPort} is healthy but not owned by this kube stack; replacing it with a tracked port-forward.`);
  }
  await stopRecordedProcess(runtime, stateName, `${kind} port-forward`);
  await freeLocalPort(runtime, localPort);
  const pid = runtime.spawnDetached("kubectl", portForwardArgs(service, localPort, remotePort), {
    stateDir: runtime.stateDir,
    logName: `${kind}-port-forward`,
  });
  await writePidState(runtime, stateName, pid);
  for (let attempt = 0; attempt < 60; attempt += 1) {
    if (await portForwardHealthy(kind)) {
      runtime.log(`${kind} port-forward healthy on :${localPort}`);
      return pid;
    }
    await runtime.sleep(1_000);
  }
  await stopProcess(runtime, pid, `${kind} port-forward`);
  await rm(pidStatePath(runtime, stateName), { force: true });
  throw new Error(`${kind} port-forward did not become healthy on :${localPort} within 60s.`);
}

async function stopEndpointForward(runtime: KubeRuntime, kind: "api" | "web", pid: number): Promise<void> {
  await stopProcess(runtime, pid, `${kind} port-forward`);
  await rm(pidStatePath(runtime, `${kind}-port-forward.pid`), { force: true });
}

export async function exposeEndpointHandles(profile: KubeProfileConfig, options: KubeLayerOptions = {}): Promise<KindEndpoints> {
  const runtime = createRuntime(options);
  await mkdir(runtime.stateDir, { recursive: true });
  let apiPid: number | undefined;
  let webPid: number | undefined;
  let owner: { token: string; email: string };
  try {
    apiPid = await ensurePortForward(runtime, "api", DEN_API_SERVICE, DEN_API_PORT, 8788);
    webPid = await ensurePortForward(runtime, "web", DEN_WEB_SERVICE, DEN_WEB_PORT, 3005);
    owner = await signInDemoOwner(runtime);
  } catch (error) {
    if (webPid !== undefined) await stopEndpointForward(runtime, "web", webPid);
    if (apiPid !== undefined) await stopEndpointForward(runtime, "api", apiPid);
    throw error;
  }
  let stopped = false;
  return {
    apiUrl: DEN_API_URL,
    webUrl: DEN_WEB_URL,
    token: owner.token,
    adminEmail: owner.email,
    async stop(): Promise<void> {
      if (stopped) return;
      stopped = true;
      await stopEndpointForward(runtime, "api", apiPid);
      await stopEndpointForward(runtime, "web", webPid);
    },
  };
}

export async function exposeEndpoints(profile: KubeProfileConfig, options: KubeLayerOptions = {}): Promise<void> {
  const runtime = createRuntime(options);
  const endpoints = await exposeEndpointHandles(profile, options);
  process.env.OMNIRUSH_EVAL_DEN_API_URL = DEN_API_URL;
  process.env.OMNIRUSH_EVAL_DEN_WEB_URL = DEN_WEB_URL;
  if (profile.orgMode === "multi_org") {
    process.env.OMNIRUSH_EVAL_DEN_MULTI_ORG = "1";
  } else {
    delete process.env.OMNIRUSH_EVAL_DEN_MULTI_ORG;
  }
  process.env.OMNIRUSH_EVAL_DEN_TOKEN = endpoints.token;
  runtime.log(`Kube Den endpoints exported: OMNIRUSH_EVAL_DEN_API_URL=${DEN_API_URL}, OMNIRUSH_EVAL_DEN_WEB_URL=${DEN_WEB_URL}${profile.orgMode === "multi_org" ? ", OMNIRUSH_EVAL_DEN_MULTI_ORG=1" : ""}`);
}

function appUserDataHome(): string {
  return DEFAULT_ELECTRON_USERDATA;
}

function appBootstrapPath(): string {
  return join(appUserDataHome(), "omnirush-dev-data", "home", ".config", "omnirush", "desktop-bootstrap.json");
}

async function hasCdpPageTarget(baseUrl: string): Promise<boolean> {
  try {
    const response = await fetch(`${baseUrl}/json/list`, { signal: AbortSignal.timeout(2_500) });
    if (!response.ok) return false;
    const targets: unknown = await response.json();
    return Array.isArray(targets)
      && targets.some((target) => isRecord(target) && target.type === "page" && typeof target.webSocketDebuggerUrl === "string");
  } catch {
    return false;
  }
}

async function freeStaleAppPorts(runtime: KubeRuntime): Promise<void> {
  for (const port of [9825, 9823, 5173]) {
    await freeLocalPort(runtime, port);
  }
  await runtime.sleep(1_500);
}

function chromeCdpPort(cdpCandidates: string[]): number {
  for (const candidate of cdpCandidates) {
    try {
      const url = new URL(candidate);
      if ((url.hostname === "127.0.0.1" || url.hostname === "localhost") && url.port) {
        const port = Number(url.port);
        if (Number.isInteger(port) && port > 0 && port <= 65_535) return port;
      }
    } catch {
      // Try the next candidate.
    }
  }
  return 9825;
}

async function ensureChromeApp(cdpCandidates: string[], options: KubeLayerOptions = {}): Promise<void> {
  const runtime = createRuntime(options);
  const cdpPort = chromeCdpPort(cdpCandidates);
  const cdpUrl = `http://127.0.0.1:${cdpPort}`;
  if (await hasCdpPageTarget(cdpUrl)) {
    runtime.log(`Chrome CDP already reachable at ${cdpUrl}`);
    return;
  }
  await freeStaleAppPorts(runtime);
  await stopRecordedProcess(runtime, "chrome.pid", "Chrome CDP surface");
  const profileDir = join(runtime.stateDir, "chrome-user-data");
  await mkdir(profileDir, { recursive: true });
  const binary = resolveChromeBinary(process.env, process.platform);
  const pid = runtime.spawnDetached(binary, [
    "--headless=new",
    "--window-size=1280,900",
    "--disable-gpu",
    "--no-first-run",
    "--no-default-browser-check",
    "--remote-debugging-address=127.0.0.1",
    `--remote-debugging-port=${cdpPort}`,
    `--user-data-dir=${profileDir}`,
    DEN_WEB_URL,
  ], {
    stateDir: runtime.stateDir,
    logName: "chrome",
  });
  await writePidState(runtime, "chrome.pid", pid);
  for (let attempt = 0; attempt < 60; attempt += 1) {
    if (await hasCdpPageTarget(cdpUrl)) {
      runtime.log(`Chrome CDP up at ${cdpUrl}`);
      return;
    }
    await runtime.sleep(1_000);
  }
  throw new Error(`Chrome CDP page target did not come up at ${cdpUrl} within 60s.`);
}

async function ensureApp(cdpCandidates: string[], options: KubeLayerOptions = {}): Promise<void> {
  const runtime = createRuntime(options);
  for (const candidate of cdpCandidates) {
    if (await hasCdpPageTarget(candidate)) {
      runtime.log(`App CDP already reachable at ${candidate} — make sure it targets the kube Den endpoints if needed.`);
      return;
    }
  }

  if (process.env.OMNIRUSH_EVAL_KUBE_SURFACE?.trim() !== "electron") {
    runtime.log("Starting Chrome CDP surface for kube Den web evals (set OMNIRUSH_EVAL_KUBE_SURFACE=electron to force dev Electron).");
    await ensureChromeApp(cdpCandidates, options);
    return;
  }

  await freeStaleAppPorts(runtime);
  const bootstrapPath = appBootstrapPath();
  await mkdir(dirname(bootstrapPath), { recursive: true });
  await writeFile(
    bootstrapPath,
    `${JSON.stringify({ baseUrl: DEN_BASE_URL, requireSignin: false }, null, 2)}\n`,
  );
  await writePidState(runtime, "bootstrap.path", bootstrapPath);
  runtime.log(`Wrote desktop bootstrap -> ${DEN_BASE_URL}`);

  const pid = runtime.spawnDetached("pnpm", ["dev"], {
    stateDir: runtime.stateDir,
    logName: "app",
    env: { OMNIRUSH_ELECTRON_USERDATA: appUserDataHome() },
  });
  await writePidState(runtime, "app.pid", pid);
  for (let attempt = 0; attempt < 45; attempt += 1) {
    for (const candidate of cdpCandidates) {
      if (await hasCdpPageTarget(candidate)) {
        runtime.log(`App CDP up at ${candidate}`);
        await runtime.sleep(8_000);
        return;
      }
    }
    await runtime.sleep(4_000);
  }
  throw new Error("Dev Electron CDP page target did not come up within 3 minutes.");
}

export async function ensureKubeStack(options: EnsureKubeStackOptions): Promise<void> {
  const runtime = createRuntime(options);
  await mkdir(runtime.stateDir, { recursive: true });
  const profile = kubeProfileConfig(options.profile ?? "single-org");
  await writePidState(runtime, "profile", profile.profile);

  await phase(runtime, "cluster", () => ensureCluster(options));
  const imagePlan = await phase(runtime, "images", async () => {
    const plan = await resolveKubeImagePlan({ exec: options.exec, images: options.images, log: runtime.log });
    await ensureImages(plan, options);
    await writePidState(runtime, "images", plan.mode);
    return plan;
  });
  await phase(runtime, "database", () => ensureDatabase(options));
  await phase(runtime, "release", () => ensureRelease(profile, imagePlan, options));
  if (options.egress === "allowlist") {
    await phase(runtime, "egress-allowlist", () => ensureEgressAllowlist(options));
  }
  await phase(runtime, "schema", () => ensureSchema(options));
  await phase(runtime, "helm-test", () => kubeStackTest(options));
  await phase(runtime, "seed", () => ensureSeed(options));
  await phase(runtime, "port-forward", () => exposeEndpoints(profile, options));
  if (options.skipApp) {
    runtime.log("Skipping dev Electron startup — selected eval flow is app-less");
  } else {
    await phase(runtime, "app", () => ensureApp(options.cdpCandidates, options));
  }
}

export async function kubeStackDown(options: KubeStackDownOptions = {}): Promise<void> {
  const runtime = createRuntime(options);
  try {
    await access(runtime.stateDir);
  } catch {
    if (options.deleteCluster) {
      await checkedExec(runtime, "kind", ["delete", "cluster", "--name", KUBE_CLUSTER_NAME], `kind delete cluster --name ${KUBE_CLUSTER_NAME}`, { timeoutMs: 180_000 });
    }
    return;
  }
  await stopRecordedProcess(runtime, "api-port-forward.pid", "api port-forward");
  await stopRecordedProcess(runtime, "web-port-forward.pid", "web port-forward");
  await stopRecordedProcess(runtime, "chrome.pid", "Chrome CDP surface");
  await stopRecordedProcess(runtime, "app.pid", "dev app");
  const bootstrapPath = await readPidState(runtime, "bootstrap.path");
  if (bootstrapPath) {
    await rm(bootstrapPath, { force: true });
    runtime.log("Removed kube desktop bootstrap override");
  }
  const uninstall = await runtime.run("helm", ["uninstall", KUBE_RELEASE_NAME, "--namespace", KUBE_NAMESPACE, "--kube-context", KUBE_CONTEXT], { timeoutMs: 180_000 });
  if (uninstall.code === 0) {
    runtime.log(`Helm release ${KUBE_RELEASE_NAME} uninstalled (kind cluster and MySQL kept)`);
  } else {
    runtime.log(`Helm uninstall skipped or failed: ${(uninstall.stderr || uninstall.stdout).trim()}`);
  }
  const hookCleanup = await runtime.run("kubectl", kubectlArgs(["delete", "job", `${KUBE_RELEASE_NAME}-env-probe`, `${KUBE_RELEASE_NAME}-migrate`, "--ignore-not-found"]), { timeoutMs: 60_000 });
  if (hookCleanup.code === 0) runtime.log("Removed Helm hook jobs");
  if (options.deleteCluster) {
    await checkedExec(runtime, "kind", ["delete", "cluster", "--name", KUBE_CLUSTER_NAME], `kind delete cluster --name ${KUBE_CLUSTER_NAME}`, { timeoutMs: 180_000 });
    runtime.log(`Deleted kind cluster ${KUBE_CLUSTER_NAME}`);
  }
  await rm(runtime.stateDir, { recursive: true, force: true });
}