Skip to content

Project NOMAD is an offline-first knowledge and education server. Wikipedia, thousands of books, courses, maps, and optional local AI, all running on hardware you own with no internet required.

TypeScript
44.9 KB
Raw Blame
import Service from '#models/service'
import InstalledResource from '#models/installed_resource'
import { inject } from '@adonisjs/core'
import { DockerService } from '#services/docker_service'
import { ServiceSlim } from '../../types/services.js'
import logger from '@adonisjs/core/services/logger'
import si from 'systeminformation'
import {
  GpuHealthStatus,
  NomadDiskInfo,
  NomadDiskInfoRaw,
  SystemInformationResponse,
} from '../../types/system.js'
import { SERVICE_NAMES } from '../../constants/service_names.js'
import { readFileSync } from 'node:fs'
import { readFile } from 'node:fs/promises'
import path, { join } from 'node:path'
import { getAllFilesystems, getFile } from '../utils/fs.js'
import axios from 'axios'
import env from '#start/env'
import KVStore from '#models/kv_store'
import { KV_STORE_SCHEMA, KVStoreKey } from '../../types/kv_store.js'
import { isNewerVersion } from '../utils/version.js'
import { isUnresolvedGpuModel } from '../utils/gpu_model.js'
import {
  classifyOllamaComputeBackend,
  diagnoseAmdCpuFallback,
  parseOllamaGpuFromLogs,
  readOllamaStartupLogs,
  resolveExpectedGpuVendor,
  type OllamaGpuInfo,
} from '#services/ollama_compute'
import { invalidateAssistantNameCache } from '../../config/inertia.js'
import { invalidateMinRelevanceCache } from '../utils/rag_relevance.js'
import { invalidateResponseStyleCache } from '../utils/response_style.js'
import { KiwixLibraryService } from '#services/kiwix_library_service'

@inject()
export class SystemService {
  private static appVersion: string | null = null
  private static diskInfoFile = '/storage/nomad-disk-info.json'

  constructor(private dockerService: DockerService) {}

  async checkServiceInstalled(serviceName: string): Promise<boolean> {
    const services = await this.getServices({ installedOnly: true })
    return services.some((service) => service.service_name === serviceName)
  }

  async getInternetStatus(): Promise<boolean> {
    // Primary endpoint stays Cloudflare's privacy-respecting utility endpoint.
    // The fallbacks are hosts the application already contacts elsewhere
    // (GitHub API for update checks, the Project NOMAD API for release-note
    // subscriptions), so no new third-party services are introduced. They exist
    // to avoid false "offline" reports on networks that block 1.1.1.1.
    const DEFAULT_TEST_URLS = [
      'https://1.1.1.1/cdn-cgi/trace',
      'https://api.github.com',
      'https://api.projectnomad.us',
    ]
    const MAX_ATTEMPTS = 3

    let testUrls = DEFAULT_TEST_URLS

    // Resolve the test endpoint in priority order: the INTERNET_STATUS_TEST_URL
    // env var always wins (legacy override for operators who intentionally point
    // connectivity checks at a specific endpoint), then the UI-configurable value
    // stored in KVStore, and finally the built-in defaults.
    const envTestUrl = env.get('INTERNET_STATUS_TEST_URL')?.trim()
    const kvTestUrl = (await KVStore.getValue('system.internetStatusTestUrl'))?.trim()
    const customTestUrl = envTestUrl || kvTestUrl

    // If a custom test URL is provided and valid, use it exclusively.
    if (customTestUrl && customTestUrl !== '') {
      try {
        new URL(customTestUrl)
        testUrls = [customTestUrl]
      } catch (error) {
        logger.warn(
          `Invalid internet status test URL: ${customTestUrl}. Falling back to default URLs.`
        )
      }
    }

    for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) {
      try {
        // Probe all test endpoints in parallel and resolve as soon as the first one
        // responds. Any HTTP response (including non-2xx) means we reached the
        // internet, so accept all status codes rather than requiring a strict 200.
        await Promise.any(
          testUrls.map((testUrl) => {
            logger.debug(`[SystemService] Checking internet connectivity via: ${testUrl}`)
            return axios.get(testUrl, { timeout: 5000, validateStatus: () => true })
          })
        )
        return true
      } catch (error) {
        // Promise.any only rejects (with an AggregateError) when every endpoint failed.
        logger.warn(
          `Internet status check attempt ${attempt}/${MAX_ATTEMPTS} failed: ${error instanceof Error ? error.message : error}`
        )

        if (attempt < MAX_ATTEMPTS) {
          // delay before next attempt
          await new Promise((resolve) => setTimeout(resolve, 1000))
        }
      }
    }

    logger.warn('All internet status check attempts failed.')
    return false
  }

  /**
   * The running nomad_ollama container and its inspect data, or null when it is
   * not running. The inspect data records the GPU config the install actually
   * applied (DeviceRequests, /dev/kfd, the ROCm image), which is what the GPU
   * health check keys on.
   */
  private async _findRunningOllamaContainer() {
    const containers = await this.dockerService.docker.listContainers({ all: false })
    const summary = containers.find((c) => c.Names.includes(`/${SERVICE_NAMES.OLLAMA}`))
    if (!summary) return null
    const container = this.dockerService.docker.getContainer(summary.Id)
    return { container, inspect: await container.inspect() }
  }

  /**
   * Probe Ollama startup logs for the canonical "inference compute" line that records
   * which compute backend was selected. This catches silent CPU fallback (e.g. when
   * /dev/kfd is mounted but ROCm initialization fails, or NVML dies after an update)
   * which the older nvidia-smi exec probe could not detect.
   *
   * Returns the parsed library, GPU model name, and VRAM in MiB, or null when:
   *   - the Ollama container is not running
   *   - the line has not been emitted (Ollama still starting up)
   *   - logs show CPU-only operation (no GPU detected)
   */
  async getOllamaInferenceComputeFromLogs(): Promise<OllamaGpuInfo | null> {
    try {
      const ollama = await this._findRunningOllamaContainer()
      if (!ollama) return null
      return parseOllamaGpuFromLogs(await readOllamaStartupLogs(ollama.container, ollama.inspect))
    } catch (error) {
      logger.warn(
        `[SystemService] Failed to probe Ollama logs for inference compute line: ${error instanceof Error ? error.message : error}`
      )
      return null
    }
  }

  async getNvidiaSmiInfo(): Promise<
    | Array<{ vendor: string; model: string; vram: number }>
    | { error: string }
    | 'OLLAMA_NOT_FOUND'
    | 'BAD_RESPONSE'
    | 'UNKNOWN_ERROR'
  > {
    try {
      const containers = await this.dockerService.docker.listContainers({ all: false })
      const ollamaContainer = containers.find((c) => c.Names.includes(`/${SERVICE_NAMES.OLLAMA}`))
      if (!ollamaContainer) {
        logger.info(
          'Ollama container not found for nvidia-smi info retrieval. This is expected if Ollama is not installed.'
        )
        return 'OLLAMA_NOT_FOUND'
      }

      // Execute nvidia-smi inside the Ollama container to get GPU info
      const container = this.dockerService.docker.getContainer(ollamaContainer.Id)
      const exec = await container.exec({
        Cmd: ['nvidia-smi', '--query-gpu=name,memory.total', '--format=csv,noheader,nounits'],
        AttachStdout: true,
        AttachStderr: true,
        Tty: true,
      })

      // Read the output stream with a timeout to prevent hanging if nvidia-smi fails
      const stream = await exec.start({ Tty: true })
      const output = await new Promise<string>((resolve) => {
        let data = ''
        const timeout = setTimeout(() => resolve(data), 5000)
        stream.on('data', (chunk: Buffer) => {
          data += chunk.toString()
        })
        stream.on('end', () => {
          clearTimeout(timeout)
          resolve(data)
        })
      })

      // Remove any non-printable characters and trim the output
      const cleaned = Array.from(output)
        .filter((character) => character.charCodeAt(0) > 8)
        .join('')
        .trim()
      if (
        cleaned &&
        !cleaned.toLowerCase().includes('error') &&
        !cleaned.toLowerCase().includes('not found')
      ) {
        // Split by newlines to handle multiple GPUs installed
        const lines = cleaned.split('\n').filter((line) => line.trim())

        // Map each line out to a useful structure for us
        const gpus = lines.map((line) => {
          const parts = line.split(',').map((s) => s.trim())
          return {
            vendor: 'NVIDIA',
            model: parts[0] || 'NVIDIA GPU',
            vram: parts[1] ? Number.parseInt(parts[1], 10) : 0,
          }
        })

        return gpus.length > 0 ? gpus : 'BAD_RESPONSE'
      }

      // If we got output but looks like an error, consider it a bad response from nvidia-smi
      return 'BAD_RESPONSE'
    } catch (error) {
      logger.error('Error getting nvidia-smi info:', error)
      if (error instanceof Error && error.message) {
        return { error: error.message }
      }
      return 'UNKNOWN_ERROR'
    }
  }

  async getExternalOllamaGpuInfo(): Promise<Array<{
    vendor: string
    model: string
    vram: number
  }> | null> {
    try {
      // If a remote Ollama URL is configured, use it directly without requiring a local container
      const remoteOllamaUrl = await KVStore.getValue('ai.remoteOllamaUrl')
      if (!remoteOllamaUrl) {
        const containers = await this.dockerService.docker.listContainers({ all: false })
        const ollamaContainer = containers.find((c) => c.Names.includes(`/${SERVICE_NAMES.OLLAMA}`))
        if (!ollamaContainer) {
          return null
        }

        const actualImage = (ollamaContainer.Image || '').toLowerCase()
        if (actualImage.includes('ollama/ollama') || actualImage.startsWith('ollama:')) {
          return null
        }
      }

      const ollamaUrl = remoteOllamaUrl || (await this.dockerService.getServiceURL(SERVICE_NAMES.OLLAMA))
      if (!ollamaUrl) {
        return null
      }

      await axios.get(new URL('/api/tags', ollamaUrl).toString(), { timeout: 3000 })

      let vramMb = 0
      try {
        const psResponse = await axios.get(new URL('/api/ps', ollamaUrl).toString(), {
          timeout: 3000,
        })
        const loadedModels = Array.isArray(psResponse.data?.models) ? psResponse.data.models : []
        const largestAllocation = loadedModels.reduce(
          (max: number, model: { size_vram?: number | string }) =>
            Math.max(max, Number(model.size_vram) || 0),
          0
        )
        vramMb = largestAllocation > 0 ? Math.round(largestAllocation / (1024 * 1024)) : 0
      } catch {}

      return [
        {
          vendor: 'NVIDIA',
          model: 'NVIDIA GPU (external Ollama)',
          vram: vramMb,
        },
      ]
    } catch (error) {
      logger.info(
        `[SystemService] External Ollama GPU probe failed: ${error instanceof Error ? error.message : error}`
      )
      return null
    }
  }

  async getServices({ installedOnly = true }: { installedOnly?: boolean }): Promise<ServiceSlim[]> {
    const statuses = await this._syncContainersWithDatabase() // Sync and reuse the fetched status list

    const query = Service.query()
      .orderBy('display_order', 'asc')
      .orderBy('friendly_name', 'asc')
      .select(
        'id',
        'service_name',
        'installed',
        'installation_status',
        'ui_location',
        'custom_url',
        'friendly_name',
        'description',
        'icon',
        'powered_by',
        'display_order',
        'container_image',
        'available_update_version',
        'auto_update_enabled',
        'is_custom',
        'is_user_modified',
        'is_deprecated',
        'is_link_tile',
        'link_color',
        'category'
      )
      .where('is_dependency_service', false)
      // Deprecated/sunset apps stay visible only while still installed, so the user can manage and
      // uninstall them — they never reappear in the install catalog once removed.
      .where((q) => {
        q.where('is_deprecated', false).orWhere('installed', true)
      })
    if (installedOnly) {
      query.where('installed', true)
    }

    const services = await query
    if (!services || services.length === 0) {
      return []
    }

    const toReturn: ServiceSlim[] = []

    for (const service of services) {
      const status = statuses.find((s) => s.service_name === service.service_name)
      toReturn.push({
        id: service.id,
        service_name: service.service_name,
        friendly_name: service.friendly_name,
        description: service.description,
        icon: service.icon,
        installed: service.installed,
        installation_status: service.installation_status,
        status: status ? status.status : 'unknown',
        ui_location: service.ui_location || '',
        custom_url: service.custom_url,
        powered_by: service.powered_by,
        display_order: service.display_order,
        container_image: service.container_image,
        available_update_version: service.available_update_version,
        auto_update_enabled: service.auto_update_enabled,
        is_custom: service.is_custom,
        is_user_modified: service.is_user_modified,
        is_deprecated: service.is_deprecated,
        is_link_tile: service.is_link_tile,
        link_color: service.link_color,
        category: service.category,
      })
    }

    return toReturn
  }

  static getAppVersion(): string {
    try {
      if (this.appVersion) {
        return this.appVersion
      }

      // Return 'dev' for development environment (version.json won't exist)
      if (process.env.NODE_ENV === 'development') {
        this.appVersion = 'dev'
        return 'dev'
      }

      const packageJson = readFileSync(join(process.cwd(), 'version.json'), 'utf-8')
      const packageData = JSON.parse(packageJson)

      const version = packageData.version || '0.0.0'

      this.appVersion = version
      return version
    } catch (error) {
      logger.error('Error getting app version:', error)
      return '0.0.0'
    }
  }

  async getSystemInfo(): Promise<SystemInformationResponse | undefined> {
    try {
      const [cpu, mem, os, currentLoad, fsSize, uptime, graphics] = await Promise.all([
        si.cpu(),
        si.mem(),
        si.osInfo(),
        si.currentLoad(),
        si.fsSize(),
        si.time(),
        si.graphics(),
      ])

      let diskInfo: NomadDiskInfoRaw | undefined
      let disk: NomadDiskInfo[] = []

      try {
        const diskInfoRawString = await getFile(
          path.join(process.cwd(), SystemService.diskInfoFile),
          'string'
        )

        diskInfo = (
          diskInfoRawString
            ? JSON.parse(diskInfoRawString.toString())
            : { diskLayout: { blockdevices: [] }, fsSize: [] }
        ) as NomadDiskInfoRaw

        disk = this.calculateDiskUsage(diskInfo)
      } catch (error) {
        logger.error('Error reading disk info file:', error)
      }

      // GPU health tracking — detect when host has a GPU runtime but Ollama can't access it.
      // Primary probe: parse Ollama's "inference compute" startup log line for both NVIDIA
      // and AMD. Secondary probe (NVIDIA only): nvidia-smi exec, retained as a fallback for
      // hardware enrichment when log parsing has not yet captured a startup line.
      let gpuHealth: GpuHealthStatus = {
        status: 'no_gpu',
        hasNvidiaRuntime: false,
        hasRocmRuntime: false,
        ollamaGpuAccessible: false,
      }

      // Query Docker API for host-level info (hostname, OS, GPU runtime)
      // si.osInfo() returns the container's info inside Docker, not the host's
      try {
        const dockerInfo = await this.dockerService.docker.info()

        if (dockerInfo.Name) {
          os.hostname = dockerInfo.Name
        }
        if (dockerInfo.OperatingSystem) {
          os.distro = dockerInfo.OperatingSystem
        }
        if (dockerInfo.KernelVersion) {
          os.kernel = dockerInfo.KernelVersion
        }

        // si.graphics() in the admin container uses lspci (pciutils ships in
        // the image for AMD detection). lspci has no real VRAM info for
        // discrete GPUs, so systeminformation parses the first PCI memory
        // Region (BAR0, typically 1-32 MiB) as `vram`. nvidia-smi / ROCm
        // tooling enrichment also can't run since neither is in the admin
        // image. No real dGPU has under 256 MiB, so any discrete-GPU controller
        // below that threshold needs the probes below to give us real data.
        // Applies to both NVIDIA and AMD; Intel iGPUs are exempt because their
        // shared-system-memory VRAM reading via lspci can legitimately be small.
        const DGPU_BOGUS_VRAM_THRESHOLD_MIB = 256
        const isDiscreteGpuVendor = (vendor: string) =>
          /nvidia|advanced micro devices|amd|ati/i.test(vendor)
        const isBogusDgpuVram = (c: { vendor?: string; vram?: number | null }) =>
          isDiscreteGpuVendor(c.vendor || '') &&
          typeof c.vram === 'number' &&
          c.vram < DGPU_BOGUS_VRAM_THRESHOLD_MIB

        // Clear the bogus value up front. If a probe replaces the entry below
        // we get the real VRAM; if no probe succeeds (Ollama not installed,
        // passthrough_failed) the UI falls back to "N/A" instead of showing
        // "1 MB" / "32 MB". The lspci model/vendor strings stay since they're
        // still useful for identifying the card.
        const hasLspciBogusDgpuVram = (graphics.controllers || []).some(isBogusDgpuVram)
        if (hasLspciBogusDgpuVram) {
          for (const c of graphics.controllers) {
            if (isBogusDgpuVram(c)) c.vram = null
          }
        }

        // The same pci.ids staleness that produces bogus VRAM also produces a
        // placeholder model name — a card newer than the container's pci.ids is
        // reported as its raw id, e.g. "Device 2d05" for an RTX 5060 (#1165).
        //
        // These are independent symptoms, not one. lspci can hand back a
        // perfectly plausible BAR0 reading alongside an unresolved name, and in
        // that case nothing above fires, no probe runs, and Settings > System
        // renders the raw PCI id as the GPU model (#1196). NVIDIA usually hides
        // this because the nvidia-smi path tends to be reached for other
        // reasons; AMD has no equivalent, so it shows the raw id.
        //
        // The probes below resolve a real name from Ollama's own startup log,
        // so trigger them on an unresolved name too rather than only on VRAM.
        const hasUnresolvedGpuName = (graphics.controllers || []).some((c) =>
          isUnresolvedGpuModel(c.model || '')
        )

        // Run the probes when controllers are empty (common inside Docker),
        // when lspci gave us bogus discrete-GPU BAR0 values that need replacing,
        // or when it named a card it couldn't resolve. Only in those cases do
        // probe results replace the lspci controllers; a clean lspci reading
        // stays as the display source.
        const lspciNeedsProbe =
          !graphics.controllers ||
          graphics.controllers.length === 0 ||
          hasLspciBogusDgpuVram ||
          hasUnresolvedGpuName
        const useProbeControllers = (controllers: typeof graphics.controllers) => {
          if (lspciNeedsProbe) graphics.controllers = controllers
        }
        const toControllers = (gpus: Array<{ vendor: string; model: string; vram: number }>) =>
          gpus.map((gpu) => ({
            model: gpu.model,
            vendor: gpu.vendor,
            bus: '',
            vram: gpu.vram,
            vramDynamic: false,
          }))

        // Runtime flags are reported whichever path runs below. They used to be set
        // only inside the probe branch, so a box whose lspci reading looked clean
        // reported hasRocmRuntime: false even with an AMD GPU configured (#1344).
        const runtimes = dockerInfo.Runtimes || {}
        gpuHealth.hasNvidiaRuntime = 'nvidia' in runtimes

        // AMD doesn't register a Docker runtime. Detection sources, in priority order:
        //   1. KV 'gpu.type' (set by DockerService._detectGPUType after first Ollama install)
        //   2. Marker file at /app/storage/.nomad-gpu-type (written by install_nomad.sh)
        // The marker file matters because the System page should reflect AMD presence
        // even before AI Assistant has been installed for the first time.
        let savedGpuType: string | null | undefined = await KVStore.getValue('gpu.type') as string | undefined
        if (!savedGpuType) {
          try {
            savedGpuType = (await readFile('/app/storage/.nomad-gpu-type', 'utf8')).trim()
          } catch {}
        }
        const amdEnabledRaw = await KVStore.getValue('ai.amdGpuAcceleration')
        const amdAccelerationEnabled = String(amdEnabledRaw) !== 'false'
        gpuHealth.hasRocmRuntime = savedGpuType === 'amd' && amdAccelerationEnabled

        // When Ollama is expected on a GPU, its startup log decides health whatever
        // lspci returned. A clean lspci reading says a GPU exists, not that Ollama
        // uses it: an AMD iGPU silently on CPU used to land in the else branch below
        // and report 'ok' (#1344).
        const ollama = await this._findRunningOllamaContainer().catch(() => null)
        const expectedVendor = resolveExpectedGpuVendor({
          hasNvidiaRuntime: gpuHealth.hasNvidiaRuntime,
          amdConfigured: gpuHealth.hasRocmRuntime,
          container: ollama?.inspect ?? null,
        })

        if (expectedVendor || lspciNeedsProbe) {
          if (expectedVendor) {
            gpuHealth.gpuVendor = expectedVendor

            // Primary probe: Ollama log parsing — works for both vendors and catches silent fallback
            const ollamaLogs = ollama
              ? await readOllamaStartupLogs(ollama.container, ollama.inspect)
              : ''
            const logInfo = parseOllamaGpuFromLogs(ollamaLogs)
            if (logInfo) {
              useProbeControllers([
                {
                  model: logInfo.name,
                  vendor:
                    logInfo.library === 'CUDA' ? 'NVIDIA' : logInfo.library === 'ROCm' ? 'AMD' : '',
                  bus: '',
                  vram: logInfo.vramMiB,
                  vramDynamic: false,
                },
              ])
              gpuHealth.status = 'ok'
              gpuHealth.ollamaGpuAccessible = true
            } else if (classifyOllamaComputeBackend(ollamaLogs) === 'cpu') {
              // Ollama reported its devices and none was a GPU. Definitive for both
              // vendors, so skip the nvidia-smi exec, which cannot run in a CPU-only
              // container anyway.
              gpuHealth.status = 'passthrough_failed'
              logger.warn(
                `${expectedVendor === 'amd' ? 'AMD' : 'NVIDIA'} GPU expected but Ollama reported a CPU-only backend at startup`
              )
            } else if (expectedVendor === 'nvidia') {
              // NVIDIA secondary path: nvidia-smi exec preserves prior behavior when
              // the log parser hasn't seen a startup line yet (e.g. log rotation,
              // very fresh container). Distinguishes "no Ollama container" from
              // "container exists but GPU broken".
              const nvidiaInfo = await this.getNvidiaSmiInfo()
              if (Array.isArray(nvidiaInfo)) {
                useProbeControllers(toControllers(nvidiaInfo))
                gpuHealth.status = 'ok'
                gpuHealth.ollamaGpuAccessible = true
              } else {
                const externalOllamaGpu = await this.getExternalOllamaGpuInfo()
                if (externalOllamaGpu) {
                  useProbeControllers(toControllers(externalOllamaGpu))
                  gpuHealth.status = 'ok'
                  gpuHealth.ollamaGpuAccessible = true
                } else if (nvidiaInfo === 'OLLAMA_NOT_FOUND') {
                  gpuHealth.status = 'ollama_not_installed'
                } else {
                  gpuHealth.status = 'passthrough_failed'
                  logger.warn(
                    `NVIDIA runtime detected but GPU passthrough failed: ${typeof nvidiaInfo === 'string' ? nvidiaInfo : JSON.stringify(nvidiaInfo)}`
                  )
                }
              }
            } else if (!ollama) {
              // AMD path: no nvidia-smi equivalent worth running — log parser is authoritative.
              // Distinguish "Ollama not running" from "Ollama running but no GPU log line".
              const externalOllamaGpu = await this.getExternalOllamaGpuInfo()
              if (externalOllamaGpu) {
                useProbeControllers(toControllers(externalOllamaGpu))
                gpuHealth.status = 'ok'
                gpuHealth.ollamaGpuAccessible = true
              } else {
                gpuHealth.status = 'ollama_not_installed'
              }
            } else {
              gpuHealth.status = 'passthrough_failed'
              logger.warn(
                'AMD GPU detected but Ollama logs show no ROCm initialization — passthrough or HSA override may have failed'
              )
            }

            // Tell the UI whether a reinstall would help or an HSA override is needed.
            // Reinstalling an AMD container whose config already matches rebuilds the
            // same CPU-bound container, so the banner must not offer that as the fix.
            if (gpuHealth.status === 'passthrough_failed' && expectedVendor === 'amd' && ollama) {
              Object.assign(
                gpuHealth,
                diagnoseAmdCpuFallback(
                  ollama.inspect,
                  ollamaLogs,
                  // Pass the container env so the diff matches what a reinstall would
                  // really build, including an override it keeps from the container (#1377).
                  await this.dockerService.getAmdHsaOverride({
                    quiet: true,
                    containerEnv: ollama.inspect.Config?.Env,
                  })
                )
              )
            }
          }
        } else {
          // si.graphics() returned usable controllers and no GPU runtime is expected
          // (host install, not Docker). Still unprobed, see #1325.
          gpuHealth.status = 'ok'
          gpuHealth.ollamaGpuAccessible = true
        }
      } catch {
        // Docker info query failed, skip host-level enrichment
      }

      return {
        cpu,
        mem,
        os,
        disk,
        currentLoad,
        fsSize,
        uptime,
        graphics,
        gpuHealth,
      }
    } catch (error) {
      logger.error('Error getting system info:', error)
      return undefined
    }
  }

  async checkLatestVersion(force?: boolean): Promise<{
    success: boolean
    updateAvailable: boolean
    currentVersion: string
    latestVersion: string
    message?: string
  }> {
    try {
      const currentVersion = SystemService.getAppVersion()
      const cachedUpdateAvailable = await KVStore.getValue('system.updateAvailable')
      const cachedLatestVersion = await KVStore.getValue('system.latestVersion')

      // Use cached values if not forcing a fresh check.
      // the CheckUpdateJob will update these values every 12 hours
      if (!force) {
        return {
          success: true,
          updateAvailable: cachedUpdateAvailable ?? false,
          currentVersion,
          latestVersion: cachedLatestVersion || '',
        }
      }

      const earlyAccess = (await KVStore.getValue('system.earlyAccess')) ?? false

      let latestVersion: string
      if (earlyAccess) {
        const response = await axios.get(
          'https://api.github.com/repos/Crosstalk-Solutions/project-nomad/releases',
          { headers: { Accept: 'application/vnd.github+json' }, timeout: 5000 }
        )
        if (!response?.data?.length) throw new Error('No releases found')
        latestVersion = response.data[0].tag_name.replace(/^v/, '').trim()
      } else {
        const response = await axios.get(
          'https://api.github.com/repos/Crosstalk-Solutions/project-nomad/releases/latest',
          { headers: { Accept: 'application/vnd.github+json' }, timeout: 5000 }
        )
        if (!response?.data?.tag_name) throw new Error('Invalid response from GitHub API')
        latestVersion = response.data.tag_name.replace(/^v/, '').trim()
      }

      logger.info(`Current version: ${currentVersion}, Latest version: ${latestVersion}`)

      const updateAvailable =
        process.env.NODE_ENV === 'development'
          ? false
          : isNewerVersion(latestVersion, currentVersion.trim(), earlyAccess)

      // Cache the results in KVStore for frontend checks
      await KVStore.setValue('system.updateAvailable', updateAvailable)
      await KVStore.setValue('system.latestVersion', latestVersion)

      return {
        success: true,
        updateAvailable,
        currentVersion,
        latestVersion,
      }
    } catch (error) {
      logger.error('Error checking latest version:', error)
      return {
        success: false,
        updateAvailable: false,
        currentVersion: '',
        latestVersion: '',
        message: `Failed to check latest version: ${error instanceof Error ? error.message : error}`,
      }
    }
  }

  async subscribeToReleaseNotes(email: string): Promise<{ success: boolean; message: string }> {
    try {
      const response = await axios.post(
        'https://api.projectnomad.us/api/v1/lists/release-notes/subscribe',
        { email },
        { timeout: 5000 }
      )

      if (response.status === 200) {
        return {
          success: true,
          message: 'Successfully subscribed to release notes',
        }
      }

      return {
        success: false,
        message: `Failed to subscribe: ${response.statusText}`,
      }
    } catch (error) {
      logger.error('Error subscribing to release notes:', error)
      return {
        success: false,
        message: `Failed to subscribe: ${error instanceof Error ? error.message : error}`,
      }
    }
  }

  async getDebugInfo(): Promise<string> {
    const appVersion = SystemService.getAppVersion()
    const environment = process.env.NODE_ENV || 'unknown'

    const [systemInfo, services, internetStatus, versionCheck] = await Promise.all([
      this.getSystemInfo(),
      this.getServices({ installedOnly: false }),
      this.getInternetStatus().catch(() => null),
      this.checkLatestVersion().catch(() => null),
    ])

    // Diagnostics tied to common support cases: storage relocation (#1050),
    // container/updater issues (#858), GPU passthrough (#755/#878), and the
    // auto-update trilogy. All best-effort so a single failure never blanks the
    // whole bundle.
    const [dockerVersion, hostStorageRoot, kiwixBookCount, gpuType] = await Promise.all([
      this.dockerService.docker
        .version()
        .then((v: any) => v?.Version ?? null)
        .catch(() => null),
      this.dockerService.getHostStorageRoot().catch(() => null),
      new KiwixLibraryService().getBookCount().catch(() => null),
      KVStore.getValue('gpu.type').catch(() => null),
    ])
    const [autoUpdateCore, autoUpdateApps, autoUpdateContent, autoDisabledReason] =
      await Promise.all([
        KVStore.getValue('autoUpdate.enabled').catch(() => null),
        KVStore.getValue('appAutoUpdate.enabled').catch(() => null),
        KVStore.getValue('contentAutoUpdate.enabled').catch(() => null),
        KVStore.getValue('autoUpdate.autoDisabledReason').catch(() => null),
      ])
    const isEnabled = (v: any) => v === true || v === 'true'

    const lines: string[] = [
      'Project NOMAD Debug Info',
      '========================',
      `App Version: ${appVersion}`,
      `Environment: ${environment}`,
    ]

    if (systemInfo) {
      const { cpu, mem, os, disk, fsSize, uptime, graphics, gpuHealth } = systemInfo

      lines.push('')
      lines.push('System:')
      if (os.distro) lines.push(`  OS: ${os.distro}`)
      if (os.hostname) lines.push(`  Hostname: ${os.hostname}`)
      if (os.kernel) lines.push(`  Kernel: ${os.kernel}`)
      if (os.arch) lines.push(`  Architecture: ${os.arch}`)
      if (dockerVersion) lines.push(`  Docker Engine: ${dockerVersion}`)
      if (uptime?.uptime) lines.push(`  Uptime: ${this._formatUptime(uptime.uptime)}`)

      lines.push('')
      lines.push('Hardware:')
      if (cpu.brand) {
        lines.push(`  CPU: ${cpu.brand} (${cpu.cores} cores)`)
      }
      if (mem.total) {
        const total = this._formatBytes(mem.total)
        const used = this._formatBytes(mem.total - (mem.available || 0))
        const available = this._formatBytes(mem.available || 0)
        lines.push(`  RAM: ${total} total, ${used} used, ${available} available`)
      }
      if (graphics.controllers && graphics.controllers.length > 0) {
        for (const gpu of graphics.controllers) {
          const vram = gpu.vram ? ` (${gpu.vram} MB VRAM)` : ''
          lines.push(`  GPU: ${gpu.model}${vram}`)
        }
      } else {
        lines.push('  GPU: None detected')
      }
      if (gpuHealth?.status) {
        const vendor = gpuType || gpuHealth.gpuVendor
        lines.push(`  GPU Passthrough: ${gpuHealth.status}${vendor ? ` (${vendor})` : ''}`)
      }

      // Disk info — try disk array first, fall back to fsSize
      const diskEntries = disk.filter((d) => d.totalSize > 0)
      if (diskEntries.length > 0) {
        for (const d of diskEntries) {
          const size = this._formatBytes(d.totalSize)
          const type = d.tran?.toUpperCase() || (d.rota ? 'HDD' : 'SSD')
          lines.push(`  Disk: ${size}, ${Math.round(d.percentUsed)}% used, ${type}`)
        }
      } else if (fsSize.length > 0) {
        const realFs = fsSize.filter((f) => f.fs.startsWith('/dev/'))
        const seen = new Set<number>()
        for (const f of realFs) {
          if (seen.has(f.size)) continue
          seen.add(f.size)
          lines.push(`  Disk: ${this._formatBytes(f.size)}, ${Math.round(f.use)}% used`)
        }
      }
    }

    lines.push('')
    lines.push('Storage:')
    lines.push(`  Host storage root: ${hostStorageRoot ?? 'unknown'}`)
    lines.push(`  Container path: ${DockerService.ADMIN_STORAGE_DEST}`)
    const storageEnv = process.env.NOMAD_STORAGE_PATH
    lines.push(
      `  NOMAD_STORAGE_PATH: ${storageEnv ? storageEnv : 'not set (auto-detected from admin mount)'}`
    )
    if (kiwixBookCount !== null) {
      lines.push(
        `  Kiwix library: ${kiwixBookCount === 0 ? 'empty (0 books)' : `${kiwixBookCount} book(s)`}`
      )
    }

    const installed = services.filter((s) => s.installed)
    lines.push('')
    if (installed.length > 0) {
      lines.push('Installed Services:')
      for (const svc of installed) {
        lines.push(`  ${svc.friendly_name} (${svc.service_name}): ${svc.status}`)
      }
    } else {
      lines.push('Installed Services: None')
    }

    if (internetStatus !== null) {
      lines.push('')
      lines.push(`Internet Status: ${internetStatus ? 'Online' : 'Offline'}`)
    }

    if (versionCheck?.success) {
      const updateMsg = versionCheck.updateAvailable
        ? `Yes (${versionCheck.latestVersion} available)`
        : `No (${versionCheck.currentVersion} is latest)`
      lines.push(`Update Available: ${updateMsg}`)
    }

    lines.push('')
    lines.push('Auto-Update:')
    lines.push(`  Core: ${isEnabled(autoUpdateCore) ? 'Enabled' : 'Disabled'}`)
    lines.push(`  Apps: ${isEnabled(autoUpdateApps) ? 'Enabled' : 'Disabled'}`)
    lines.push(`  Content: ${isEnabled(autoUpdateContent) ? 'Enabled' : 'Disabled'}`)
    if (autoDisabledReason) {
      lines.push(`  Auto-disabled reason: ${autoDisabledReason}`)
    }

    return lines.join('\n')
  }

  private _formatUptime(seconds: number): string {
    const days = Math.floor(seconds / 86400)
    const hours = Math.floor((seconds % 86400) / 3600)
    const minutes = Math.floor((seconds % 3600) / 60)
    if (days > 0) return `${days}d ${hours}h ${minutes}m`
    if (hours > 0) return `${hours}h ${minutes}m`
    return `${minutes}m`
  }

  private _formatBytes(bytes: number, decimals = 1): string {
    if (bytes === 0) return '0 Bytes'
    const k = 1024
    const sizes = ['Bytes', 'KB', 'MB', 'GB', 'TB']
    const i = Math.floor(Math.log(bytes) / Math.log(k))
    return Number.parseFloat((bytes / Math.pow(k, i)).toFixed(decimals)) + ' ' + sizes[i]
  }

  async updateSetting(key: KVStoreKey, value: any): Promise<void> {
    if (
      (value === '' || value === undefined || value === null) &&
      KV_STORE_SCHEMA[key] === 'string'
    ) {
      await KVStore.clearValue(key)
    } else {
      await KVStore.setValue(key, value)
    }
    if (key === 'ai.assistantCustomName') {
      invalidateAssistantNameCache()
    }
    if (key === 'rag.minRelevance') {
      invalidateMinRelevanceCache()
    }
    if (key === 'ai.responseStyle') {
      invalidateResponseStyleCache()
    }
    // Re-enabling auto-update after a backoff-driven auto-disable clears the
    // failure state so it gets a fresh start instead of immediately re-tripping.
    if (key === 'autoUpdate.enabled' && (value === true || value === 'true')) {
      await KVStore.setValue('autoUpdate.consecutiveFailures', '0')
      await KVStore.clearValue('autoUpdate.autoDisabledReason')
    }
    // Re-enabling the global app auto-update master switch clears every app's
    // per-app failure backoff so previously self-disabled apps get a fresh start.
    if (key === 'appAutoUpdate.enabled' && (value === true || value === 'true')) {
      await Service.query().update({
        auto_update_consecutive_failures: 0,
        auto_update_disabled_reason: null,
      })
    }
    // Re-enabling content auto-update clears the feature-level backoff and every
    // resource's per-resource backoff so previously self-disabled content gets a
    // fresh start.
    if (key === 'contentAutoUpdate.enabled' && (value === true || value === 'true')) {
      await KVStore.setValue('contentAutoUpdate.consecutiveFailures', '0')
      await KVStore.clearValue('contentAutoUpdate.autoDisabledReason')
      await InstalledResource.query().update({
        auto_update_consecutive_failures: 0,
        auto_update_disabled_reason: null,
      })
    }
  }

  /**
   * Checks the current state of Docker containers against the database records and updates the database accordingly.
   * It will mark services as not installed if their corresponding containers do not exist, regardless of their running state.
   * Handles cases where a container might have been manually removed, ensuring the database reflects the actual existence of containers.
   * Containers that exist but are stopped, paused, or restarting will still be considered installed.
   * Returns the fetched service status list so callers can reuse it without a second Docker API call.
   */
  private async _syncContainersWithDatabase(): Promise<{ service_name: string; status: string }[]> {
    try {
      const allServices = await Service.all()
      const serviceStatusList = await this.dockerService.getServicesStatus()

      for (const service of allServices) {
        // Link tiles are shortcuts with no container behind them, so container
        // reconciliation does not apply: they are always "installed" in the only
        // sense that matters, which is that the dashboard should show them.
        // Without this they are marked not-installed on the next sync and vanish.
        if (service.is_link_tile) continue

        const containerExists = serviceStatusList.find(
          (s) => s.service_name === service.service_name
        )

        if (service.installed) {
          // If marked as installed but container doesn't exist, mark as not installed
          if (!containerExists) {
            // Exception: remote Ollama is configured without a local container — don't reset it
            if (service.service_name === SERVICE_NAMES.OLLAMA) {
              const remoteUrl = await KVStore.getValue('ai.remoteOllamaUrl')
              if (remoteUrl) continue
            }
            logger.warn(
              `Service ${service.service_name} is marked as installed but container does not exist. Marking as not installed.`
            )
            service.installed = false
            service.installation_status = 'idle'
            await service.save()
          }
        } else {
          // If marked as not installed but container exists (any state), mark as installed
          if (containerExists) {
            logger.warn(
              `Service ${service.service_name} is marked as not installed but container exists. Marking as installed.`
            )
            service.installed = true
            service.installation_status = 'idle'
            await service.save()
          }
        }
      }

      return serviceStatusList
    } catch (error) {
      logger.error('Error syncing containers with database:', error)
      return []
    }
  }

  private calculateDiskUsage(diskInfo: NomadDiskInfoRaw): NomadDiskInfo[] {
    const { diskLayout, fsSize } = diskInfo

    if (!diskLayout?.blockdevices || !fsSize) {
      return []
    }

    // Deduplicate: same device path mounted in multiple places (Docker bind-mounts)
    // Keep the entry with the largest size — that's the real partition
    const deduped = new Map<string, NomadDiskInfoRaw['fsSize'][0]>()
    for (const entry of fsSize) {
      const existing = deduped.get(entry.fs)
      if (!existing || entry.size > existing.size) {
        deduped.set(entry.fs, entry)
      }
    }
    const dedupedFsSize = Array.from(deduped.values())

    return diskLayout.blockdevices
      .filter((disk) => disk.type === 'disk') // Only physical disks
      .map((disk) => {
        const filesystems = getAllFilesystems(disk, dedupedFsSize)

        // Across all partitions
        const totalUsed = filesystems.reduce((sum, p) => sum + (p.used || 0), 0)
        const totalSize = filesystems.reduce((sum, p) => sum + (p.size || 0), 0)
        const percentUsed = totalSize > 0 ? (totalUsed / totalSize) * 100 : 0

        return {
          name: disk.name,
          model: disk.model || 'Unknown',
          vendor: disk.vendor || '',
          rota: disk.rota || false,
          tran: disk.tran || '',
          size: disk.size,
          totalUsed,
          totalSize,
          percentUsed: Math.round(percentUsed * 100) / 100,
          filesystems: filesystems.map((p) => ({
            fs: p.fs,
            mount: p.mount,
            used: p.used,
            size: p.size,
            percentUsed: p.use,
          })),
        }
      })
  }

  /**
   * Check whether the host has enough free memory and disk to comfortably run an app.
   * Returns an array of human-readable warning strings; an empty array means no concerns.
   * These are advisory only — the caller decides whether to block or warn.
   */
  async checkResourceWarnings(minMemoryMB: number, minDiskMB: number): Promise<string[]> {
    const warnings: string[] = []

    try {
      const mem = await si.mem()
      const availableMB = Math.floor(mem.available / 1024 / 1024)
      if (availableMB < minMemoryMB) {
        warnings.push(
          `Low memory: ${availableMB} MB available, this app recommends at least ${minMemoryMB} MB free.`
        )
      }
    } catch (err: any) {
      logger.warn(`[SystemService] checkResourceWarnings mem check failed: ${err.message}`)
    }

    try {
      const storagePath = env.get('NOMAD_STORAGE_PATH', '/opt/project-nomad/storage')
      const fsSizes = await si.fsSize()
      // Find the filesystem whose mount point is the longest prefix of storagePath
      const fs = fsSizes
        .filter((f) => storagePath.startsWith(f.mount))
        .sort((a, b) => b.mount.length - a.mount.length)[0]

      if (fs) {
        const availableDiskMB = Math.floor((fs.size - fs.used) / 1024 / 1024)
        if (availableDiskMB < minDiskMB) {
          warnings.push(
            `Low disk space: ${availableDiskMB} MB available on ${fs.mount}, this app recommends at least ${minDiskMB} MB free.`
          )
        }
      }
    } catch (err: any) {
      logger.warn(`[SystemService] checkResourceWarnings disk check failed: ${err.message}`)
    }

    return warnings
  }

  /**
   * Return the next suggested host port for a custom app in the 8600+ range.
   * Looks at existing custom service records and all Docker container port bindings.
   */
  async getNextSuggestedCustomPort(): Promise<number> {
    const CUSTOM_PORT_START = 8600
    const occupied = new Set<number>()

    try {
      // Ports used by existing custom services in the DB
      const customServices = await Service.query().where('is_custom', true)
      for (const svc of customServices) {
        const config = svc.container_config ? JSON.parse(svc.container_config) : null
        const bindings = config?.HostConfig?.PortBindings ?? {}
        for (const binding of Object.values(bindings) as any[]) {
          const port = parseInt(binding?.[0]?.HostPort, 10)
          if (!isNaN(port)) occupied.add(port)
        }
      }

      // Ports used by any running Docker container in the 8600+ range
      const containers = await this.dockerService.docker.listContainers({ all: true })
      for (const c of containers) {
        for (const p of c.Ports) {
          if (p.PublicPort && p.PublicPort >= CUSTOM_PORT_START) occupied.add(p.PublicPort)
        }
      }
    } catch (err: any) {
      logger.warn(`[SystemService] getNextSuggestedCustomPort probe failed: ${err.message}`)
    }

    let candidate = CUSTOM_PORT_START
    while (occupied.has(candidate)) candidate += 10
    return candidate
  }
}