Skip to content

Project NOMAD is an offline-first knowledge and education server. Wikipedia, thousands of books, courses, maps, and optional local AI, all running on hardware you own with no internet required.

TypeScript
24.0 KB
Raw Blame
import logger from '@adonisjs/core/services/logger'
import { DateTime } from 'luxon'
import KVStore from '#models/kv_store'
import InstalledResource from '#models/installed_resource'
import { DownloadService } from '#services/download_service'
import { CollectionUpdateService } from '#services/collection_update_service'
import { DrugReferenceService } from '#services/drug_reference_service'
import {
  KiwixCatalogService,
  reconcileResourceUpdateState,
  type CatalogResult,
} from '#services/kiwix_catalog_service'
import { isWithinWindow, parseWindowMinutes } from '../utils/update_window.js'
import { recordResourceUpdateFailure } from '../utils/content_auto_update_backoff.js'
import type { Blocker, PreflightResult } from '../utils/image_disk_preflight.js'

/**
 * Content auto-update is opt-in via a single global master switch and runs on
 * its OWN window + per-window data cap (deliberately separate from the core/app
 * `autoUpdate.*` window, since ZIM downloads are multi-GB and bandwidth
 * sensitive). Defaults err toward an overnight window with no cap; the UI
 * strongly recommends setting a cap.
 */
const DEFAULT_WINDOW_START = '02:00'
const DEFAULT_WINDOW_END = '05:00'
const DEFAULT_COOLOFF_HOURS = 72

/** Whole-feature failures (e.g. catalog unreachable) before it self-disables. */
const MAX_FEATURE_FAILURES = 3

export interface ContentAutoUpdateConfig {
  /** Global master switch (`contentAutoUpdate.enabled`). */
  enabled: boolean
  windowStart: string
  windowEnd: string
  cooloffHours: number
  /** Max NEW bytes initiated per window instance. 0 = unlimited. */
  maxBytesPerWindow: number
}

/** Per-resource eligibility verdict (drives both selection and the status UI). */
export interface ContentEligibility {
  eligible: boolean
  reason: string
  cooloffRemainingHours: number | null
}

/** An eligible resource paired with the catalog facts needed to download it. */
export interface ContentCandidate {
  resource: InstalledResource
  version: string
  download_url: string
  size_bytes: number
  installed_at: DateTime
}

/** Outcome of the cap-bounded greedy selection. */
export interface ContentSelection {
  selected: ContentCandidate[]
  /** Single files larger than the whole cap — never auto-started (manual only). */
  skippedOversize: ContentCandidate[]
  /** Fit the cap but not this window's remaining budget — retried next window. */
  deferred: ContentCandidate[]
}

export interface ContentAutoUpdateResourceStatus {
  resource_id: string
  resource_type: 'zim' | 'map'
  current_version: string
  available_update_version: string | null
  size_bytes: number | null
  eligible: boolean
  reason: string
  cooloff_remaining_hours: number | null
  exceeds_cap: boolean
  consecutive_failures: number
  auto_disabled_reason: string | null
}

export interface ContentAutoUpdateStatus extends ContentAutoUpdateConfig {
  withinWindow: boolean
  windowBytesUsed: number
  lastAttemptAt: string | null
  lastResult: string | null
  lastError: string | null
  autoDisabledReason: string | null
  resources: ContentAutoUpdateResourceStatus[]
}

/**
 * Decision + safety layer for automatic content (ZIM/map) updates. This is the
 * content-side counterpart to {@link AppAutoUpdateService}: it decides *whether*
 * each installed resource with an available update should be downloaded now
 * (master switch on + in the content window + past cool-off + within the data
 * cap) and then drives the existing manual download path
 * ({@link CollectionUpdateService.applyUpdate} → {@link RunDownloadJob}).
 *
 * It never installs synchronously — it dispatches resumable download jobs and
 * lets the existing job-completion path advance the installed version and
 * rebuild the Kiwix library.
 */
export class ContentAutoUpdateService {
  constructor(
    private downloadService: DownloadService,
    private catalog: KiwixCatalogService = new KiwixCatalogService(),
    private collectionUpdateService: CollectionUpdateService = new CollectionUpdateService()
  ) {}

  /** Read the master switch plus the content-specific window/cool-off/cap. */
  async getConfig(): Promise<ContentAutoUpdateConfig> {
    const [enabled, windowStart, windowEnd, cooloffHours, maxBytes] = await Promise.all([
      KVStore.getValue('contentAutoUpdate.enabled'),
      KVStore.getValue('contentAutoUpdate.windowStart'),
      KVStore.getValue('contentAutoUpdate.windowEnd'),
      KVStore.getValue('contentAutoUpdate.cooloffHours'),
      KVStore.getValue('contentAutoUpdate.maxBytesPerWindow'),
    ])

    const parsedCooloff = Number(cooloffHours)
    const parsedCap = Number(maxBytes)
    return {
      enabled: enabled ?? false,
      windowStart: windowStart || DEFAULT_WINDOW_START,
      windowEnd: windowEnd || DEFAULT_WINDOW_END,
      // `Number(null) === 0`, so an unset value must fall through to the default
      // rather than silently resolving to a zero cool-off. An explicit 0 is honored.
      cooloffHours:
        cooloffHours !== null && Number.isFinite(parsedCooloff) && parsedCooloff >= 0
          ? parsedCooloff
          : DEFAULT_COOLOFF_HOURS,
      maxBytesPerWindow:
        maxBytes !== null && Number.isFinite(parsedCap) && parsedCap >= 0 ? parsedCap : 0,
    }
  }

  /**
   * Pure per-resource eligibility verdict. A resource is eligible when it has a
   * detected newer version, is not self-disabled, and is past its cool-off
   * (measured from first-detected). Version comparison is a lexicographic
   * compare of the YYYY-MM stamps, which sorts chronologically.
   */
  resourceEligibility(
    resource: InstalledResource,
    cooloffHours: number,
    now: DateTime
  ): ContentEligibility {
    if (!resource.available_update_version) {
      return { eligible: false, reason: 'Up to date', cooloffRemainingHours: null }
    }
    if (resource.auto_update_disabled_reason) {
      return {
        eligible: false,
        reason: 'Auto-update disabled after repeated failures',
        cooloffRemainingHours: null,
      }
    }
    if (!(resource.available_update_version > resource.version)) {
      return { eligible: false, reason: 'Up to date', cooloffRemainingHours: null }
    }
    if (!resource.available_update_first_seen_at) {
      return { eligible: false, reason: 'Cool-off pending', cooloffRemainingHours: cooloffHours }
    }

    const ageHours = now.diff(resource.available_update_first_seen_at, 'hours').hours
    const remaining = cooloffHours - ageHours
    if (remaining > 0) {
      const rounded = Math.ceil(remaining)
      return {
        eligible: false,
        reason: `In cool-off (${rounded}h remaining)`,
        cooloffRemainingHours: rounded,
      }
    }

    return {
      eligible: true,
      reason: `Eligible → ${resource.available_update_version}`,
      cooloffRemainingHours: 0,
    }
  }

  /**
   * Pure cap-bounded greedy selection. Oldest-installed first (stale content is
   * prioritized), tie-broken smallest-first for predictability.
   *
   * - size unknown (0) → deferred (can't budget safely)
   * - size > the WHOLE cap → skippedOversize (never auto-started; manual only)
   * - size ≤ remaining budget → selected
   * - otherwise → deferred (fits the cap, not this window)
   */
  selectUnderCap(
    candidates: ContentCandidate[],
    capBytes: number,
    usedBytes: number
  ): ContentSelection {
    const cap = capBytes > 0 ? capBytes : Number.POSITIVE_INFINITY
    let remaining = Math.max(0, cap - usedBytes)

    const selected: ContentCandidate[] = []
    const skippedOversize: ContentCandidate[] = []
    const deferred: ContentCandidate[] = []

    const ordered = [...candidates].sort((a, b) => {
      const at = a.installed_at?.toMillis?.() ?? 0
      const bt = b.installed_at?.toMillis?.() ?? 0
      if (at !== bt) return at - bt
      return a.size_bytes - b.size_bytes
    })

    for (const candidate of ordered) {
      if (candidate.size_bytes <= 0) {
        deferred.push(candidate)
      } else if (candidate.size_bytes > cap) {
        skippedOversize.push(candidate)
      } else if (candidate.size_bytes <= remaining) {
        selected.push(candidate)
        remaining -= candidate.size_bytes
      } else {
        deferred.push(candidate)
      }
    }

    return { selected, skippedOversize, deferred }
  }

  /**
   * Run-wide pre-flight: never auto-update content while ANY download is already
   * running. Because content downloads are multi-GB and resumable, an in-flight
   * download from a prior window naturally blocks new starts here — exactly the
   * "let in-flight finish, don't start new" behavior we want. Transient → `skip`.
   */
  async runGlobalPreflight(): Promise<PreflightResult> {
    const blockers: Blocker[] = []
    try {
      const downloads = await this.downloadService.listDownloadJobs()
      const active = downloads.filter(
        (d) => !!d.status && ['waiting', 'active', 'delayed'].includes(d.status)
      )
      if (active.length > 0) {
        blockers.push({ reason: `${active.length} download(s) in progress`, severity: 'skip' })
      }
    } catch (error) {
      const message = error instanceof Error ? error.message : String(error)
      logger.warn(`[ContentAutoUpdateService] Could not check active downloads: ${message}`)
    }
    return { ok: blockers.length === 0, blockers }
  }

  /**
   * Entry point invoked by ContentAutoUpdateJob. Gates on the master switch +
   * window, runs the local catalog check, then downloads as many eligible
   * resources as fit under the per-window data cap.
   */
  async attempt(): Promise<{ started: number; reason: string }> {
    const config = await this.getConfig()
    const now = DateTime.now()

    if (!config.enabled) {
      return { started: 0, reason: 'Content auto-update is disabled' }
    }
    if (!isWithinWindow(config.windowStart, config.windowEnd, now)) {
      const reason = `Outside update window (${config.windowStart}-${config.windowEnd})`
      await this.recordRun(reason)
      return { started: 0, reason }
    }

    try {
      // Reset the per-window budget once per window instance (the cron fires
      // hourly but a window can span several hours).
      await this.maybeResetWindowBudget(config, now)

      // Local catalog check + persist available-update state for every resource.
      // ZIM/map catalog path only — `dataset` resources are excluded here because
      // their freshness key (openFDA `export_date`) and apply path (the drug
      // download/ingest chain) don't fit the catalog-version model. They refresh
      // via `attemptDrugDataset()`, which the same content-update job runs under
      // the same master switch + window.
      const installed = await InstalledResource.query().whereNot('resource_type', 'dataset')
      const latestByKey = await this.catalog.getLatestForResources(
        // `dataset` rows are filtered out above, so the type narrows to ZIM/map.
        installed.map((r) => ({
          resource_id: r.resource_id,
          resource_type: r.resource_type as 'zim' | 'map',
        }))
      )
      for (const resource of installed) {
        const latest = latestByKey.get(`${resource.resource_type}:${resource.resource_id}`) ?? null
        await reconcileResourceUpdateState(resource, latest, now)
      }

      const eligible = installed.filter(
        (r) => this.resourceEligibility(r, config.cooloffHours, now).eligible
      )
      if (eligible.length === 0) {
        await this.recordFeatureSuccess()
        const reason = 'No eligible content updates'
        await this.recordRun(reason)
        return { started: 0, reason }
      }

      const global = await this.runGlobalPreflight()
      if (!global.ok) {
        await this.recordFeatureSuccess()
        const reason = `Pre-flight blocked: ${global.blockers.map((b) => b.reason).join('; ')}`
        await this.recordRun(reason)
        return { started: 0, reason }
      }

      const candidates: ContentCandidate[] = eligible.map((r) => {
        const latest = latestByKey.get(`${r.resource_type}:${r.resource_id}`) as CatalogResult
        return {
          resource: r,
          version: latest.version,
          download_url: latest.download_url,
          size_bytes: r.available_update_size_bytes ?? latest.size_bytes ?? 0,
          installed_at: r.installed_at,
        }
      })

      const usedBytes = await this.getWindowBytesUsed()
      const { selected, skippedOversize, deferred } = this.selectUnderCap(
        candidates,
        config.maxBytesPerWindow,
        usedBytes
      )

      let started = 0
      let failed = 0
      let initiatedBytes = 0
      for (const candidate of selected) {
        const result = await this.collectionUpdateService.applyUpdate(
          {
            resource_id: candidate.resource.resource_id,
            // `dataset` rows are filtered out of `installed` above, so ZIM/map.
            resource_type: candidate.resource.resource_type as 'zim' | 'map',
            installed_version: candidate.resource.version,
            latest_version: candidate.version,
            download_url: candidate.download_url,
            size_bytes: candidate.size_bytes || undefined,
          },
          { auto: true }
        )

        if (result.success) {
          // Success is NOT recorded here: applyUpdate only enqueues a resumable
          // download. The per-resource backoff is cleared once the download
          // actually completes (RunDownloadJob.onComplete) and incremented when it
          // fails terminally (the worker `failed` handler). Recording success on
          // dispatch would reset the counter every window and defeat self-disable.
          initiatedBytes += candidate.size_bytes
          started++
          logger.info(
            `[ContentAutoUpdateService] Started ${candidate.resource.resource_id} → ${candidate.version}`
          )
        } else {
          // A failure to even enqueue is a genuine auto-update failure; no job runs,
          // so no terminal `failed` event will follow — count it here.
          await recordResourceUpdateFailure(candidate.resource, result.error ?? 'dispatch failed')
          failed++
        }
      }

      if (initiatedBytes > 0) {
        await this.addWindowBytesUsed(initiatedBytes)
      }

      const parts = [`${started} started`]
      if (failed) parts.push(`${failed} failed`)
      if (skippedOversize.length) parts.push(`${skippedOversize.length} skipped (exceeds cap)`)
      if (deferred.length) parts.push(`${deferred.length} deferred (over budget)`)
      const reason = parts.join(', ')

      await this.recordFeatureSuccess()
      await this.recordRun(reason)
      logger.info(`[ContentAutoUpdateService] Run complete: ${reason}`)
      return { started, reason }
    } catch (error) {
      const message = error instanceof Error ? error.message : String(error)
      await this.recordFeatureFailure(message)
      await this.recordRun(`Failed: ${message}`)
      logger.error(`[ContentAutoUpdateService] Run failed: ${message}`)
      return { started: 0, reason: `Failed: ${message}` }
    }
  }

  /**
   * Freshness pass for the FDA drug dataset (`resource_type` 'dataset'), run by
   * the same hourly ContentAutoUpdateJob as the ZIM/map `attempt()` and gated on
   * the SAME master switch + window. The dataset's apply path differs from the
   * catalog loop (openFDA `export_date` + the drug download/ingest chain rather
   * than an InstalledResource catalog-version row), so it runs as its own step
   * instead of through `selectUnderCap` — but sharing `contentAutoUpdate.*` means
   * it never updates while content auto-update is off, and refreshes alongside
   * ZIMs and maps when it is on. Isolated so a drug-side failure can't affect the
   * catalog run.
   */
  async attemptDrugDataset(): Promise<{ started: number; reason: string }> {
    const config = await this.getConfig()
    if (!config.enabled) {
      return { started: 0, reason: 'Content auto-update is disabled' }
    }
    if (!isWithinWindow(config.windowStart, config.windowEnd, DateTime.now())) {
      return {
        started: 0,
        reason: `Outside update window (${config.windowStart}-${config.windowEnd})`,
      }
    }

    try {
      const result = await new DrugReferenceService().attemptAutoUpdate()
      return { started: result.started ? 1 : 0, reason: `drug dataset: ${result.reason}` }
    } catch (error) {
      const message = error instanceof Error ? error.message : String(error)
      logger.warn(`[ContentAutoUpdateService] Drug dataset freshness check failed: ${message}`)
      return { started: 0, reason: `drug dataset check failed: ${message}` }
    }
  }

  /**
   * Evaluate what the next run *would* do, without hitting the network,
   * persisting state, or dispatching anything. Operates on the available-update
   * state already persisted by the last check (manual or auto), so run a "Check
   * for Content Updates" first if you want fresh catalog data. Used by the
   * `content-auto-update:dry-run` command.
   */
  async dryRun(
    overrides: {
      now?: DateTime
      forceEnabled?: boolean
      cooloffHours?: number
      windowStart?: string
      windowEnd?: string
      maxBytesPerWindow?: number
      windowBytesUsed?: number
    } = {}
  ): Promise<{
    enabled: boolean
    withinWindow: boolean
    config: ContentAutoUpdateConfig
    eligibleCount: number
    selection: ContentSelection
  }> {
    const base = await this.getConfig()
    const config: ContentAutoUpdateConfig = {
      enabled: overrides.forceEnabled ? true : base.enabled,
      windowStart: overrides.windowStart ?? base.windowStart,
      windowEnd: overrides.windowEnd ?? base.windowEnd,
      cooloffHours: overrides.cooloffHours ?? base.cooloffHours,
      maxBytesPerWindow: overrides.maxBytesPerWindow ?? base.maxBytesPerWindow,
    }
    const now = overrides.now ?? DateTime.now()
    const withinWindow = isWithinWindow(config.windowStart, config.windowEnd, now)

    const pending = await InstalledResource.query()
      .whereNotNull('available_update_version')
      .whereNot('resource_type', 'dataset')
    const eligible = pending.filter(
      (r) => this.resourceEligibility(r, config.cooloffHours, now).eligible
    )
    const candidates: ContentCandidate[] = eligible.map((r) => ({
      resource: r,
      version: r.available_update_version!,
      download_url: '(dry-run)',
      size_bytes: r.available_update_size_bytes ?? 0,
      installed_at: r.installed_at,
    }))

    const usedBytes = overrides.windowBytesUsed ?? (await this.getWindowBytesUsed())
    const selection = this.selectUnderCap(candidates, config.maxBytesPerWindow, usedBytes)

    return {
      enabled: config.enabled,
      withinWindow,
      config,
      eligibleCount: eligible.length,
      selection,
    }
  }

  // ── Per-window budget ─────────────────────────────────────────────────────────

  /** Most-recent window-open boundary as an absolute timestamp (handles wrap). */
  windowStartBoundary(windowStart: string, now: DateTime): DateTime {
    const minutes = parseWindowMinutes(windowStart) ?? 0
    const todayStart = now.startOf('day').plus({ minutes })
    return now >= todayStart ? todayStart : todayStart.minus({ days: 1 })
  }

  /** Reset the window budget exactly once per entry into the window. */
  private async maybeResetWindowBudget(
    config: ContentAutoUpdateConfig,
    now: DateTime
  ): Promise<void> {
    const boundary = this.windowStartBoundary(config.windowStart, now)
    const resetAtRaw = await KVStore.getValue('contentAutoUpdate.windowResetAt')
    const resetAt = resetAtRaw ? DateTime.fromISO(resetAtRaw) : null
    if (!resetAt || !resetAt.isValid || resetAt < boundary) {
      await KVStore.setValue('contentAutoUpdate.windowBytesUsed', '0')
      await KVStore.setValue('contentAutoUpdate.windowResetAt', now.toISO()!)
    }
  }

  private async getWindowBytesUsed(): Promise<number> {
    const raw = await KVStore.getValue('contentAutoUpdate.windowBytesUsed')
    const num = Number(raw)
    return Number.isFinite(num) && num > 0 ? num : 0
  }

  private async addWindowBytesUsed(bytes: number): Promise<void> {
    const used = await this.getWindowBytesUsed()
    await KVStore.setValue('contentAutoUpdate.windowBytesUsed', String(used + bytes))
  }

  // ── Backoff + run recording ───────────────────────────────────────────────────
  // Per-resource backoff lives in ../utils/content_auto_update_backoff.ts so the
  // job-completion path and the worker `failed` handler can share it without an
  // import cycle. The feature-level backoff below stays here.

  /** Clear the feature-level backoff after a clean run. */
  private async recordFeatureSuccess(): Promise<void> {
    await KVStore.setValue('contentAutoUpdate.consecutiveFailures', '0')
    await KVStore.clearValue('contentAutoUpdate.lastError')
  }

  /** Record a whole-feature failure and self-disable the feature at the threshold. */
  private async recordFeatureFailure(reason: string): Promise<void> {
    const raw = await KVStore.getValue('contentAutoUpdate.consecutiveFailures')
    const failures = (Number(raw) || 0) + 1
    await KVStore.setValue('contentAutoUpdate.consecutiveFailures', String(failures))
    await KVStore.setValue('contentAutoUpdate.lastError', reason)
    if (failures >= MAX_FEATURE_FAILURES) {
      await KVStore.setValue('contentAutoUpdate.enabled', false)
      await KVStore.setValue(
        'contentAutoUpdate.autoDisabledReason',
        `Content auto-update disabled after ${failures} consecutive failures. Last error: ${reason}`
      )
      logger.error(
        `[ContentAutoUpdateService] Feature auto-disabled after ${failures} consecutive failures`
      )
    }
  }

  private async recordRun(reason: string): Promise<void> {
    await KVStore.setValue('contentAutoUpdate.lastAttemptAt', DateTime.now().toISO()!)
    await KVStore.setValue('contentAutoUpdate.lastResult', reason)
  }

  // ── Status snapshot ───────────────────────────────────────────────────────────

  /** Full state snapshot for the settings UI (resources with pending updates). */
  async getStatus(): Promise<ContentAutoUpdateStatus> {
    const config = await this.getConfig()
    const now = DateTime.now()

    const pending = await InstalledResource.query()
      .whereNotNull('available_update_version')
      .whereNot('resource_type', 'dataset')
    const resources: ContentAutoUpdateResourceStatus[] = pending.map((resource) => {
      const verdict = this.resourceEligibility(resource, config.cooloffHours, now)
      const size = resource.available_update_size_bytes ?? null
      const exceedsCap =
        config.maxBytesPerWindow > 0 && size !== null && size > config.maxBytesPerWindow
      return {
        resource_id: resource.resource_id,
        // `dataset` rows are filtered out of `pending` above, so ZIM/map.
        resource_type: resource.resource_type as 'zim' | 'map',
        current_version: resource.version,
        available_update_version: resource.available_update_version,
        size_bytes: size,
        eligible: verdict.eligible && !exceedsCap,
        reason: exceedsCap ? 'Exceeds data cap — update manually' : verdict.reason,
        cooloff_remaining_hours: verdict.cooloffRemainingHours,
        exceeds_cap: exceedsCap,
        consecutive_failures: resource.auto_update_consecutive_failures || 0,
        auto_disabled_reason: resource.auto_update_disabled_reason,
      }
    })

    const [lastAttemptAt, lastResult, lastError, autoDisabledReason, windowBytesUsed] =
      await Promise.all([
        KVStore.getValue('contentAutoUpdate.lastAttemptAt'),
        KVStore.getValue('contentAutoUpdate.lastResult'),
        KVStore.getValue('contentAutoUpdate.lastError'),
        KVStore.getValue('contentAutoUpdate.autoDisabledReason'),
        this.getWindowBytesUsed(),
      ])

    return {
      ...config,
      withinWindow: isWithinWindow(config.windowStart, config.windowEnd, now),
      windowBytesUsed,
      lastAttemptAt: lastAttemptAt || null,
      lastResult: lastResult || null,
      lastError: lastError || null,
      autoDisabledReason: autoDisabledReason || null,
      resources,
    }
  }
}