Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

14.8 KB
Raw Blame
/**
 * Apple Health XML Import Service
 *
 * Dependency-free streaming parser for Apple Health exports (500MB+).
 * Normalizes HK identifiers to metric names matching JSON ingest format.
 * Emits WebSocket progress events every 10k records.
 */

import { createReadStream } from 'fs';
import { unlink } from 'fs/promises';
import { extractDateStr, readDayFile, writeDayFile, upsertPoints, queueHealthDayMutation } from './appleHealthIngest.js';
import { createAppleHealthRecordStream } from './appleHealthXmlParser.js';
import { bucketFor } from '../lib/objects.js';

// === Mapping Tables ===

const XML_TO_METRIC_NAME = {
  // Core vitals
  'hkquantitytypeidentifierheartrate': 'heart_rate',
  'hkquantitytypeidentifierheartratevariancessdnn': 'heart_rate_variability_sdnn',
  'hkquantitytypeidentifierrestingheartrate': 'resting_heart_rate',
  'hkquantitytypeidentifierwalkingheartrateaverage': 'walking_heart_rate_average',
  'hkquantitytypeidentifierheartraterecoveryoneminute': 'heart_rate_recovery',
  'hkquantitytypeidentifieroxygensaturation': 'blood_oxygen_saturation',
  'hkquantitytypeidentifierrespiratoryrate': 'respiratory_rate',
  'hkquantitytypeidentifierbodytemperature': 'body_temperature',
  'hkquantitytypeidentifiervo2max': 'vo2_max',

  // Activity
  'hkquantitytypeidentifierstepcount': 'step_count',
  'hkquantitytypeidentifieractiveenergyburned': 'active_energy',
  'hkquantitytypeidentifierbasalenergyburned': 'basal_energy_burned',
  'hkquantitytypeidentifierdistancewalkingrunning': 'walking_running_distance',
  'hkquantitytypeidentifierflightsclimbed': 'flights_climbed',
  'hkquantitytypeidentifierappleexercisetime': 'apple_exercise_time',
  'hkquantitytypeidentifierapplestandtime': 'apple_stand_time',
  'hkcategorytypeidentifierapplestandhour': 'apple_stand_hour',
  'hkquantitytypeidentifierphysicaleffort': 'physical_effort',
  'hkquantitytypeidentifiertimeindaylight': 'time_in_daylight',

  // Walking metrics
  'hkquantitytypeidentifierwalkingspeed': 'walking_speed',
  'hkquantitytypeidentifierwalkingsteplength': 'walking_step_length',
  'hkquantitytypeidentifierwalkingdoublesupportpercentage': 'walking_double_support_percentage',
  'hkquantitytypeidentifierwalkingasymmetrypercentage': 'walking_asymmetry_percentage',
  'hkquantitytypeidentifierstairdescentspeed': 'stair_speed_down',
  'hkquantitytypeidentifierstairascentspeed': 'stair_speed_up',
  'hkquantitytypeidentifierapplewalkingsteadiness': 'walking_steadiness',
  'hkquantitytypeidentifiersixminutewalktestdistance': 'six_minute_walk_test',

  // Running
  'hkquantitytypeidentifierrunningspeed': 'running_speed',
  'hkquantitytypeidentifierrunningpower': 'running_power',
  'hkquantitytypeidentifierrunningverticaloscillation': 'running_vertical_oscillation',
  'hkquantitytypeidentifierrunninggroundcontacttime': 'running_ground_contact_time',
  'hkquantitytypeidentifierrunningstridelength': 'running_stride_length',

  // Cycling
  'hkquantitytypeidentifierdistancecycling': 'distance_cycling',
  'hkquantitytypeidentifiercyclingspeed': 'cycling_speed',
  'hkquantitytypeidentifiercyclingcadence': 'cycling_cadence',
  'hkquantitytypeidentifiercyclingpower': 'cycling_power',
  'hkquantitytypeidentifiercyclingfunctionalthresholdpower': 'cycling_ftp',

  // Cardiovascular
  'hkquantitytypeidentifierbloodpressuresystolic': 'blood_pressure_systolic',
  'hkquantitytypeidentifierbloodpressurediastolic': 'blood_pressure_diastolic',

  // Body measurements
  'hkquantitytypeidentifierbodymass': 'body_mass',
  'hkquantitytypeidentifierbodymassindex': 'body_mass_index',
  'hkquantitytypeidentifierbodyfatpercentage': 'body_fat_percentage',
  'hkquantitytypeidentifierleanbodymass': 'lean_body_mass',
  'hkquantitytypeidentifierheight': 'height',

  // Sleep
  'hkcategorytypeidentifiersleepanalysis': 'sleep_analysis',
  'hkquantitytypeidentifierapplesleepingbreathingdisturbances': 'breathing_disturbances',

  // Audio exposure
  'hkquantitytypeidentifierenvironmentalaudioexposure': 'environmental_audio_exposure',
  'hkquantitytypeidentifierheadphoneaudioexposure': 'headphone_audio_exposure',
  'hkquantitytypeidentifierenvironmentalsoundreduction': 'environmental_sound_reduction',

  // Other categories
  'hkcategorytypeidentifierhandwashingevent': 'handwashing',
  'hkcategorytypeidentifiermindfulsession': 'mindful_session',
};

// HKQuantityTypeIdentifierHeartRateVariabilitySDNN mapped by direct lowercase below
// Using the exact HK identifier (already lowercase) as fallback for unknowns

const SLEEP_STAGE_MAP = {
  'hkcategoryvaluesleepanalysisasleepdeep': 'deep',
  'hkcategoryvaluesleepanalysisasleeprem': 'rem',
  'hkcategoryvaluesleepanalysisasleepcore': 'core',
  'hkcategoryvaluesleepanalysisawake': 'awake',
  'hkcategoryvaluesleepanalysisinbed': 'inBed',
  'hkcategoryvaluesleepanalysisasleep': 'asleep',  // Legacy pre-iOS 16
};

// === Pure Functions ===

/**
 * Normalize an XML Record node into a standard data point.
 * Uses lowercase attribute names (matching the parser's lowercase-name mode).
 *
 * @param {Object} node - Record node with lowercase attribute names
 * @returns {{ metricName: string, dateStr: string, dataPoint: Object }|null}
 */
export function normalizeXmlRecord(node) {
  const attrs = node.attributes;
  const type = attrs.type;
  const value = attrs.value;
  const startdate = attrs.startdate;
  const enddate = attrs.enddate;
  const unit = attrs.unit;
  const sourcename = attrs.sourcename;

  if (!type || !startdate) return null;

  const typeLower = type.toLowerCase();
  const metricName = XML_TO_METRIC_NAME[typeLower] ?? typeLower;

  const dateStr = extractDateStr(startdate);
  if (!dateStr) return null;

  // Sleep analysis: categorical value → duration-based stage data
  if (typeLower === 'hkcategorytypeidentifiersleepanalysis') {
    const durationHours = enddate
      ? (new Date(enddate) - new Date(startdate)) / 3600000
      : 0;
    const valueLower = value?.toLowerCase() ?? '';
    const stage = SLEEP_STAGE_MAP[valueLower] ?? value ?? 'unknown';
    const dataPoint = { date: startdate, stage, durationHours, src: sourcename ?? null, origin: 'xml' };
    return { metricName, dateStr, dataPoint };
  }

  // Heart rate: include end timestamp
  if (typeLower === 'hkquantitytypeidentifierheartrate') {
    const parsed = parseFloat(value);
    if (!Number.isFinite(parsed)) return null;
    const dataPoint = {
      date: startdate,
      qty: parsed,
      unit: unit ?? null,
      src: sourcename ?? null,
      end: enddate ?? null,
      origin: 'xml',
    };
    return { metricName, dateStr, dataPoint };
  }

  // All other numeric types
  const parsed = parseFloat(value);
  if (!Number.isFinite(parsed)) return null;
  const dataPoint = {
    date: startdate,
    qty: parsed,
    unit: unit ?? null,
    src: sourcename ?? null,
    origin: 'xml',
  };
  return { metricName, dateStr, dataPoint };
}

// === Aggregation ===

/**
 * Group data points by their `src` (source device name), preserving insertion order.
 *
 * @param {Array} points - Data points, each optionally carrying `src`
 * @returns {Map<string|null, Array>} Points grouped by src
 */
function groupBySrc(points) {
  const bySrc = new Map();
  for (const p of points) {
    const src = p.src ?? null;
    if (!bySrc.has(src)) bySrc.set(src, []);
    bySrc.get(src).push(p);
  }
  return bySrc;
}

/**
 * Aggregate step_count entries in a day bucket into one total PER SOURCE DEVICE.
 * Apple Health emits per-activity step records, often from multiple devices
 * (iPhone + Watch) covering the same walk — summing across sources would
 * double-count, so this keeps one aggregate per `src` and defers picking the
 * single source of truth to read time (see `pickDaySumPoints` in appleHealthQuery.js).
 *
 * @param {Array} points - Array of step_count data points
 * @returns {Array} One aggregated entry per source device
 */
function aggregateStepCount(points) {
  if (!points.length) return points;
  return Array.from(groupBySrc(points).entries()).map(([src, pts]) => ({
    date: pts[0].date,
    qty: pts.reduce((sum, p) => sum + (p.qty || 0), 0),
    unit: pts[0].unit ?? null,
    src,
    origin: 'xml',
  }));
}

/**
 * Aggregate sleep_analysis entries in a day bucket into one summary PER SOURCE DEVICE.
 * Multiple sleep sources (Watch, iPhone, third-party trackers) can record the
 * same night — summing across sources would double-count, so this keeps one
 * summary per `src` and defers picking the single source of truth to read
 * time (see `pickDaySumPoints` in appleHealthQuery.js).
 *
 * @param {Array} points - Array of sleep_analysis data points with stage/durationHours
 * @returns {Array} One aggregated summary per source device: { date, totalSleep, deep, rem, core, awake, inBed, asleep, src, origin }
 */
function aggregateSleepAnalysis(points) {
  if (!points.length) return points;
  return Array.from(groupBySrc(points).entries()).map(([src, pts]) => {
    const summary = { date: pts[0].date, totalSleep: 0, deep: 0, rem: 0, core: 0, awake: 0, inBed: 0, asleep: 0, src, origin: 'xml' };
    for (const p of pts) {
      const stage = p.stage;
      const dur = p.durationHours || 0;
      if (stage === 'deep') summary.deep += dur;
      else if (stage === 'rem') summary.rem += dur;
      else if (stage === 'core') summary.core += dur;
      else if (stage === 'awake') summary.awake += dur;
      else if (stage === 'inBed') summary.inBed += dur;
      else if (stage === 'asleep') summary.asleep += dur;
    }
    // totalSleep = meaningful sleep stages (deep + rem + core)
    summary.totalSleep = summary.deep + summary.rem + summary.core;
    return summary;
  });
}

// === Main Export ===

const FLUSH_INTERVAL = 200000; // Flush to disk every 200K records to stay under memory limits

/**
 * Flush accumulated day buckets to disk: aggregate, merge with existing, write, clear.
 * Uses upsert to handle re-imports that update existing day totals.
 *
 * @param {Object} dayBuckets - { [dateStr]: { [metricName]: [...dataPoints] } }
 * @returns {Promise<Set<string>>} Set of date strings that were flushed
 */
async function flushDayBuckets(dayBuckets) {
  const flushedDates = new Set();
  const allDates = Object.keys(dayBuckets);

  for (const dateStr of allDates) {
    // Admission is per day, so a live health restore can run between days and
    // every later day re-reads the restored file rather than a stale one.
    await queueHealthDayMutation(dateStr, async () => {
      const metrics = dayBuckets[dateStr];

      // Aggregate step_count: sum all qty values into single daily total
      if (metrics.step_count) {
        metrics.step_count = aggregateStepCount(metrics.step_count);
      }

      // Aggregate sleep_analysis: sum stage durations into daily summary
      if (metrics.sleep_analysis) {
        metrics.sleep_analysis = aggregateSleepAnalysis(metrics.sleep_analysis);
      }

      const dayData = await readDayFile(dateStr);

      for (const [metricName, newPoints] of Object.entries(metrics)) {
        const existing = dayData.metrics[metricName] || [];
        const { result } = upsertPoints(existing, newPoints);
        if (result.length > 0) {
          dayData.metrics[metricName] = result;
        }
      }

      await writeDayFile(dateStr, dayData, Object.keys(metrics));
      flushedDates.add(dateStr);
    });

    delete dayBuckets[dateStr];
  }

  return flushedDates;
}

/**
 * Stream-parse an Apple Health export.xml file with the dependency-free
 * `<Record>` parser. Accumulates records in batches, flushing to disk
 * periodically to avoid OOM.
 *
 * @param {string} filePath - Absolute path to the XML file (temp upload)
 * @param {Object|null} io - Socket.IO server instance for progress events
 * @returns {Promise<{ days: number, records: number }>}
 */
export async function importAppleHealthXml(filePath, io = null) {
  // dayBuckets: { [dateStr]: { [metricName]: [...dataPoints] } }
  const dayBuckets = {};
  let processedRecords = 0;
  let lastFlush = 0;
  const allDays = new Set();
  let flushing = false;

  const inputStream = createReadStream(filePath);

  try {
    await new Promise((resolve, reject) => {
      const parser = createAppleHealthRecordStream({
        onRecord: (node) => {
          const normalized = normalizeXmlRecord(node);
          if (!normalized) return;

          const { metricName, dateStr, dataPoint } = normalized;

          // dateStr and metricName both come from the uploaded export.
          const metrics = bucketFor(dayBuckets, dateStr, () => ({}));
          const points = metrics && bucketFor(metrics, metricName, () => []);
          if (!points) return;
          points.push(dataPoint);
          allDays.add(dateStr);

          processedRecords++;

          if (processedRecords % 10000 === 0) {
            io?.emit('health:xml:progress', { processed: processedRecords });
            console.log(`🍎 XML import progress: ${processedRecords} records`);
          }

          // Batch flush to prevent OOM — pause input stream, flush to disk, resume
          if (processedRecords - lastFlush >= FLUSH_INTERVAL && !flushing) {
            flushing = true;
            lastFlush = processedRecords;
            inputStream.pause();
            flushDayBuckets(dayBuckets).then(() => {
              console.log(`🍎 Flushed batch at ${processedRecords} records (${allDays.size} days total)`);
              flushing = false;
              inputStream.resume();
            }).catch(reject);
          }
        },
      });

      // Malformed records can't throw here — records missing required attributes
      // are dropped by normalizeXmlRecord, so minor XML malformations are skipped
      // rather than fatal (matching the prior sax error-skip behavior).
      parser.on('finish', resolve);
      parser.on('error', reject);

      inputStream.on('error', reject);
      inputStream.pipe(parser);
    });

    console.log(`🍎 XML parsing done: ${processedRecords} raw records across ${allDays.size} days — flushing remaining`);

    // Final flush for remaining records
    await flushDayBuckets(dayBuckets);
  } finally {
    // Runs on success AND on every rejection route (parser error, input-stream
    // error, rejected batch flush). Destroy before unlink: a batch-flush
    // rejection leaves the stream pause()d and still holding its fd, and on
    // Windows unlink fails while the file is open. An Apple Health export.xml
    // is routinely 0.5-3GB, so a leaked temp file is expensive.
    inputStream.destroy();
    // destroy() tears the fd down asynchronously, so wait for 'close' before
    // unlinking — on Windows the unlink fails while the handle is open and the
    // failure is swallowed below (same wait the ZIP write-stream cleanup does).
    if (!inputStream.closed) await new Promise((res) => inputStream.once('close', res));
    await unlink(filePath).catch(() => {});
  }

  io?.emit('health:xml:complete', { days: allDays.size, records: processedRecords });
  console.log(`🍎 XML import complete: ${processedRecords} records across ${allDays.size} days`);

  return { days: allDays.size, records: processedRecords };
}