Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

10.6 KB
Raw Blame
/**
 * Agent Activity Service
 *
 * Logs and tracks all agent activities for monitoring, analytics,
 * and rate limit enforcement. Activity is stored per-agent per-day.
 */

import { readFile, readdir, unlink } from 'fs/promises';
import { existsSync } from 'fs';
import { join } from 'path';
import EventEmitter from 'events';
import { ensureDir, getDateString, atomicWrite, PATHS } from '../lib/fileUtils.js';
import { createKeyedFileWriteQueue } from '../lib/fileWriteQueue.js';

const AGENTS_DIR = PATHS.agentPersonalities;
const ACTIVITY_DIR = join(AGENTS_DIR, 'activity');

// A day file is `<YYYY-MM-DD>.json`. Anything else in an agent directory is not
// activity and is never read as a day nor unlinked as an expired one.
const DAY_FILE_RE = /^\d{4}-\d{2}-\d{2}\.json$/;

// Each day file is a whole-file JSON read-modify-write. Keyed per agent/day so
// concurrent logActivity calls for one file serialize (no lost entries) while
// different agents or days still write in parallel.
const queueActivityWrite = createKeyedFileWriteQueue();

// Event emitter for activity events
export const activityEvents = new EventEmitter();

// Cache for today's activity (per account)
const todayCache = new Map();

async function ensureActivityDir(agentId = null) {
  await ensureDir(ACTIVITY_DIR);
  if (agentId) {
    await ensureDir(join(ACTIVITY_DIR, agentId));
  }
}

function getActivityFilePath(agentId, date = new Date()) {
  const dateStr = typeof date === 'string' ? date : getDateString(date);
  return join(ACTIVITY_DIR, agentId, `${dateStr}.json`);
}

/**
 * Load activity for a specific agent and date
 */
async function loadActivity(agentId, date = new Date()) {
  const filePath = getActivityFilePath(agentId, date);
  await ensureActivityDir(agentId);

  if (!existsSync(filePath)) {
    return { activities: [] };
  }

  const content = await readFile(filePath, 'utf-8');
  return JSON.parse(content);
}

/**
 * Save activity for a specific agent and date
 */
async function saveActivity(agentId, date, data) {
  await ensureActivityDir(agentId);
  const filePath = getActivityFilePath(agentId, date);
  await atomicWrite(filePath, data);
}

/**
 * Log an activity
 */
export async function logActivity(activity) {
  const {
    agentId,
    accountId,
    scheduleId,
    action,
    params,
    status,
    result,
    error,
    code,
    timestamp
  } = activity;

  const date = new Date(timestamp || Date.now());
  const dateStr = getDateString(date);

  const entry = {
    id: `${Date.now()}-${Math.random().toString(36).substr(2, 9)}`,
    accountId,
    scheduleId,
    action,
    params,
    status,
    result,
    error,
    ...(typeof code === 'string' && code ? { code } : {}),
    timestamp: timestamp || date.toISOString()
  };

  await queueActivityWrite(`${agentId}/${dateStr}`, async () => {
    const data = await loadActivity(agentId, date);
    data.activities.push(entry);
    await saveActivity(agentId, dateStr, data);
  });

  // Invalidate cache for this account
  todayCache.delete(`${accountId}-${action}`);

  // Emit event
  activityEvents.emit('activity', { agentId, ...entry });

  console.log(`📊 Activity logged: ${agentId}/${action} - ${status}`);
  return entry;
}

/**
 * Get activities for an agent
 */
export async function getActivities(agentId, options = {}) {
  const { date, limit = 100, offset = 0, action = null } = options;

  const targetDate = date || new Date();
  const data = await loadActivity(agentId, targetDate);

  let activities = data.activities || [];

  // Filter by action if specified
  if (action) {
    activities = activities.filter(a => a.action === action);
  }

  // Sort by timestamp descending (newest first)
  activities.sort((a, b) => new Date(b.timestamp) - new Date(a.timestamp));

  // Apply pagination
  return activities.slice(offset, offset + limit);
}

/**
 * Get recent activities across all agents
 */
export async function getRecentActivities(options = {}) {
  const { limit = 50, agentIds = null, action = null } = options;

  const today = getDateString();
  const activities = [];

  // Get list of agent directories
  await ensureActivityDir();
  let agentDirs = [];

  if (existsSync(ACTIVITY_DIR)) {
    const entries = await readdir(ACTIVITY_DIR, { withFileTypes: true });
    agentDirs = entries.filter(e => e.isDirectory()).map(e => e.name);
  }

  // Filter to specific agents if provided
  if (agentIds) {
    agentDirs = agentDirs.filter(d => agentIds.includes(d));
  }

  // Load today's activities from each agent
  for (const agentId of agentDirs) {
    const data = await loadActivity(agentId, today);
    for (const activity of data.activities || []) {
      if (!action || activity.action === action) {
        activities.push({ agentId, ...activity });
      }
    }
  }

  // Sort and limit
  activities.sort((a, b) => new Date(b.timestamp) - new Date(a.timestamp));
  return activities.slice(0, limit);
}

/**
 * Get today's action count for an account (for rate limiting)
 */
export async function getTodayActionCount(accountId, action) {
  const cacheKey = `${accountId}-${action}`;

  // Check cache first
  if (todayCache.has(cacheKey)) {
    const cached = todayCache.get(cacheKey);
    if (Date.now() - cached.timestamp < 5000) { // 5 second cache
      return cached.count;
    }
  }

  // Count from all agents for this account
  const today = getDateString();
  let count = 0;

  await ensureActivityDir();

  if (existsSync(ACTIVITY_DIR)) {
    const entries = await readdir(ACTIVITY_DIR, { withFileTypes: true });
    const agentDirs = entries.filter(e => e.isDirectory()).map(e => e.name);

    for (const agentId of agentDirs) {
      const data = await loadActivity(agentId, today);
      for (const activity of data.activities || []) {
        if (activity.accountId === accountId && activity.action === action) {
          count++;
        }
      }
    }
  }

  // Update cache
  todayCache.set(cacheKey, { count, timestamp: Date.now() });

  return count;
}

/**
 * Get activity stats for an agent
 */
export async function getAgentStats(agentId, days = 7) {
  const stats = {
    totalActivities: 0,
    byAction: {},
    byStatus: {},
    byDay: {}
  };

  const today = new Date();

  for (let i = 0; i < days; i++) {
    const date = new Date(today);
    date.setDate(date.getDate() - i);
    const dateStr = getDateString(date);

    const data = await loadActivity(agentId, date);
    const dayActivities = data.activities || [];

    stats.byDay[dateStr] = dayActivities.length;
    stats.totalActivities += dayActivities.length;

    for (const activity of dayActivities) {
      stats.byAction[activity.action] = (stats.byAction[activity.action] || 0) + 1;
      stats.byStatus[activity.status] = (stats.byStatus[activity.status] || 0) + 1;
    }
  }

  return stats;
}

/**
 * Get activity timeline for display
 */
export async function getActivityTimeline(options = {}) {
  const { agentIds = null, actions = null, limit = 100, beforeTimestamp = null } = options;
  const before = beforeTimestamp ? new Date(beforeTimestamp).getTime() : null;

  await ensureActivityDir();
  if (!existsSync(ACTIVITY_DIR)) return [];

  const entries = await readdir(ACTIVITY_DIR, { withFileTypes: true });
  const agentDirs = entries
    .filter(entry => entry.isDirectory() && (!agentIds || agentIds.includes(entry.name)))
    .map(entry => entry.name);
  const dates = new Set();

  for (const agentId of agentDirs) {
    const files = await readdir(join(ACTIVITY_DIR, agentId));
    for (const file of files) {
      if (DAY_FILE_RE.test(file)) dates.add(file.slice(0, -5));
    }
  }

  const activities = [];
  for (const date of [...dates].sort().reverse()) {
    const dayActivities = [];
    for (const agentId of agentDirs) {
      const data = await loadActivity(agentId, date);
      for (const activity of data.activities || []) {
        const timestamp = new Date(activity.timestamp).getTime();
        // Filter before the limit cut so a filtered page stays full.
        if ((before === null || timestamp < before) && (!actions || actions.includes(activity.action))) {
          dayActivities.push({ agentId, ...activity });
        }
      }
    }

    dayActivities.sort((a, b) => {
      const timestampOrder = new Date(b.timestamp) - new Date(a.timestamp);
      if (timestampOrder !== 0) return timestampOrder;
      const agentOrder = a.agentId.localeCompare(b.agentId);
      return agentOrder || String(a.id).localeCompare(String(b.id));
    });
    activities.push(...dayActivities);

    // Files are visited newest-day first. Once a complete day fills the page,
    // older files cannot contribute an entry ahead of the current window.
    if (activities.length >= limit) break;
  }

  return activities.slice(0, limit);
}

// Smallest retention window this function will honour. A caller asking for 0
// puts the cutoff at now and unlinks the ENTIRE archive; a negative value walks
// the cutoff into the future and takes today's file with it. Neither is a
// retention window, so both fall back to the default rather than deleting.
const MIN_DAYS_TO_KEEP = 1;
const DEFAULT_DAYS_TO_KEEP = 30;

/**
 * Clean up old activity files (older than N days).
 *
 * The floor lives here and not only at the HTTP boundary because schedulers
 * call this directly.
 */
export async function cleanupOldActivity(daysToKeep = DEFAULT_DAYS_TO_KEEP) {
  const days = Number.isInteger(daysToKeep) && daysToKeep >= MIN_DAYS_TO_KEEP
    ? daysToKeep
    : DEFAULT_DAYS_TO_KEEP;
  if (days !== daysToKeep) {
    console.warn(`⚠️ Ignoring unusable activity retention window ${daysToKeep}; keeping ${days} days`);
  }

  // Day files are named with the LOCAL date (getDateString), so the cutoff is
  // compared as a `YYYY-MM-DD` string too. Parsing the name with `new Date()`
  // would yield UTC midnight and compare it against a local now-with-time —
  // deleting the boundary day early, and shifting a whole day west of UTC.
  const cutoff = new Date();
  cutoff.setDate(cutoff.getDate() - days);
  const cutoffDateStr = getDateString(cutoff);

  let deletedCount = 0;

  await ensureActivityDir();

  if (!existsSync(ACTIVITY_DIR)) return deletedCount;

  const entries = await readdir(ACTIVITY_DIR, { withFileTypes: true });
  const agentDirs = entries.filter(e => e.isDirectory()).map(e => e.name);

  for (const agentId of agentDirs) {
    const agentDir = join(ACTIVITY_DIR, agentId);
    const files = await readdir(agentDir);

    for (const file of files) {
      if (!DAY_FILE_RE.test(file)) continue;

      if (file.slice(0, -5) < cutoffDateStr) {
        await unlink(join(agentDir, file));
        deletedCount++;
      }
    }
  }

  if (deletedCount > 0) {
    console.log(`🧹 Cleaned up ${deletedCount} old activity files`);
  }

  return deletedCount;
}