Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

17.4 KB
Raw Blame
/**
 * CoS Agent Feedback Module
 *
 * Per-agent feedback capture + aggregation.
 * Extracted from the former monolithic cosAgents.js (issue #2530).
 *
 * The `cosAgents.js` barrel that used to re-export this module is retired
 * (#3450) — callers import from here directly.
 */

import { readFile } from 'node:fs/promises';
import { join } from 'node:path';
import { cosEvents, emitLog } from './cosEvents.js';
import { loadState, saveState, withStateLock } from './cosState.js';
import { atomicWrite, safeJSONParse, tryReadFile } from '../lib/fileUtils.js';
import { loadAgentIndex, getAgentDir } from './cosAgentIndex.js';
import {
  loadCompletionOrderIndex,
  markCompletionFeedbackResolved,
  projectArchivedAgent,
  recordArchivedCompletions,
} from './cosAgentCompletionIndex.js';
import { ServerError } from '../lib/errorHandler.js';
import { bucketFor } from '../lib/objects.js';
import { recordUserAction } from './userActions.js';
import {
  FEEDBACK_RATINGS,
  feedbackArchiveDate,
  hasValidAgentFeedback,
  isAgentFeedbackEligible,
  isAgentFeedbackTarget,
  isAgentFeedbackUpdateTarget,
  isFeedbackRating,
} from '../lib/cosAgentFeedback.js';
import {
  listPendingAgentFeedbackRefs,
  removePendingAgentFeedbackRef,
  upsertPendingAgentFeedbackRef,
} from './cosAgentFeedbackStore.js';
import { extractTaskType } from './taskLearning/store.js';

const ARCHIVE_READ_BATCH_SIZE = 50;
const hasValidFeedback = hasValidAgentFeedback;

// Completed agents are written to their date-bucket archive before they age out
// of live state. Feedback statistics therefore have to read both stores and
// de-duplicate by agent id; reading state alone makes almost all historical
// ratings disappear from the learning view as soon as normal retention runs.
async function loadArchivedAgentsWithFeedback() {
  const idx = await loadAgentIndex();
  const entries = [...idx.entries()];
  const agents = [];

  // Match the archive reader's bounded fan-out so a long-lived install does not
  // open every metadata file at once.
  for (let i = 0; i < entries.length; i += ARCHIVE_READ_BATCH_SIZE) {
    const batch = entries.slice(i, i + ARCHIVE_READ_BATCH_SIZE);
    const reads = batch.map(async ([agentId, dateBucket]) => {
      const content = await tryReadFile(join(getAgentDir(agentId, dateBucket), 'metadata.json'));
      if (!content) return null;
      const raw = safeJSONParse(content, null);
      return hasValidFeedback(raw) ? { ...raw, id: raw.id || raw.agentId || agentId } : null;
    });
    const settled = await Promise.allSettled(reads);
    for (const result of settled) {
      if (result.status === 'fulfilled' && result.value) agents.push(result.value);
    }
  }

  return agents;
}

async function readArchivedAgent(agentId, dateBucket) {
  const metadataPath = join(getAgentDir(agentId, dateBucket), 'metadata.json');
  const read = await readFile(metadataPath, 'utf8').then(
    (content) => ({ content }),
    (error) => ({ error }),
  );
  if (read.error) {
    return { kind: read.error.code === 'ENOENT' ? 'missing' : 'read-failed', metadataPath };
  }
  const { content } = read;
  const raw = safeJSONParse(content, null, { allowArray: false });
  if (!raw || typeof raw !== 'object' || Array.isArray(raw)) {
    return { kind: 'read-failed', metadataPath };
  }
  return {
    kind: 'found',
    metadataPath,
    agent: { ...raw, id: raw.id || raw.agentId || agentId },
  };
}

function unavailableFeedbackRef(ref, reason) {
  return {
    id: ref.agentId,
    agentId: ref.agentId,
    archiveDate: ref.archiveDate || null,
    availability: 'unavailable',
    unavailableReason: reason,
  };
}

/**
 * The durable references to consider, paired with live state.
 *
 * Enrollment happens here rather than only on the `agent:completed` event so a
 * live completion stays discoverable when the event arrived during a restart
 * window or before the migration ran — every reader of the pending set needs
 * that, so it belongs on the one path all of them take.
 */
async function collectPendingFeedbackRefs() {
  const state = await loadState();
  const refs = await listPendingAgentFeedbackRefs();
  const refsById = new Map(refs.map((ref) => [ref.agentId, ref]));
  const liveById = new Map(Object.entries(state?.agents || {}).map(([id, agent]) => [agent.id || id, agent]));

  for (const [agentId, agent] of liveById) {
    if (!isAgentFeedbackEligible(agent)) continue;
    const ref = { agentId, archiveDate: feedbackArchiveDate(agent) };
    refsById.set(agentId, ref);
    await upsertPendingAgentFeedbackRef(ref);
  }

  return { refsById, liveById };
}

/**
 * Reconcile the durable references with live state and indexed archive records.
 * The only broad read here is live state plus the referenced archive files; it
 * never scans the historical archive looking for work.
 */
export async function getPendingAgentFeedback({ includeUnavailable = false } = {}) {
  const { refsById, liveById } = await collectPendingFeedbackRefs();

  const idx = refsById.size > 0 ? await loadAgentIndex() : new Map();
  const agents = [];
  const unavailable = [];
  // This path already paid for the archive read, so hand what it learned to the
  // eligibility projection the scalar/paged readers answer from.
  const learned = [];

  for (const ref of refsById.values()) {
    const live = liveById.get(ref.agentId);
    if (live) {
      if (isAgentFeedbackEligible(live)) {
        agents.push(live);
        continue;
      }

      // A rated live record is not actionable, but retain its reference until
      // the same rating is durable in the archive. This prevents archive
      // eviction from resurrecting a rating obligation after a partial write.
      if (isAgentFeedbackTarget(live) && hasValidFeedback(live)) {
        const archived = await readArchivedAgent(ref.agentId, idx.get(ref.agentId) || ref.archiveDate);
        if (archived.kind === 'found' && hasValidFeedback(archived.agent)) {
          await removePendingAgentFeedbackRef(ref.agentId);
        }
        continue;
      }

      await removePendingAgentFeedbackRef(ref.agentId);
      continue;
    }

    const archiveDate = idx.get(ref.agentId) || ref.archiveDate;
    if (!archiveDate) {
      if (includeUnavailable) unavailable.push(unavailableFeedbackRef(ref, 'deleted'));
      continue;
    }

    const archived = await readArchivedAgent(ref.agentId, archiveDate);
    if (archived.kind === 'read-failed') {
      // An indexed reference with an unreadable metadata file is not proof that
      // the run was deleted. Keep it pending and let History show the failure.
      if (includeUnavailable) unavailable.push(unavailableFeedbackRef({ ...ref, archiveDate }, 'read-failed'));
      continue;
    }
    if (archived.kind === 'missing') {
      if (includeUnavailable) unavailable.push(unavailableFeedbackRef({ ...ref, archiveDate }, 'deleted'));
      continue;
    }
    if (idx.has(ref.agentId)) learned.push([ref.agentId, projectArchivedAgent(archived.agent)]);
    if (!isAgentFeedbackTarget(archived.agent) || hasValidFeedback(archived.agent)) {
      await removePendingAgentFeedbackRef(ref.agentId);
      continue;
    }
    agents.push(archived.agent);
  }

  if (learned.length > 0) await recordArchivedCompletions(learned);
  return { agents, unavailable, count: agents.length };
}

/**
 * Which pending references are still actionable, resolved WITHOUT reading the
 * archive: a live record answers from state, an archived one from the
 * completion-order projection. Only a reference the projection has never seen —
 * a fresh federation import, or an install that has not run the backfill — costs
 * one metadata read, and that read teaches the projection so the next call is
 * free. Deliberately read-only apart from live enrollment: the scalar badge and
 * the paged list must not reconcile the durable store, which
 * `getPendingAgentFeedback` owns.
 */
async function resolveEligiblePendingFeedback() {
  const { refsById, liveById } = await collectPendingFeedbackRefs();
  if (refsById.size === 0) return [];
  const idx = await loadAgentIndex();
  const order = await loadCompletionOrderIndex();
  const eligible = [];
  const learned = [];

  for (const ref of refsById.values()) {
    const live = liveById.get(ref.agentId);
    if (live) {
      if (isAgentFeedbackEligible(live)) eligible.push({ agentId: ref.agentId, agent: live });
      continue;
    }

    const archiveDate = idx.get(ref.agentId) || ref.archiveDate;
    if (!archiveDate) continue;

    // The projection's keyspace is the index's, so consult it only for an id the
    // index still owns — anything else would be learned and immediately pruned.
    const indexed = idx.has(ref.agentId);
    const projected = indexed ? order.get(ref.agentId) : null;
    if (projected) {
      if (projected.feedbackEligible) eligible.push({ agentId: ref.agentId, archiveDate });
      continue;
    }

    const archived = await readArchivedAgent(ref.agentId, archiveDate);
    if (archived.kind !== 'found') continue;
    if (indexed) learned.push([ref.agentId, projectArchivedAgent(archived.agent)]);
    if (isAgentFeedbackTarget(archived.agent) && !hasValidFeedback(archived.agent)) {
      eligible.push({ agentId: ref.agentId, archiveDate, agent: archived.agent });
    }
  }

  if (learned.length > 0) await recordArchivedCompletions(learned);
  return eligible;
}

export async function getPendingAgentFeedbackCount() {
  return (await resolveEligiblePendingFeedback()).length;
}

/**
 * One bounded page of pending feedback, newest agent id first — the same order
 * and cursor the route used when it sliced the fully-hydrated list. Only the
 * returned rows are read off disk.
 */
export async function getPendingAgentFeedbackPage({ limit = 25, cursor } = {}) {
  const eligible = await resolveEligiblePendingFeedback();
  const remaining = eligible
    .filter((entry) => !cursor || entry.agentId < cursor)
    .sort((a, b) => (a.agentId < b.agentId ? 1 : a.agentId > b.agentId ? -1 : 0));

  const items = [];
  for (const entry of remaining.slice(0, limit)) {
    if (entry.agent) { items.push(entry.agent); continue; }
    const archived = await readArchivedAgent(entry.agentId, entry.archiveDate);
    if (archived.kind === 'found') items.push(archived.agent);
  }

  return {
    items,
    total: eligible.length,
    nextCursor: remaining.length > limit ? remaining[limit - 1].agentId : null,
  };
}

// Submit feedback for a completed agent.
//
// The operator-action ledger write (#5594) happens AFTER the state lock releases,
// not inside it: a rating is a human verdict every caller should record, so it
// belongs at this one boundary rather than in the HTTP route — but nesting a DB
// write inside the CoS state lock would hold the lock across an I/O round trip
// for a log line. The lock's own failures still reject before anything is logged.
export async function submitAgentFeedback(agentId, feedback) {
  if (!isFeedbackRating(feedback?.rating)) {
    throw new ServerError(`rating must be one of: ${FEEDBACK_RATINGS.join(', ')}`, {
      status: 400,
      code: 'VALIDATION_ERROR',
    });
  }

  // Lazy index migration takes backup admission. Resolve it before the state
  // lock so a cut cannot invert admission/state-lock order during first access.
  const idx = await loadAgentIndex();

  // The durable reference is removed only after the source metadata write. Keep
  // that small store mutation outside the CoS state lock so a database/file I/O
  // round trip never serializes unrelated agent state updates.
  const result = await withStateLock(async () => {
    const state = await loadState();
    const feedbackData = {
      rating: feedback.rating,
      comment: feedback.comment || null,
      submittedAt: new Date().toISOString()
    };

    // Try state first (recently completed agents still in state)
    if (state.agents[agentId]) {
      const agent = state.agents[agentId];
      if (!isAgentFeedbackUpdateTarget(agent)) {
        throw new ServerError('Can only submit feedback for completed agents', { status: 400, code: 'INVALID_STATE' });
      }
      state.agents[agentId] = { ...agent, feedback: feedbackData };
      await saveState(state);

      // Also update on-disk metadata. The live state write is authoritative for
      // the current card, but the pending reference is cleared only after the
      // source archive carries the same rating, so eviction cannot resurrect it.
      const dateBucket = feedbackArchiveDate(agent);
      const archived = await readArchivedAgent(agentId, dateBucket);
      if (archived.kind === 'found') {
        await atomicWrite(archived.metadataPath, { ...archived.agent, feedback: feedbackData });
      }

      emitLog('info', `Feedback received for agent ${agentId}: ${feedback.rating}`, { agentId, rating: feedback.rating });
      return { success: true, agent: state.agents[agentId], feedbackData, clearPendingRef: archived.kind === 'found' };
    }

    // Agent not in state — look up from disk via the preloaded index
    const dateStr = idx.get(agentId);
    if (!dateStr) throw new ServerError('Agent not found', { status: 404, code: 'NOT_FOUND' });

    const archived = await readArchivedAgent(agentId, dateStr);
    if (archived.kind !== 'found') throw new ServerError('Agent not found', { status: 404, code: 'NOT_FOUND' });
    if (!isAgentFeedbackUpdateTarget(archived.agent)) {
      throw new ServerError('Can only submit feedback for completed agents', { status: 400, code: 'INVALID_STATE' });
    }

    const updated = { ...archived.agent, feedback: feedbackData };
    await atomicWrite(archived.metadataPath, updated);

    emitLog('info', `Feedback received for agent ${agentId}: ${feedback.rating}`, { agentId, rating: feedback.rating });
    return { success: true, agent: { ...updated, id: agentId }, feedbackData, clearPendingRef: true };
  });

  const { feedbackData, clearPendingRef, ...response } = result;
  if (clearPendingRef) await removePendingAgentFeedbackRef(agentId);
  // A rated archive is no longer an eligible target; clearing the projected bit
  // keeps the scalar badge correct without re-reading the record.
  await markCompletionFeedbackResolved(agentId);
  cosEvents.emit('agent:feedback', { agentId, feedback: feedbackData });
  await recordUserAction({
    type: 'cos.agent.feedback',
    target: agentId,
    targetName: response.agent?.metadata?.taskDescription,
    summary: `Rated agent ${agentId} ${feedbackData.rating}`,
    payload: {
      agentId,
      taskId: response.agent?.taskId ?? null,
      rating: feedbackData.rating,
      comment: feedbackData.comment,
      taskType: extractTaskType({
        description: response.agent?.metadata?.taskDescription,
        metadata: response.agent?.metadata,
        taskType: response.agent?.metadata?.taskType,
      }),
    },
    source: { service: 'cosAgentFeedback', fn: 'submitAgentFeedback' },
    happenedAt: feedbackData.submittedAt,
    dedupeKey: `cos.agent.feedback:${agentId}:${feedbackData.submittedAt}`,
  });
  return response;
}

// Get aggregated feedback statistics
export async function getFeedbackStats() {
  const state = await loadState();
  const archived = await loadArchivedAgentsWithFeedback();
  const byAgentId = new Map(archived.map(agent => [agent.id, agent]));

  // Live state is freshest, but only overwrite an archived rating when the live
  // record actually carries valid feedback. This preserves a durable rating if
  // a prior best-effort live-state update did not make it into both stores.
  for (const [agentId, agent] of Object.entries(state.agents)) {
    if (hasValidFeedback(agent)) byAgentId.set(agent.id || agentId, agent);
  }

  const withFeedback = [...byAgentId.values()];
  const positive = withFeedback.filter(a => a.feedback.rating === 'positive').length;
  const negative = withFeedback.filter(a => a.feedback.rating === 'negative').length;
  const neutral = withFeedback.filter(a => a.feedback.rating === 'neutral').length;

  // Group by task type
  const byTaskType = {};
  withFeedback.forEach(a => {
    const taskType = extractTaskType({
      description: a.metadata?.taskDescription,
      metadata: a.metadata,
      taskType: a.metadata?.taskType,
    });
    const bucket = bucketFor(byTaskType, taskType, () => ({ positive: 0, negative: 0, neutral: 0, total: 0 }));
    if (!bucket) return;
    bucket[a.feedback.rating]++;
    bucket.total++;
  });

  // Recent feedback (last 10 with comments)
  const recentWithComments = withFeedback
    .filter(a => a.feedback.comment)
    .sort((a, b) => new Date(b.feedback.submittedAt) - new Date(a.feedback.submittedAt))
    .slice(0, 10)
    .map(a => ({
      agentId: a.id,
      taskDescription: a.metadata?.taskDescription,
      rating: a.feedback.rating,
      comment: a.feedback.comment,
      submittedAt: a.feedback.submittedAt
    }));

  const satisfactionRate = withFeedback.length > 0
    ? Math.round((positive / withFeedback.length) * 100)
    : null;

  return {
    total: withFeedback.length,
    positive,
    negative,
    neutral,
    satisfactionRate,
    byTaskType,
    recentWithComments
  };
}

let feedbackEventsInitialized = false;

/** Register future-completion enrollment without doing any cold-start provider work. */
export function initializeAgentFeedback() {
  if (feedbackEventsInitialized) return;
  feedbackEventsInitialized = true;
  cosEvents.on('agent:completed', (agent) => {
    if (!isAgentFeedbackEligible(agent)) return;
    upsertPendingAgentFeedbackRef({
      agentId: agent.id,
      archiveDate: feedbackArchiveDate(agent),
    }).catch((err) => {
      console.error(`❌ Failed to enroll CoS feedback for ${agent.id}: ${err.message}`);
    });
  });
}