Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

14.0 KB
Raw Blame
/**
 * Memory Extractor Service
 *
 * Extracts memories from agent output and task completion.
 * Supports structured MEMORY blocks and pattern-based extraction.
 */

import { createMemory, searchMemories, getMemories } from './memoryBackend.js';
import { generateMemoryEmbedding } from './memoryEmbeddings.js';
import { cosEvents } from './cosEvents.js';
import * as notifications from './notifications.js';
import { classifyMemories, isAvailable as isClassifierAvailable } from './memoryClassifier.js';
import { getDomainAutonomyMode } from './cosState.js';
import { getDomainBudgetStatus, recordDomainUsage } from './domainUsage.js';

const DEDUP_SIMILARITY_THRESHOLD = 0.82;

/**
 * Parse structured MEMORY blocks from agent output
 * Format:
 * <MEMORY type="learning" category="codebase" confidence="0.9">
 * Content here
 * Tags: tag1, tag2
 * </MEMORY>
 */
function parseMemoryBlocks(output) {
  const memories = [];
  const memoryRegex = /<MEMORY\s+([^>]*)>([\s\S]*?)<\/MEMORY>/gi;

  let match;
  while ((match = memoryRegex.exec(output)) !== null) {
    const attrs = match[1];
    const content = match[2].trim();

    // Parse attributes
    const type = attrs.match(/type="([^"]+)"/)?.[1] || 'observation';
    const category = attrs.match(/category="([^"]+)"/)?.[1] || 'other';
    const confidence = parseFloat(attrs.match(/confidence="([^"]+)"/)?.[1] || '0.8');

    // Extract tags from content
    const tagsMatch = content.match(/Tags?:\s*(.+)$/mi);
    const tags = tagsMatch ? tagsMatch[1].split(',').map(t => t.trim()) : [];
    const cleanContent = content.replace(/Tags?:\s*.+$/mi, '').trim();

    memories.push({
      type,
      category,
      confidence,
      content: cleanContent,
      tags,
      structured: true
    });
  }

  return memories;
}

/**
 * Extract implicit patterns from agent output
 * Focuses on user preferences and values — not implementation details
 */
function extractPatterns(output) {
  const memories = [];
  let match;

  // Pattern: "User prefers..." or "The user wants..." or "User values..."
  const prefersRegex = /(?:The\s+)?user\s+(?:prefers|wants|likes|requested|values|cares about|insists on|prioritizes|expects)\s+(.+?)(?:\.|$)/gi;
  while ((match = prefersRegex.exec(output)) !== null) {
    const content = match[1].trim();
    // Only keep if substantive (not just a single word or implementation detail)
    if (content.length >= 15 && !isImplementationDetail(content)) {
      memories.push({
        type: 'preference',
        content: `User prefers ${content}`,
        confidence: 0.8,
        category: 'preferences',
        tags: ['user-preference']
      });
    }
  }

  // Pattern: User pushed back on something (reveals values)
  const pushbackRegex = /user\s+(?:pushed back on|rejected|didn't like|asked (?:us|me) to (?:change|redo|fix))\s+(.+?)(?:\.|$)/gi;
  while ((match = pushbackRegex.exec(output)) !== null) {
    const content = match[1].trim();
    if (content.length >= 15 && !isImplementationDetail(content)) {
      memories.push({
        type: 'preference',
        content: `User pushed back on: ${content}`,
        confidence: 0.8,
        category: 'values',
        tags: ['user-feedback', 'values']
      });
    }
  }

  return memories;
}

/**
 * Check if content is an implementation detail rather than a user insight
 */
function isImplementationDetail(content) {
  const lower = content.toLowerCase();

  // File paths, function names, component references
  if (/\.(jsx?|tsx?|css|json|md|py|sh|yml)\b/i.test(content)) return true;
  if (/(?:function|class|component|const|import|export|require)\s/i.test(content)) return true;

  // Specific code references (line numbers, variable names with dots)
  if (/\b(?:line\s+\d+|\.js\b|\.ts\b)/i.test(content)) return true;
  if (/[a-z]+\.[a-z]+\(/i.test(content)) return true;

  // CSS/styling specifics
  if (/\b(?:\d+px|#[0-9a-f]{3,8}|p[xytblr]-\d|sm:|md:|lg:)\b/i.test(content)) return true;

  // Package/dependency names
  if (/\b(?:npm|yarn|package\.json|node_modules|import\s+\{)\b/i.test(content)) return true;

  // Port numbers, URLs, endpoints
  if (/(?:port\s+\d{4}|localhost|\/api\/|endpoint)/i.test(content)) return true;

  // Architecture descriptions (easily discoverable)
  if (/\b(?:uses?\s+(?:express|react|vite|pm2|socket\.io|zod))\b/i.test(lower)) return true;
  if (/\b(?:monorepo|middleware|route\s+handler|service\s+layer)\b/i.test(lower)) return true;

  return false;
}

/**
 * Extract task-related context
 * Only extracts if the output reveals something about user preferences
 */
function extractTaskContext(_task, _output, _success) {
  // Task completion summaries are not useful memories — they're git history.
  // Only the LLM classifier should extract user-insight memories from task output.
  return [];
}

const normalizeText = s => (s || '').toLowerCase().replace(/\s+/g, ' ').trim();

/**
 * Deduplicate memories within a single extraction batch
 */
function deduplicateMemories(memories) {
  const unique = [];
  const seen = new Set();

  for (const memory of memories) {
    const key = `${memory.type}:${normalizeText(memory.content).substring(0, 100)}`;

    if (!seen.has(key)) {
      seen.add(key);
      unique.push(memory);
    }
  }

  return unique;
}

/**
 * Check if a proposed memory duplicates an existing active or pending memory.
 * Uses vector similarity for active memories and text prefix matching for pending.
 */
async function findExistingDuplicate(embedding, content, pendingMemories) {
  if (embedding) {
    const results = await searchMemories(embedding, {
      minRelevance: DEDUP_SIMILARITY_THRESHOLD,
      limit: 3
    }).catch(err => {
      console.log(`⚠️ Dedup vector search failed: ${err.message}`);
      return { memories: [] };
    });

    if (results.memories.length > 0) {
      return results.memories[0];
    }
  }

  // Also check pending_approval memories (searchMemories only returns active).
  // Note: getMemories() returns index/summary rows without `content`, so we
  // compare against `summary` (which we also normalize on the incoming memory
  // for a reasonable cross-match). This keeps the dedup cheap without an
  // extra round-trip to load full bodies.
  if (pendingMemories.length > 0) {
    const normalizedIncoming = normalizeText(content).substring(0, 80);
    for (const mem of pendingMemories) {
      const memText = mem.content || mem.summary || '';
      if (!memText) continue;
      if (normalizeText(memText).substring(0, 80) === normalizedIncoming) {
        return mem;
      }
    }
  }

  return null;
}

/**
 * Main extraction function
 * Processes agent output and creates memories
 *
 * Uses LLM-based classification when available, falls back to pattern matching.
 */
export async function extractAndStoreMemories(agentId, taskId, output, task = null) {
  // Per-domain autonomy gate: `off` skips memory extraction entirely; `dry-run`
  // routes even high-confidence memories to pending approval (nothing stores as
  // active without a human confirm); `execute` is the historical behavior.
  const mode = await getDomainAutonomyMode('memory');
  if (mode === 'off') {
    console.log(`🧠 Memory auto-extract is OFF — skipping extraction for agent ${agentId}`);
    return { created: 0, pendingApproval: 0, memories: [], pendingMemories: [], skipped: 'domain-off' };
  }

  // Daily memory budget (#711): once today's extraction actions/minutes reach the
  // cap, skip for the rest of the day — same outcome as `off`, distinct reason.
  const budget = await getDomainBudgetStatus('memory');
  if (!budget.withinBudget) {
    console.log(`🧠 Memory auto-extract daily ${budget.exceeded} budget reached — skipping extraction for agent ${agentId}`);
    return { created: 0, pendingApproval: 0, memories: [], pendingMemories: [], skipped: 'domain-budget' };
  }

  // Time the extraction (incl. the LLM classify) for the minutes budget.
  const startTime = Date.now();
  const allMemories = [];
  let usedLLM = false;

  // Try LLM-based classification first (if available)
  const classifierAvailable = await isClassifierAvailable().catch(() => false);

  if (classifierAvailable && task) {
    const llmResult = await classifyMemories(task, output).catch(err => {
      console.log(`⚠️ LLM memory classification failed: ${err.message}`);
      return null;
    });

    if (llmResult?.memories?.length > 0) {
      usedLLM = true;
      console.log(`🧠 LLM classified ${llmResult.memories.length} memories`);

      // Convert LLM results to our format
      for (const mem of llmResult.memories) {
        allMemories.push({
          type: mem.type || 'observation',
          category: mem.category || 'other',
          content: mem.content,
          confidence: mem.confidence || 0.7,
          tags: mem.tags || [],
          reasoning: mem.reasoning
        });
      }
    }
  }

  // Fall back to pattern-based extraction if LLM didn't produce results
  if (!usedLLM) {
    // Parse structured memory blocks
    const structured = parseMemoryBlocks(output);
    allMemories.push(...structured);

    // Extract patterns from text
    const patterns = extractPatterns(output);
    allMemories.push(...patterns);

    // Extract task context (but only if meaningful, not just task echoes)
    if (task) {
      const taskContext = extractTaskContext(task, output, true);
      // Filter out low-quality task context memories
      const filtered = taskContext.filter(m => {
        // Reject memories that just echo the task description
        if (m.content.startsWith(`Task "${task.description.substring(0, 50)}`)) return false;
        // Reject memories that are just "## Summary" or similar
        if (/^(##?\s*)?Summary\s*$/i.test(m.content)) return false;
        // Reject very short memories
        if (m.content.length < 30) return false;
        return true;
      });
      allMemories.push(...filtered);
    }
  }

  // Deduplicate
  const unique = deduplicateMemories(allMemories);

  // Filter by confidence — only high-quality memories pass through
  const highConfidence = unique.filter(m => m.confidence >= 0.85);
  const mediumConfidence = unique.filter(m => m.confidence >= 0.7 && m.confidence < 0.85);

  const sourceAppId = task?.metadata?.app || null;

  // Pre-fetch pending memories once for dedup checks across both loops.
  // Pass a high limit to avoid relying on the default pagination (50) — dedup
  // should see the full pending backlog so duplicates aren't missed when the
  // queue grows.
  const pendingForDedup = await getMemories({ status: 'pending_approval', limit: 1000 })
    .catch(err => {
      console.log(`⚠️ Failed to load pending memories for dedup: ${err.message}`);
      return { memories: [] };
    });

  let skippedDuplicates = 0;

  async function dedupAndCreate(mem, extraFields = {}) {
    const embedding = await generateMemoryEmbedding(mem);
    const existing = await findExistingDuplicate(embedding, mem.content, pendingForDedup.memories);
    if (existing) {
      // Active-memory hits from searchMemories() return metadata with \`summary\`
      // but no \`content\`; pending hits carry \`content\`. Prefer whichever is present.
      const existingText = existing.content || existing.summary || '';
      console.log(`🧠 Skipping duplicate memory (similar to "${normalizeText(existingText).substring(0, 60)}…")`);
      skippedDuplicates++;
      return null;
    }
    return createMemory({
      ...mem, sourceAgentId: agentId, sourceTaskId: taskId, sourceAppId, ...extraFields
    }, embedding);
  }

  // In dry-run, high-confidence memories that would auto-store as active are
  // instead routed to pending approval — nothing enters active memory without a
  // human confirm, but the extraction work is still surfaced for review.
  const created = [];
  const pendingMemories = [];
  for (const mem of highConfidence) {
    if (mode === 'dry-run') {
      const memory = await dedupAndCreate(mem, { status: 'pending_approval' });
      if (memory) pendingMemories.push(memory);
    } else {
      const memory = await dedupAndCreate(mem);
      if (memory) created.push(memory);
    }
  }

  for (const mem of mediumConfidence) {
    const memory = await dedupAndCreate(mem, { status: 'pending_approval' });
    if (memory) pendingMemories.push(memory);
  }

  if (pendingMemories.length > 0) {
    console.log(`🧠 ${pendingMemories.length} memories pending approval`);
    cosEvents.emit('memory:approval-needed', {
      agentId,
      taskId,
      memories: pendingMemories.map(m => ({
        id: m.id,
        type: m.type,
        content: (m.content || '').slice(0, 500),
        confidence: m.confidence
      }))
    });

    // Create notification for user
    for (const mem of pendingMemories) {
      const alreadyExists = await notifications.exists(
        notifications.NOTIFICATION_TYPES.MEMORY_APPROVAL,
        'memoryId',
        mem.id
      );
      if (!alreadyExists) {
        await notifications.addNotification({
          type: notifications.NOTIFICATION_TYPES.MEMORY_APPROVAL,
          title: `Memory needs approval`,
          description: (mem.summary || mem.content || '').slice(0, 300),
          priority: notifications.PRIORITY_LEVELS.MEDIUM,
          link: '/cos/memory',
          metadata: {
            memoryId: mem.id,
            category: 'memory-approval',
            actionKind: 'memory.approval',
            sourceOwned: true,
            memoryType: mem.type,
            agentId,
            taskId
          }
        });
      }
    }
  }

  if (skippedDuplicates > 0) {
    console.log(`🧠 Skipped ${skippedDuplicates} duplicate memories from agent ${agentId}`);
  }
  console.log(`🧠 Extracted ${created.length} memories from agent ${agentId}`);
  cosEvents.emit('memory:extracted', {
    agentId,
    taskId,
    count: created.length,
    pendingApproval: pendingMemories.length,
    skippedDuplicates
  });

  // Record the extraction against the memory domain's daily budget (#711).
  await recordDomainUsage('memory', { actions: 1, ms: Date.now() - startTime })
    .catch(err => console.error(`❌ Failed to record memory budget usage for agent ${agentId}: ${err.message}`));

  return {
    created: created.length,
    pendingApproval: pendingMemories.length,
    memories: created,
    pendingMemories
  };
}