Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

14.0 KB
Raw Blame
/**
 * Memory Classifier Service
 *
 * Uses an LLM to intelligently evaluate agent output and extract useful memories.
 * Falls back to pattern-based extraction if LLM is unavailable.
 */

import { existsSync } from 'fs';
import { join } from 'path';
import { getStageTemplate } from './promptService.js';
import { atomicWrite, ensureDir, readJSONFileStrict, unreadableStoreError, PATHS } from '../lib/fileUtils.js';
import { extractJson } from '../lib/jsonExtract.js';
import { fetchWithTimeout } from '../lib/fetchWithTimeout.js';
import { readResponseJson } from '../lib/readResponseJson.js';
import { getMemories } from './memoryBackend.js';

const MODEL_LIST_TIMEOUT_MS = 5000;
const MODEL_LOAD_TIMEOUT_MS = 120000;
const HEALTH_CHECK_TIMEOUT_MS = 5000;

const MEMORY_CONFIG_FILE = join(PATHS.data, 'memory-classifier-config.json');

// Default configuration
const DEFAULT_CONFIG = {
  enabled: true,
  provider: 'lmstudio',
  endpoint: process.env.LM_STUDIO_URL ? `${process.env.LM_STUDIO_URL.replace(/\/+$/, '').replace(/\/v1$/, '')}/v1/chat/completions` : 'http://localhost:1234/v1/chat/completions',
  model: 'gptoss-20b',
  timeout: 60000,
  maxOutputLength: 10000,
  fallbackToPatterns: true
};

let configCache = null;

/**
 * Load classifier configuration
 */
async function loadConfig() {
  if (configCache) return configCache;

  const { ok, value } = await readJSONFileStrict(MEMORY_CONFIG_FILE, null, { allowArray: false });
  if (!ok) {
    // Serve defaults to readers without caching, so a repaired file is picked up.
    console.warn('⚠️ Memory classifier config unreadable, using defaults');
    return DEFAULT_CONFIG;
  }
  configCache = value ? { ...DEFAULT_CONFIG, ...value } : DEFAULT_CONFIG;
  return configCache;
}

// updateConfig refuses while the stored bytes are unreadable, so defaults
// served to readers are never persisted over the user's config.
async function assertConfigWritable() {
  const { ok } = await readJSONFileStrict(MEMORY_CONFIG_FILE, null, { allowArray: false, logError: false });
  if (!ok) throw unreadableStoreError(MEMORY_CONFIG_FILE);
}

/**
 * Get current configuration
 */
export async function getConfig() {
  return loadConfig();
}

/**
 * Update configuration
 */
export async function updateConfig(updates) {
  await assertConfigWritable();
  const config = await loadConfig();
  const newConfig = { ...config, ...updates };

  if (!existsSync(PATHS.data)) {
    await ensureDir(PATHS.data);
  }

  await atomicWrite(MEMORY_CONFIG_FILE, newConfig);
  configCache = newConfig;

  return newConfig;
}

/**
 * Load existing active + pending memories as a summary for the LLM prompt.
 * Defense-in-depth: the LLM sees what's already stored so it avoids proposing
 * duplicates. The backend dedup in memoryExtractor is the actual safety net.
 */
async function loadExistingMemorySummary() {
  const [active, pending] = await Promise.all([
    getMemories({ status: 'active', sortBy: 'importance', sortOrder: 'desc', limit: 30 })
      .catch(() => ({ memories: [] })),
    getMemories({ status: 'pending_approval', limit: 30 })
      .catch(() => ({ memories: [] }))
  ]);

  const allMemories = [...active.memories, ...pending.memories];
  if (allMemories.length === 0) return '';

  // getMemories() returns index/metadata rows without \`content\` — fall back
  // to \`summary\` so each bullet has a real sentence instead of an empty string.
  const lines = allMemories.slice(0, 30).map(m => {
    const text = (m.content || m.summary || '').substring(0, 150);
    return `- [${m.type}] ${text}`;
  });
  return lines.join('\n');
}

/**
 * Build the classification prompt
 */
async function buildClassificationPrompt(task, agentOutput, config) {
  // Try to load the template
  const template = await getStageTemplate('memory-evaluate').catch(() => null);

  // Load existing memories to inject into the prompt
  const existingMemories = await loadExistingMemorySummary();

  if (!template) {
    // Fallback inline template
    return buildFallbackPrompt(task, agentOutput, existingMemories);
  }

  // Apply variables to template
  const variables = {
    taskId: task.id || 'unknown',
    taskDescription: task.description || 'No description',
    taskStatus: task.status || 'completed',
    appName: task.metadata?.app || 'PortOS',
    agentOutput: agentOutput.substring(0, config.maxOutputLength || 10000),
    existingMemories: existingMemories || 'No existing memories yet.'
  };

  let prompt = template;
  for (const [key, value] of Object.entries(variables)) {
    prompt = prompt.replace(new RegExp(`\\{\\{${key}\\}\\}`, 'g'), value);
  }

  return prompt;
}

/**
 * Fallback prompt if template not found
 */
function buildFallbackPrompt(task, agentOutput, existingMemories = '') {
  const existingSection = existingMemories
    ? `\n## Existing Memories (DO NOT duplicate these)\nThe following memories are already stored. Do NOT propose any memory that overlaps with or restates these:\n${existingMemories}\n`
    : '';

  return `Analyze this agent output and extract memories about the USER — their values, preferences, work patterns, and qualities they care about.

Task: ${task.description || 'Unknown task'}
Output:
${agentOutput.substring(0, 8000)}
${existingSection}
Return JSON with memories array. Each memory should have:
- type: preference|decision|learning
- category: values|workflow|preferences|communication|aesthetics|patterns
- content: the actual memory about the user
- confidence: 0.7-1.0
- tags: relevant tags
- reasoning: what this reveals about the user

DO NOT include:
- Implementation details (file paths, function names, CSS values, component structures)
- Architecture descriptions (easily discoverable from code)
- Task completion summaries (that's git history)
- Generic best practices any developer would know
- One-time code observations or status assessments
- Anything already captured in the existing memories listed above

Most outputs should produce ZERO memories. Only extract when you observe something genuinely revealing about the user's values, preferences, or work patterns that is NOT already stored.

Return: {"memories": [...], "rejected": [...]}`;
}

/**
 * Derive the LM Studio base URL from the chat/completions endpoint
 */
function getBaseUrl(endpoint) {
  return endpoint.replace(/\/v1\/chat\/completions\/?$/, '').replace(/\/+$/, '');
}

/**
 * Ensure an LLM model is loaded in LM Studio. If none is loaded, discover
 * downloaded LLM models via `/api/v0/models` and auto-load one via
 * `/api/v1/models/load`. Prefers the configured model when available,
 * otherwise falls back to any downloaded LLM.
 *
 * @returns {Promise<string|null>} The id of a loaded LLM model, or null
 */
async function ensureLLMModelLoaded(config) {
  const baseUrl = getBaseUrl(config.endpoint);

  const listResponse = await fetchWithTimeout(`${baseUrl}/api/v0/models`, {
    method: 'GET'
  }, MODEL_LIST_TIMEOUT_MS).catch(() => null);

  if (!listResponse?.ok) return null;

  const payload = await readResponseJson(listResponse);
  const allModels = payload?.data || [];
  const llmModels = allModels.filter(m => m.type === 'llm');

  if (llmModels.length === 0) {
    console.warn('⚠️ No LLM models downloaded in LM Studio — download one in the Discover tab');
    return null;
  }

  // If any LLM is already loaded, prefer the configured one; otherwise use any loaded LLM
  const loaded = llmModels.filter(m => m.state === 'loaded');
  if (loaded.length > 0) {
    const match = loaded.find(m => m.id === config.model || m.id.includes(config.model));
    return (match || loaded[0]).id;
  }

  // Need to load one — prefer configured model, else first available
  const preferred = llmModels.find(m => m.id === config.model || m.id.includes(config.model))
    || llmModels[0];

  console.log(`📦 Auto-loading LLM model: ${preferred.id}`);

  const loadResponse = await fetchWithTimeout(`${baseUrl}/api/v1/models/load`, {
    method: 'POST',
    headers: { 'Content-Type': 'application/json' },
    body: JSON.stringify({ model: preferred.id })
  }, MODEL_LOAD_TIMEOUT_MS).catch(err => ({ ok: false, _err: err.message }));

  if (!loadResponse.ok) {
    const errText = loadResponse._err || await loadResponse.text?.().catch(() => 'unknown error') || 'unknown error';
    console.error(`❌ Failed to auto-load LLM model ${preferred.id}: ${errText}`);
    return null;
  }

  console.log(`✅ LLM model loaded: ${preferred.id}`);
  return preferred.id;
}

/**
 * Call LM Studio API for classification
 */
async function callLLM(prompt, config) {
  // Make sure a model is loaded before sending the request — LM Studio returns
  // 400 "No models loaded" otherwise.
  const loadedModel = await ensureLLMModelLoaded(config);
  const modelToUse = loadedModel || config.model;

  const response = await fetchWithTimeout(config.endpoint, {
    method: 'POST',
    headers: {
      'Content-Type': 'application/json',
      'Authorization': `Bearer lm-studio`
    },
    body: JSON.stringify({
      model: modelToUse,
      messages: [
        {
          role: 'system',
          content: 'You are a memory classification assistant. Return only valid JSON.'
        },
        {
          role: 'user',
          content: prompt
        }
      ],
      temperature: 0.1,
      max_tokens: 2000
    })
  }, config.timeout);

  if (!response.ok) {
    const error = await response.text().catch(() => 'Unknown error');
    throw new Error(`LLM API error ${response.status}: ${error}`);
  }

  const data = await readResponseJson(response);
  return data.choices?.[0]?.message?.content || '';
}

/**
 * Parse LLM response to extract memories
 */
function parseLLMResponse(response, promptToStrip = '') {
  const source = promptToStrip && response?.includes(promptToStrip)
    ? response.replace(promptToStrip, '')
    : response;
  const { value: parsed } = extractJson(source, {
    // The classifier prompt includes a schema object. Walk all candidates and
    // prefer the response shape so a prompt echo cannot look like an empty
    // classification result.
    skipInnerFence: true,
    shapePredicate: (value) => value && typeof value === 'object' && !Array.isArray(value)
      && Array.isArray(value.memories),
  });

  if (parsed === undefined || !parsed || typeof parsed !== 'object' || Array.isArray(parsed)) {
    console.log('⚠️ Could not find JSON in LLM response');
    return { memories: [], rejected: [], parseError: true };
  }

  // Validate structure
  if (!Array.isArray(parsed.memories)) {
    return { memories: [], rejected: parsed.rejected || [], parseError: false };
  }

  // Validate each memory — reject implementation details and low-value noise
  const validMemories = parsed.memories.filter(m => {
    if (!m.type || !m.content || typeof m.confidence !== 'number') return false;
    if (m.confidence < 0.7) return false;
    if (m.content.length < 15) return false;

    // Reject obvious task echoes
    if (/^Task\s+['"].*['"]\s*:/i.test(m.content)) return false;
    if (/was\s+(completed|successful|done)/i.test(m.content) && m.content.length < 80) return false;

    // Reject implementation details — file paths, function names, CSS values
    if (/\.(jsx?|tsx?|css|json|md|py|sh|yml)\b/i.test(m.content) && m.type !== 'preference') return false;
    if (/\b\d+px\b/.test(m.content)) return false;
    if (/\b#[0-9a-f]{6}\b/i.test(m.content) && m.type !== 'preference') return false;
    if (/\bport\s+\d{4}\b/i.test(m.content)) return false;

    // Reject architecture descriptions (easily discoverable from code)
    if (/\b(?:uses?\s+(?:express|react|vite|pm2|tailwind|socket\.io|zod))\b/i.test(m.content) && m.type === 'fact') return false;

    // Reject positive status assessments (not memories)
    if (/\b(?:no\s+issues?\s+found|well[- ]optimized|already\s+(?:has|implements)|no\s+(?:fixes?|changes?)\s+(?:required|needed))\b/i.test(m.content)) return false;

    // Reject one-time observations about code state
    if (/\b(?:imported?\s+but\s+(?:not\s+used|unused|never)|is\s+sized|has\s+\d+\s+lines?)\b/i.test(m.content)) return false;

    return true;
  });

  return {
    memories: validMemories,
    rejected: parsed.rejected || [],
    parseError: false
  };
}

/**
 * Main classification function
 *
 * @param {Object} task - Task object with id, description, metadata
 * @param {string} agentOutput - The agent's output text
 * @returns {Object} { memories: [], rejected: [], usedLLM: boolean, error?: string }
 */
export async function classifyMemories(task, agentOutput) {
  const config = await loadConfig();

  // Skip if output is too short
  if (!agentOutput || agentOutput.length < 100) {
    return { memories: [], rejected: [], usedLLM: false, skipped: 'output-too-short' };
  }

  // Skip if disabled
  if (!config.enabled) {
    return { memories: [], rejected: [], usedLLM: false, skipped: 'classifier-disabled' };
  }

  const prompt = await buildClassificationPrompt(task, agentOutput, config);

  // Call LLM for classification
  const llmResponse = await callLLM(prompt, config).catch(err => {
    console.log(`⚠️ LLM classification failed: ${err.message}`);
    return null;
  });

  if (!llmResponse) {
    return {
      memories: [],
      rejected: [],
      usedLLM: false,
      error: 'LLM call failed',
      fallbackAvailable: config.fallbackToPatterns
    };
  }

  const result = parseLLMResponse(llmResponse, prompt);

  if (result.parseError) {
    console.log(`⚠️ Failed to parse LLM response, raw: ${llmResponse.substring(0, 200)}`);
    return {
      memories: [],
      rejected: [],
      usedLLM: true,
      error: 'Failed to parse LLM response',
      fallbackAvailable: config.fallbackToPatterns
    };
  }

  console.log(`🧠 LLM classified ${result.memories.length} memories, rejected ${result.rejected.length}`);

  return {
    memories: result.memories,
    rejected: result.rejected,
    usedLLM: true
  };
}

/**
 * Check if the classifier is available (LLM endpoint reachable)
 */
export async function isAvailable() {
  const config = await loadConfig();

  if (!config.enabled) return false;

  // Quick health check
  const response = await fetchWithTimeout(
    config.endpoint.replace('/chat/completions', '/models'),
    { method: 'GET' },
    HEALTH_CHECK_TIMEOUT_MS
  ).catch(() => null);

  return response?.ok === true;
}