Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

44.2 KB
Raw Blame
/**
 * Brain Service
 *
 * Core business logic for the Brain feature:
 * - Capture and classify thoughts
 * - Route to appropriate databases
 * - Generate daily digests and weekly reviews
 * - Handle corrections and fixes
 */

import * as storage from './brainStorage.js';
import { brainEvents } from './brainStorage.js';
import { getInstanceId, ensureInstanceId, UNKNOWN_INSTANCE_ID } from './instanceIdentity.js';
import { getActiveProvider, getProviderById } from './providers.js';
import { buildPrompt } from './promptService.js';
import { validate } from '../lib/validation.js';
import { extractJson } from '../lib/jsonExtract.js';
import { runPromptThroughProvider } from './promptRunner.js';
import { isToolFreeOneShotProvider } from '../lib/providerVendors.js';
import { ServerError } from '../lib/errorHandler.js';
import { getDomainAutonomyMode } from './cosState.js';
import { getDomainBudgetStatus, recordDomainUsage } from './domainUsage.js';
import { deleteMemoryAssets } from './chatgptImport.js';
import { withBackupAssetPublication } from '../lib/backupSnapshotBoundary.js';
import * as repoCloner from './repoCloner.js';
import { deriveRepoLinkFields } from '../lib/repoLinkFields.js';
import { parseBareUrl } from '../lib/bareUrl.js';
import { normalizeRepoIntake } from '../lib/repoIntakeActions.js';
import {
  classifierOutputSchema,
  digestOutputSchema,
  reviewOutputSchema,
  extractedPeopleSchema,
  extractedProjectSchema,
  extractedIdeaSchema,
  extractedAdminSchema,
  extractedMemorySchema
} from '../lib/brainValidation.js';

export const TOOL_FREE_REQUIRED = 'TOOL_FREE_REQUIRED';

// Extracted field validators by destination
const EXTRACTED_VALIDATORS = {
  people: extractedPeopleSchema,
  projects: extractedProjectSchema,
  ideas: extractedIdeaSchema,
  admin: extractedAdminSchema,
  memories: extractedMemorySchema
};

/**
 * Call AI provider with a prompt.
 * `toolFree` is for prompts that embed stored Brain records (writable by any
 * caller on a password-free install): the run is pinned to a provider that can
 * disable its host tools, with no fallback to one that cannot.
 */
async function callAI(promptStageName, variables, providerOverride, modelOverride, { toolFree = false } = {}) {
  const provider = providerOverride
    ? await getProviderById(providerOverride)
    : await getActiveProvider();

  if (!provider || !provider.enabled) {
    throw new Error('No AI provider available');
  }

  if (toolFree && !isToolFreeOneShotProvider(provider)) {
    throw new ServerError(
      `Brain provider '${provider.id}' cannot run without tools, so it cannot process stored Brain records unattended. Choose an API provider or a tool-free CLI (claude, grok, pi).`,
      { status: 400, code: TOOL_FREE_REQUIRED }
    );
  }

  const prompt = await buildPrompt(promptStageName, variables);
  let model = modelOverride || provider.defaultModel;

  // gemini-cli default is a thinking model (3.1-pro); prefer the provider's configured
  // light tier (populated from data.reference/providers.json on new installs) and only fall
  // back to the hard-coded flash if nothing is configured at all.
  if (provider.id === 'gemini-cli' && !model) {
    model = provider.lightModel || 'gemini-2.5-flash';
  }

  console.log(`🧠 Calling AI: ${provider.id} / ${model} / ${promptStageName}`);

  // brain runs are headless classification — append the provider's
  // headlessArgs (e.g. claude-code's --no-session-persistence) so the
  // user's session list doesn't fill up with classifier transcripts.
  // The clone leaves the saved provider config untouched.
  const providerForCall = provider.headlessArgs?.length
    ? { ...provider, args: [...(provider.args || []), ...provider.headlessArgs] }
    : provider;

  const { text, model: effectiveModel } = await runPromptThroughProvider({
    provider: providerForCall, prompt, source: `brain-${promptStageName}`, model,
    // Pin the admitted provider so runtime fallback cannot switch to one with tools.
    ...(toolFree && { toolFree: true, allowFallback: false }),
  });
  return { content: text, model: effectiveModel || model, providerId: provider.id, prompt };
}

/**
 * Parse JSON from AI response (handles markdown code blocks)
 */
function parseJsonResponse(content, responseSchema = null, promptToStrip = '') {
  if (!content || typeof content !== 'string') {
    throw new Error('Empty or invalid AI response');
  }

  const shapePredicate = responseSchema
    ? (value) => responseSchema.safeParse(value).success
    : undefined;
  const source = promptToStrip && content.includes(promptToStrip)
    ? content.replace(promptToStrip, '')
    : content;
  const { value, lastError } = extractJson(source, {
    // Brain prompts include fenced JSON examples. Walk all candidates so an
    // echoed example cannot win before the actual answer.
    skipInnerFence: true,
    shapePredicate,
  });
  if (value === undefined) {
    throw new Error(`Failed to parse AI JSON response: ${lastError?.message || 'No JSON found'}`);
  }
  return value;
}

/**
 * Safe version of parseJsonResponse that returns null instead of throwing.
 * Used in background classification where errors can't bubble to middleware.
 *
 * The schema is the single validation authority: it picks the matching JSON
 * candidate AND supplies the returned data. extractJson falls back to the first
 * raw JSON value when no candidate matches; that fallback is discarded here
 * (null) rather than re-validated by the caller.
 */
function safeParseJsonResponse(content, responseSchema, promptToStrip = '') {
  if (!content || typeof content !== 'string') return null;
  const source = promptToStrip && content.includes(promptToStrip)
    ? content.replace(promptToStrip, '')
    : content;
  let matched = null;
  const shapePredicate = (value) => {
    const result = responseSchema.safeParse(value);
    if (result.success) matched = { value, data: result.data };
    return result.success;
  };
  const { value } = extractJson(source, { skipInnerFence: true, shapePredicate });
  return matched && matched.value === value ? matched.data : null;
}

/**
 * Capture a thought and classify it
 * Returns immediately after creating the inbox entry.
 * AI classification runs in the background and emits a socket event on completion.
 */
export async function captureThought(text, providerOverride, modelOverride, { creative = false, repoIntake = null, note } = {}) {
  // A capture that is nothing but a URL is a bookmark, not a thought: file it
  // straight to Links exactly as the Links tab would, and skip the classifier
  // LLM call entirely. This outranks the creative flag — a bare URL carries no
  // prose for the Catalog to work from — so both capture boxes disable the
  // Creative toggle for a URL rather than the two rules disagreeing.
  const bareUrl = parseBareUrl(text);
  if (bareUrl) {
    return captureUrlAsLink(bareUrl, text, { repoIntake, note });
  }

  const meta = await storage.loadMeta();
  const provider = providerOverride || meta.defaultProvider;
  const model = modelOverride || meta.defaultModel;

  // Per-domain autonomy gate: `off` captures the thought but skips auto-classify
  // entirely (it lands in the inbox for manual review); `dry-run` classifies and
  // surfaces the suggestion but doesn't auto-file it.
  const autonomyMode = await getDomainAutonomyMode('brain');

  // Daily brain budget (#711): when today's auto-classify actions/minutes reach
  // the cap, fall back to `off` for the rest of the day — the thought is still
  // captured for manual review, just not auto-classified. A manual retry
  // (retryClassification) is user-initiated and never gated or counted.
  const withinBudget = autonomyMode === 'off' || (await getDomainBudgetStatus('brain')).withinBudget;
  const classify = autonomyMode !== 'off' && withinBudget;
  const budgetPaused = autonomyMode !== 'off' && !withinBudget;

  // Create initial inbox log entry. When auto-classify is off the entry goes
  // straight to needs_review rather than the transient classifying state — and
  // carries no `ai` metadata, since no provider/model was ever invoked.
  const inboxEntry = await storage.createInboxLog({
    capturedText: text,
    source: 'brain_ui',
    ...(creative ? { creative: true } : {}),
    ...(!classify
      ? {}
      : { ai: { providerId: provider, modelId: model, promptTemplateId: 'brain-classifier' } }),
    status: !classify ? 'needs_review' : 'classifying'
  });

  if (!classify) {
    console.log(`🧠 Thought captured, ${budgetPaused ? 'auto-classify daily budget reached' : 'auto-classify is OFF'} — left for manual review: ${inboxEntry.id}`);
    return {
      inboxLog: inboxEntry,
      message: budgetPaused
        ? 'Thought captured! Auto-classify daily budget reached — review it in the inbox.'
        : 'Thought captured! Auto-classify is off — review it in the inbox.'
    };
  }

  console.log(`🧠 Thought captured, classifying in background${autonomyMode === 'dry-run' ? ' (dry-run, no auto-file)' : ''}: ${inboxEntry.id}`);

  // Run AI classification in background (don't await)
  // Pass resolved provider/model so callAI uses brain's configured provider, not the system active one
  classifyInBackground({
    entryId: inboxEntry.id,
    text,
    meta,
    provider,
    model,
    autoFile: autonomyMode !== 'dry-run',
    countsTowardBudget: true
  })
    .catch(err => failClassification(inboxEntry.id, err));

  return {
    inboxLog: inboxEntry,
    message: autonomyMode === 'dry-run'
      ? 'Thought captured! AI is suggesting a classification (dry-run — confirm to file).'
      : 'Thought captured! AI is classifying...'
  };
}

/**
 * File a bare-URL capture to the links collection and log it as an already-filed
 * inbox entry (so the capture is still auditable from the inbox, showing where
 * it went). Re-pasting a URL that's already saved reuses that link instead of
 * failing the capture — the inbox entry then points at the existing bookmark.
 *
 * `repoIntake` carries the capture box's opt-in post-clone agent actions
 * (malware scan / repo study). It applies only to a NEW GitHub link: re-pasting
 * an already-saved URL performs no clone, so there is no fresh clone for the
 * agents to read and re-queueing them silently would be a surprise.
 */
async function captureUrlAsLink(url, capturedText, { repoIntake = null, note } = {}) {
  const existing = await storage.getLinkByUrl(url);
  const link = existing || await createLinkFromUrl(url, { repoIntake, note });

  // No `classification` block: nothing classified this — the destination was
  // decided by shape, not by a model, so there is no confidence or extraction to
  // record. That also keeps the entry safe on a federated peer running older
  // code (inbox records sync): its inbox renders `classification.destination`
  // through a fixed destination map, so an absent classification degrades to the
  // existing "Unknown" badge instead of an unmapped value it can't render.
  const inboxEntry = await storage.createInboxLog({
    capturedText,
    source: 'brain_ui',
    status: 'filed',
    filed: { destination: 'links', destinationId: link.id }
  });

  console.log(`🔗 Captured URL filed to links: ${link.id}${existing ? ' (existing)' : ''} via ${inboxEntry.id}`);

  return {
    inboxLog: inboxEntry,
    link,
    message: existing
      ? 'Already saved in Links.'
      : (link.isRepo ? repoCaptureMessage(link.repoIntake) : 'Saved to Links!')
  };
}

/**
 * What a freshly-captured repo link tells the user will happen next. The
 * clone is always implied; the agent runs only when they ticked the boxes, and
 * they only start once the clone lands.
 */
function repoCaptureMessage(repoIntake) {
  const queued = [
    repoIntake?.malwareScan && 'malware scan',
    repoIntake?.learn && 'repo study',
  ].filter(Boolean);
  return queued.length
    ? `Repo saved — cloning, then queueing ${queued.join(' + ')}.`
    : 'Repo saved to Links — cloning now.';
}

/**
 * Last-resort handler for a throw escaping classifyInBackground (e.g. the filing
 * write failed after a successful AI call). Without it the entry stays
 * `classifying` until the next boot's recoverStuckClassifications.
 */
async function failClassification(entryId, err) {
  console.error(`❌ Background classification failed for ${entryId}: ${err.message}`);
  const updated = await storage.updateInboxLog(entryId, {
    status: 'needs_review',
    error: { message: err.message }
  }).catch(updateErr => {
    console.error(`❌ Failed to mark ${entryId} needs_review: ${updateErr.message}`);
    return null;
  });
  if (updated) brainEvents.emit('classified', { entryId, status: 'needs_review', error: err.message });
}

/**
 * Background AI classification for a captured thought.
 * Updates the inbox entry and emits a brain:classified event when done.
 */
async function classifyInBackground({ entryId, text, meta, provider, model, autoFile = true, countsTowardBudget = false }) {
  let classification = null;
  let aiError = null;

  const startTime = Date.now();
  const aiResult = await callAI(
    'brain-classifier',
    { capturedText: text, now: new Date().toISOString() },
    provider,
    model
  ).catch(err => {
    aiError = err;
    return null;
  });

  const durationMs = Date.now() - startTime;
  const elapsed = (durationMs / 1000).toFixed(1);

  // Daily brain budget accounting (#711): an auto-classify consumed compute
  // whether or not it parsed, so count the attempt + its time. Only the auto
  // path (captureThought) passes countsTowardBudget — a manual retry doesn't count.
  if (countsTowardBudget) {
    await recordDomainUsage('brain', { actions: 1, ms: durationMs })
      .catch(err => console.error(`❌ Failed to record brain budget usage for ${entryId}: ${err.message}`));
  }
  const aiResponse = aiResult?.content;
  // Patch ai metadata so the inbox entry reflects the model actually invoked
  // (captureThought/retryClassification stored the pre-resolved value, which may
  // have been null when gemini-cli fell back internally).
  const aiMeta = aiResult
    ? { providerId: aiResult.providerId, modelId: aiResult.model, promptTemplateId: 'brain-classifier' }
    : null;

  if (aiResponse) {
    console.log(`🧠 AI responded in ${elapsed}s for ${entryId}`);
    classification = safeParseJsonResponse(aiResponse, classifierOutputSchema, aiResult.prompt);
    if (!classification) aiError = new Error('Could not parse a valid classification from AI response');
  } else {
    console.log(`🧠 AI failed after ${elapsed}s for ${entryId}`);
  }

  // If AI failed, mark as needs_review
  if (!classification) {
    const errorMessage = aiError?.message || 'AI classification failed';
    await storage.updateInboxLog(entryId, {
      ...(aiMeta ? { ai: aiMeta } : {}),
      classification: {
        destination: 'unknown',
        confidence: 0,
        title: 'Classification failed',
        extracted: {},
        reasons: [errorMessage]
      },
      status: 'needs_review',
      error: { message: errorMessage }
    });

    console.log(`🧠 Classification failed for ${entryId}: ${errorMessage}`);
    brainEvents.emit('classified', { entryId, status: 'needs_review', error: errorMessage });
    return;
  }

  // Check confidence threshold
  if (classification.confidence < meta.confidenceThreshold || classification.destination === 'unknown') {
    await storage.updateInboxLog(entryId, {
      ai: aiMeta,
      classification,
      status: 'needs_review'
    });

    console.log(`🧠 Low confidence (${classification.confidence}) for ${entryId}`);
    brainEvents.emit('classified', { entryId, status: 'needs_review', confidence: classification.confidence });
    return;
  }

  // Dry-run: a confident classification was produced, but the auto-file side
  // effect is withheld. Surface the suggestion as needs_review so the user can
  // confirm filing it from the inbox.
  if (!autoFile) {
    await storage.updateInboxLog(entryId, {
      ai: aiMeta,
      classification,
      status: 'needs_review'
    });
    console.log(`🧠 [dry-run] Classified ${entryId} → ${classification.destination} (not filed; awaiting confirmation)`);
    brainEvents.emit('classified', {
      entryId,
      status: 'needs_review',
      destination: classification.destination,
      title: classification.title,
      dryRun: true
    });
    return;
  }

  // File to appropriate destination
  const filedRecord = await fileToDestination(classification.destination, classification.extracted, classification.title);

  await storage.updateInboxLog(entryId, {
    ai: aiMeta,
    classification,
    status: 'filed',
    filed: {
      destination: classification.destination,
      destinationId: filedRecord.id
    }
  });

  console.log(`🧠 Classified and filed to ${classification.destination}: ${filedRecord.id}`);
  brainEvents.emit('classified', {
    entryId,
    status: 'filed',
    destination: classification.destination,
    title: classification.title
  });
}

/**
 * File extracted data to destination database
 */
async function fileToDestination(destination, extracted, title) {
  const validator = EXTRACTED_VALIDATORS[destination];
  if (!validator) {
    throw new Error(`Unknown destination: ${destination}`);
  }

  // Validate and set defaults
  const validationResult = validator.safeParse(extracted);
  const data = validationResult.success ? validationResult.data : extracted;

  switch (destination) {
    case 'people':
      return storage.createPerson({
        name: data.name || title,
        context: data.context || '',
        followUps: data.followUps || [],
        lastTouched: data.lastTouched || null,
        tags: data.tags || []
      });

    case 'projects':
      return storage.createProject({
        name: data.name || title,
        status: data.status || 'active',
        nextAction: data.nextAction || 'Define next action',
        notes: data.notes || '',
        tags: data.tags || []
      });

    case 'ideas':
      return storage.createIdea({
        title: data.title || title,
        oneLiner: data.oneLiner || title,
        notes: data.notes || '',
        tags: data.tags || []
      });

    case 'admin':
      return storage.createAdminItem({
        title: data.title || title,
        status: data.status || 'open',
        dueDate: data.dueDate || null,
        nextAction: data.nextAction || null,
        notes: data.notes || ''
      });

    case 'memories':
      return storage.createMemoryEntry({
        title: data.title || title,
        content: data.content || '',
        mood: data.mood || null,
        tags: data.tags || []
      });

    default:
      throw new Error(`Cannot file to destination: ${destination}`);
  }
}

/**
 * Title to file an inbox entry under. Not every entry carries a classification —
 * a URL filed to Links skipped the classifier, and so does every capture taken
 * while auto-classify is off — so fall back to the captured text (bounded to the
 * 200-char record-title limit) before the last-resort placeholder.
 */
function entryTitle(inboxLog) {
  return inboxLog.classification?.title
    || inboxLog.capturedText?.trim().slice(0, 200)
    || 'Untitled';
}

/**
 * Resolve a needs_review inbox item
 */
export async function resolveReview(inboxLogId, destination, editedExtracted) {
  const inboxLog = await storage.getInboxLogById(inboxLogId);
  if (!inboxLog) {
    throw new Error('Inbox log entry not found');
  }

  if (inboxLog.status !== 'needs_review') {
    throw new Error('Inbox entry is not in needs_review status');
  }

  // Merge extracted data with edits
  const extracted = { ...inboxLog.classification?.extracted, ...editedExtracted };
  const title = entryTitle(inboxLog);

  // File to destination
  const filedRecord = await fileToDestination(destination, extracted, title);

  // Update inbox log
  await storage.updateInboxLog(inboxLogId, {
    classification: {
      ...inboxLog.classification,
      destination,
      extracted,
      confidence: 1.0,
      reasons: [...(inboxLog.classification?.reasons || []), 'Manually resolved']
    },
    status: 'filed',
    filed: {
      destination,
      destinationId: filedRecord.id
    }
  });

  console.log(`🧠 Resolved review to ${destination}: ${filedRecord.id}`);
  return {
    inboxLog: await storage.getInboxLogById(inboxLogId),
    filedRecord
  };
}

/**
 * Fix/correct a filed inbox item
 */
export async function fixClassification(inboxLogId, newDestination, updatedFields, note) {
  const inboxLog = await storage.getInboxLogById(inboxLogId);
  if (!inboxLog) {
    throw new Error('Inbox log entry not found');
  }

  if (inboxLog.status !== 'filed' && inboxLog.status !== 'corrected') {
    throw new Error('Can only fix filed or previously corrected entries');
  }

  const previousDestination = inboxLog.filed?.destination || inboxLog.classification?.destination;
  const previousId = inboxLog.filed?.destinationId;

  // Create new record in new destination
  const extracted = { ...inboxLog.classification?.extracted, ...updatedFields };
  const title = entryTitle(inboxLog);
  const newRecord = await fileToDestination(newDestination, extracted, title);

  // Mark old record as archived (soft delete by adding archived flag)
  if (previousId && previousDestination) {
    await archiveRecord(previousDestination, previousId);
  }

  // Update inbox log with correction info
  await storage.updateInboxLog(inboxLogId, {
    status: 'corrected',
    filed: {
      destination: newDestination,
      destinationId: newRecord.id
    },
    correction: {
      correctedAt: new Date().toISOString(),
      previousDestination: previousDestination || 'unknown',
      newDestination,
      note
    }
  });

  console.log(`🧠 Fixed classification from ${previousDestination} to ${newDestination}`);
  return {
    inboxLog: await storage.getInboxLogById(inboxLogId),
    newRecord
  };
}

/**
 * Archive a record (soft delete)
 */
async function archiveRecord(destination, id) {
  // A bookmark has no archived state — the Links tab renders every record — so
  // correcting a bare-URL capture into a real destination removes the link it
  // auto-created, rather than leaving it behind in Links.
  if (destination === 'links') {
    await storage.deleteLink(id);
    return;
  }

  const updateFn = {
    people: storage.updatePerson,
    projects: storage.updateProject,
    ideas: storage.updateIdea,
    admin: storage.updateAdminItem,
    memories: storage.updateMemoryEntry
  }[destination];

  if (updateFn) {
    await updateFn(id, { archived: true });
  }
}

/**
 * Run daily digest
 */
export async function runDailyDigest(providerOverride, modelOverride) {
  const meta = await storage.loadMeta();

  // Gather data for digest
  const [activeProjects, openAdmin, allPeople, needsReviewLogs] = await Promise.all([
    storage.getProjects({ status: 'active' }),
    storage.getAdminItems({ status: 'open' }),
    storage.getPeople(),
    storage.getInboxLog({ status: 'needs_review' })
  ]);

  // Filter people with follow-ups
  const peopleWithFollowUps = allPeople.filter(p => p.followUps && p.followUps.length > 0);

  // Skip AI call when brain has no data (fresh instance)
  if (!activeProjects.length && !openAdmin.length && !peopleWithFollowUps.length && !needsReviewLogs.length) {
    console.log('🧠 Skipping daily digest: no brain data yet');
    await storage.updateMeta({ lastDailyDigest: new Date().toISOString() });
    return null;
  }

  const aiResult = await callAI(
    'brain-daily-digest',
    {
      activeProjects: JSON.stringify(activeProjects),
      openAdmin: JSON.stringify(openAdmin),
      peopleFollowUps: JSON.stringify(peopleWithFollowUps),
      needsReview: JSON.stringify(needsReviewLogs),
      now: new Date().toISOString()
    },
    providerOverride || meta.defaultProvider,
    modelOverride || meta.defaultModel,
    { toolFree: true }
  );

  const parsed = parseJsonResponse(aiResult.content, digestOutputSchema, aiResult.prompt);
  const validationResult = digestOutputSchema.safeParse(parsed);

  if (!validationResult.success) {
    throw new Error(`Invalid digest output: ${JSON.stringify(validationResult.error.issues)}`);
  }

  const digestData = validationResult.data;

  // Enforce word limit
  const wordCount = digestData.digestText.split(/\s+/).length;
  if (wordCount > 150) {
    digestData.digestText = digestData.digestText.split(/\s+/).slice(0, 150).join(' ') + '...';
  }

  // Store digest — ai.modelId reflects the resolved model so attribution stays
  // accurate even when callAI falls back (e.g., gemini-cli → gemini-2.5-flash).
  const digest = await storage.createDigest({
    ...digestData,
    ai: {
      providerId: aiResult.providerId,
      modelId: aiResult.model,
      promptTemplateId: 'brain-daily-digest'
    }
  });

  console.log(`🧠 Generated daily digest: ${digest.id}`);
  return digest;
}

/**
 * Run weekly review
 */
export async function runWeeklyReview(providerOverride, modelOverride) {
  const meta = await storage.loadMeta();

  // Get inbox log from last 7 days
  const sevenDaysAgo = new Date(Date.now() - 7 * 24 * 60 * 60 * 1000).toISOString();
  const allInboxLogs = await storage.getInboxLog({ limit: 500 });
  const recentInboxLogs = allInboxLogs.filter(log => log.capturedAt >= sevenDaysAgo);

  // Get active projects
  const activeProjects = await storage.getProjects({ status: 'active' });

  // Skip AI call when brain has no data (fresh instance)
  if (!recentInboxLogs.length && !activeProjects.length) {
    console.log('🧠 Skipping weekly review: no brain data yet');
    await storage.updateMeta({ lastWeeklyReview: new Date().toISOString() });
    return null;
  }

  const aiResult = await callAI(
    'brain-weekly-review',
    {
      inboxLogLast7Days: JSON.stringify(recentInboxLogs),
      activeProjects: JSON.stringify(activeProjects),
      now: new Date().toISOString()
    },
    providerOverride || meta.defaultProvider,
    modelOverride || meta.defaultModel,
    { toolFree: true }
  );

  const parsed = parseJsonResponse(aiResult.content, reviewOutputSchema, aiResult.prompt);
  const validationResult = reviewOutputSchema.safeParse(parsed);

  if (!validationResult.success) {
    throw new Error(`Invalid review output: ${JSON.stringify(validationResult.error.issues)}`);
  }

  const reviewData = validationResult.data;

  // Enforce word limit
  const wordCount = reviewData.reviewText.split(/\s+/).length;
  if (wordCount > 250) {
    reviewData.reviewText = reviewData.reviewText.split(/\s+/).slice(0, 250).join(' ') + '...';
  }

  // Store review — ai.modelId reflects the resolved model so attribution stays
  // accurate even when callAI falls back (e.g., gemini-cli → gemini-2.5-flash).
  const review = await storage.createReview({
    ...reviewData,
    ai: {
      providerId: aiResult.providerId,
      modelId: aiResult.model,
      promptTemplateId: 'brain-weekly-review'
    }
  });

  console.log(`🧠 Generated weekly review: ${review.id}`);
  return review;
}

/**
 * Retry classification for a needs_review item.
 * Returns immediately after setting status to 'classifying'.
 * AI classification runs in the background and emits a socket event on completion.
 */
export async function retryClassification(inboxLogId, providerOverride, modelOverride) {
  const inboxLog = await storage.getInboxLogById(inboxLogId);
  if (!inboxLog) {
    throw new Error('Inbox log entry not found');
  }

  const meta = await storage.loadMeta();
  const provider = providerOverride || meta.defaultProvider;
  const model = modelOverride || meta.defaultModel;

  // Set status to classifying so UI shows spinner
  await storage.updateInboxLog(inboxLogId, {
    ai: {
      providerId: provider,
      modelId: model,
      promptTemplateId: 'brain-classifier'
    },
    status: 'classifying',
    error: null
  });

  console.log(`🧠 Retrying classification in background: ${inboxLogId}`);

  // Run AI classification in background (don't await)
  // A manual retry is a user decision to file: it skips the autonomy gate and isn't budget-counted.
  classifyInBackground({
    entryId: inboxLogId,
    text: inboxLog.capturedText,
    meta,
    provider,
    model,
    autoFile: true,
    countsTowardBudget: false
  })
    .catch(err => failClassification(inboxLogId, err));

  return {
    inboxLog: await storage.getInboxLogById(inboxLogId),
    message: 'Retrying classification...'
  };
}

/**
 * Mark inbox entry as done
 */
export async function markInboxDone(inboxLogId) {
  const inboxLog = await storage.getInboxLogById(inboxLogId);
  if (!inboxLog) {
    return null;
  }

  const updated = await storage.updateInboxLog(inboxLogId, {
    status: 'done',
    doneAt: new Date().toISOString()
  });

  console.log(`🧠 Marked inbox entry done: ${inboxLogId}`);
  return updated;
}

/**
 * Mark a batch of creative inbox notes as consumed by a catalog ingest that just
 * committed — stamps `sentToCatalogAt` so they drop out of the inbox's "ready to
 * become ingredients" banner and can't be accidentally re-sent. Idempotent and
 * forgiving: ids that no longer exist (deleted/tombstoned) are skipped silently
 * so a partially-stale list still stamps the rest. Returns the updated entries.
 */
export async function markInboxSentToCatalog(ids) {
  const sentToCatalogAt = new Date().toISOString();
  // Single batched write (one store rewrite + one sync-log pass) — updateMany
  // already skips unknown/tombstoned ids, preserving the "stamp the rest" contract.
  const updated = await storage.updateMany('inbox', ids.map(id => ({ id, sentToCatalogAt })));
  console.log(`🧠 Marked ${updated.length} creative note(s) sent to catalog`);
  return updated;
}

/**
 * Update inbox entry (edit captured text)
 */
export async function updateInboxEntry(inboxLogId, updates) {
  const updated = await storage.updateInboxLog(inboxLogId, updates);
  if (!updated) {
    return null;
  }

  console.log(`🧠 Updated inbox entry text: ${inboxLogId}`);
  return updated;
}

/**
 * Delete inbox entry
 */
export async function deleteInboxEntry(inboxLogId) {
  const deleted = await storage.deleteInboxLog(inboxLogId);
  if (!deleted) {
    return false;
  }

  console.log(`🧠 Deleted inbox entry: ${inboxLogId}`);
  return true;
}

/**
 * Recover inbox entries stuck in 'classifying' status from a previous server restart.
 * Resets them to 'needs_review' so the user can retry.
 *
 * Only touches entries THIS instance originated. Now that inbox rows are synced
 * brain records, a peer's entry can be legitimately mid-classification (status
 * 'classifying') on ITS machine while we hold a synced copy. Flipping that to
 * 'needs_review' here would stamp a fresh updatedAt and, as the LWW winner,
 * clobber the origin's real classification on the next sync. Classification only
 * ever runs on the capturing (origin) machine, so a stuck 'classifying' entry is
 * only ours to recover when we created it. (Pre-091 records with no
 * originInstanceId are treated as local — backfillOriginInstanceId stamps them
 * with our id at boot anyway, so they ARE ours.)
 */
export async function recoverStuckClassifications() {
  const instanceId = await getInstanceId();
  const entries = await storage.getInboxLog({ status: 'classifying', limit: 100 });
  let recovered = 0;
  for (const entry of entries) {
    if (entry.originInstanceId && entry.originInstanceId !== instanceId) continue;
    await storage.updateInboxLog(entry.id, { status: 'needs_review' });
    recovered++;
    console.log(`🧠 Recovered stuck classification: ${entry.id}`);
  }
  if (recovered > 0) {
    console.log(`🧠 Recovered ${recovered} stuck classification(s)`);
  }
}

// Matches the client's clone stall window (#5442). Past it, a `cloning` record
// this install cannot attribute to itself is treated as orphaned rather than
// left in place — see `shouldRecoverInterruptedClone`.
const CLONE_STALE_MS = 10 * 60 * 1000;

/**
 * Whether THIS install should reset `link` out of `cloning` at boot.
 *
 * Ours, or unattributed (pre-#5463 records carry no `cloneInstanceId`) — always,
 * because no in-process clone survives a restart. A clone another instance owns
 * is left alone while it could still be running, but ages out of that
 * protection: a peer that crashed mid-clone and never comes back must not
 * strand the link at `cloning` forever, which is the whole bug (#5463).
 */
const shouldRecoverInterruptedClone = (link, instanceId) => {
  const owner = link.cloneInstanceId ?? link.originInstanceId ?? null;
  if (!owner || owner === instanceId) return true;
  const updatedAt = Date.parse(link.updatedAt);
  return Number.isFinite(updatedAt) && Date.now() - updatedAt >= CLONE_STALE_MS;
};

/**
 * Recover repository clones interrupted by a previous server shutdown.
 * Clone work is published only after git exits successfully; an orphaned child
 * can keep writing its private staging directory without racing a retry.
 */
export async function recoverInterruptedRepoClones() {
  // `ensureInstanceId`, not `getInstanceId`: this comparison decides a durable
  // record mutation, and the sentinel would match every other uninitialized
  // install's records while missing our own.
  const instanceId = await ensureInstanceId();
  const links = await storage.getLinks({ cloneStatus: 'cloning' });
  let recovered = 0;
  for (const link of links) {
    // Keep the state guard even though storage applies the filter so a future
    // storage implementation cannot broaden this boot-time mutation by accident.
    if (link.cloneStatus !== 'cloning') continue;
    if (!shouldRecoverInterruptedClone(link, instanceId)) continue;
    await storage.updateLink(link.id, {
      cloneStatus: 'failed',
      cloneError: 'Clone interrupted by a server restart. Retry to clone the repository again.',
      cloneInstanceId: null,
      cloneInterrupted: true
    });
    recovered++;
  }
  if (recovered > 0) {
    console.log(`🧠 Recovered ${recovered} interrupted repository clone(s)`);
  }
  // Detached on purpose: boot AWAITS this function so the first links request
  // can't see an orphaned `cloning` badge, and a recursive scan plus `rm -rf` of
  // abandoned partial checkouts must not sit in front of `startListening()`.
  repoCloner.reapStaleCloneStaging()
    .catch(err => console.error(`❌ Clone staging recovery failed: ${err.message}`));
}

// Re-export storage functions for convenience
export const loadMeta = storage.loadMeta;
export const updateMeta = storage.updateMeta;
export const getSummary = storage.getSummary;
export const getInboxLog = storage.getInboxLog;
export const getInboxPage = storage.getInboxPage;
export const getInboxLogById = storage.getInboxLogById;
export const getInboxLogCounts = storage.getInboxLogCounts;
export const getEntityPage = storage.getEntityPage;
export const toMemoryListItem = storage.toMemoryListItem;
export const getPeople = storage.getPeople;
export const getPersonById = storage.getPersonById;
export const createPerson = storage.createPerson;
export const updatePerson = storage.updatePerson;
export const deletePerson = storage.deletePerson;
export const getProjects = storage.getProjects;
export const getProjectById = storage.getProjectById;
export const createProject = storage.createProject;
export const updateProject = storage.updateProject;
export const deleteProject = storage.deleteProject;
export const getIdeas = storage.getIdeas;
export const getIdeaById = storage.getIdeaById;
export const createIdea = storage.createIdea;
export const updateIdea = storage.updateIdea;
export const deleteIdea = storage.deleteIdea;
export const getAdminItems = storage.getAdminItems;
export const getAdminById = storage.getAdminById;
export const createAdminItem = storage.createAdminItem;
export const updateAdminItem = storage.updateAdminItem;
export const deleteAdminItem = storage.deleteAdminItem;
export const getDigests = storage.getDigests;
export const getLatestDigest = storage.getLatestDigest;
export const getReviews = storage.getReviews;
export const getLatestReview = storage.getLatestReview;
export const getMemoryEntries = storage.getMemoryEntries;
export const getMemoryEntryById = storage.getMemoryEntryById;
export const createMemoryEntry = storage.createMemoryEntry;
export const updateMemoryEntry = storage.updateMemoryEntry;

/**
 * Delete a memory entry AND clean up its on-disk assets.
 *
 * A `chatgpt-import` memory references an archived transcript (`sourceRef`) and
 * any number of served asset files (images/audio/PDFs) embedded in its markdown
 * `content`. The storage-level delete only tombstones the JSON record — without
 * this wrapper those files orphan forever under `data/brain/imports/`. We load
 * the record first (to know what it referenced) and the surviving import
 * memories (so a shared transcript/asset still in use — e.g. the same export
 * imported twice — isn't pulled out from under them).
 */
export const deleteMemoryEntry = (id) => withBackupAssetPublication(() => deleteMemoryEntryLeased(id));

// The record drop and the asset/transcript unlinks are one backup workflow (#9982).
async function deleteMemoryEntryLeased(id) {
  const record = await storage.getMemoryEntryById(id);
  const deleted = await storage.deleteMemoryEntry(id);
  if (deleted && record?.source === 'chatgpt-import') {
    const survivors = (await storage.getMemoryEntries())
      .filter((m) => m.id !== id && m.source === 'chatgpt-import');
    await deleteMemoryAssets(record, survivors);
  }
  return deleted;
}

export const getLinks = storage.getLinks;
// Paginated read path — filters/sorts/counts off the cached link summary index
// and loads only the requested page's bodies (issue #3509).
export const getLinksPage = storage.getLinksPage;
export const listLinkIds = storage.listLinkIds;
export const getLinkById = storage.getLinkById;
export const getLinkByUrl = storage.getLinkByUrl;
export const updateLink = storage.updateLink;
export const reorderLinks = storage.reorderLinks;
export const deleteLink = storage.deleteLink;

export const getBuckets = storage.getBuckets;
export const getBucketById = storage.getBucketById;
export const createBucket = storage.createBucket;
export const updateBucket = storage.updateBucket;
export const reorderBuckets = storage.reorderBuckets;
export const deleteBucket = storage.deleteBucket;

/**
 * Create a bucket appended after all existing ones (next-order logic).
 * Folds the next-order computation into the service so the route stays thin.
 */
export async function createBucketAppended({ name, color, icon }) {
  const existing = await storage.getBuckets();
  const nextOrder = existing.reduce((max, b) => Math.max(max, b.order ?? 0), -1) + 1;
  return storage.createBucket({ name, color: color || 'accent', icon: icon || '', order: nextOrder });
}

/**
 * Delete a bucket and unassign (bucketId → null) all links that belonged to it.
 * Links survive; they fall back to the ungrouped list rather than being orphaned.
 */
export async function deleteBucketAndUnlinkChildren(id) {
  const links = await storage.getLinks();
  let unassigned = 0;
  for (const link of links) {
    if (link.bucketId === id) {
      await storage.updateLink(link.id, { bucketId: null });
      unassigned++;
    }
  }
  await storage.deleteBucket(id);
  return { deleted: true, unassigned };
}

/**
 * Extract a clean hostname from a URL (strip a leading www.), or null if unparseable.
 */
function hostnameFromUrl(url) {
  return URL.parse(url)?.hostname.replace(/^www\./, '') ?? null;
}

/**
 * Clone a repo in the background, tracking progress on the link record.
 * Runs outside the request lifecycle, so every failure is caught and recorded
 * on the link rather than left to bubble.
 *
 * The opt-in agent actions the user ticked at capture time are read off the
 * link's own `repoIntake` field, not passed in — so EVERY path that reaches a
 * successful clone honors them, including the Links tab's Clone/Retry button
 * after a first clone failed. They dispatch only after the clone SUCCEEDS:
 * there is nothing on disk to read before that, and a failed clone must not
 * queue an agent against a path that doesn't exist.
 */
export async function cloneRepoInBackground(linkId, url) {
  const previous = await storage.getLinkById(linkId);
  // `ensureInstanceId`, and null rather than the sentinel: `cloneInstanceId`
  // lands on a durable, federated record, and a stamped 'unknown' would read as
  // "some other install owns this" at boot on every machine — re-stranding the
  // exact record this recovery exists to free.
  const resolvedInstanceId = await ensureInstanceId();
  const cloneInstanceId = resolvedInstanceId === UNKNOWN_INSTANCE_ID ? null : resolvedInstanceId;
  await storage.updateLink(linkId, {
    cloneStatus: 'cloning',
    // The Links tab renders `cloneError` whenever it is set, independent of
    // status — leaving the previous attempt's message would show the failure
    // text beside the new attempt's spinner.
    cloneError: null,
    cloneInstanceId,
    cloneInterrupted: false
  });

  repoCloner.cloneRepo(url, { replaceIncomplete: previous?.cloneInterrupted === true })
    .then(async (result) => {
      const link = await storage.updateLink(linkId, {
        localPath: result.localPath,
        cloneStatus: 'cloned',
        cloneError: null,
        cloneInstanceId: null,
        cloneInterrupted: false
      });
      console.log(`✅ Background clone complete: ${linkId}`);
      // `link` is null when the user deleted the bookmark mid-clone — nothing
      // left to scan or study. The whole intake is chained off its OWN catch, so
      // a queueing failure can't rewrite a clone that genuinely succeeded as
      // `failed` via the outer handler below.
      if (!link?.repoIntake) return;
      // Dynamic on purpose, and NOT for boot cost — routes/brainLinks.js already
      // imports repoIntake.js statically, so the CoS task graph is loaded either
      // way. It keeps that graph out of *brain.js's own module graph*, which the
      // heavily-mocked brain.test.js suite instantiates; a static edge drags
      // cos.js → taskSchedule.js in and the suite dies on an incomplete mock.
      // Same reason malwareScanReports.js imports brain.js this way.
      await import('./repoIntake.js')
        .then(({ runRepoIntake }) => runRepoIntake(link, link.repoIntake))
        .then(patch => (Object.keys(patch).length ? storage.updateLink(linkId, patch) : null))
        .catch(err => console.error(`❌ Post-clone intake failed for ${linkId}: ${err.message}`));
    })
    .catch(async (err) => {
      await storage.updateLink(linkId, {
        cloneStatus: 'failed',
        cloneError: err.message,
        cloneInstanceId: null,
        cloneInterrupted: false
      });
      console.error(`❌ Background clone failed: ${linkId} - ${err.message}`);
    });
}

/**
 * Create a link from a URL: derives the repository metadata + a readable default
 * title and kicks off the background clone for a repo. Shared by the Links
 * route's quick-add and the bare-URL capture short-circuit so a URL pasted into
 * the Brain inbox lands exactly as it would from the Links tab.
 *
 * Callers own duplicate handling (the route 409s; capture reuses the existing
 * link) — this always creates.
 */
export async function createLinkFromUrl(url, {
  title, description, note, linkType, tags, bucketId, bucketOrder, autoClone, repoIntake
} = {}) {
  const repoFields = deriveRepoLinkFields(url);
  const shouldClone = repoFields.isRepo && autoClone !== false;
  // Opt-in post-clone agent actions. Only meaningful when a clone will actually
  // happen, and persisted on the link so the record says what was asked for even
  // if the clone is still running.
  const intake = shouldClone ? normalizeRepoIntake(repoIntake) : null;

  // Derive a readable default title: repo slug for a repository, hostname for
  // plain URLs (so quick-added bucket chips read "example.com" instead of the
  // full URL).
  const defaultTitle = repoFields.isRepo
    ? `${repoFields.repoOwner}/${repoFields.repoName}`
    : (hostnameFromUrl(url) || url);
  const cleanNote = typeof note === 'string' ? note.trim() : '';

  const link = await storage.createLink({
    url,
    title: title || defaultTitle,
    description: description || '',
    note: cleanNote,
    linkType: linkType || (repoFields.isRepo ? 'repo' : 'other'),
    tags: tags || [],
    ...repoFields,
    localPath: null,
    cloneStatus: shouldClone ? 'pending' : 'none',
    cloneError: null,
    ...(intake ? { repoIntake: intake } : {}),
    ...(bucketId !== undefined ? { bucketId } : {}),
    ...(bucketOrder !== undefined ? { bucketOrder } : {})
  });
  console.log(`🔗 Created link: ${link.id} (${repoFields.isRepo ? `${repoFields.repoHost} repo` : 'regular URL'})`);

  if (shouldClone) {
    cloneRepoInBackground(link.id, url).catch(err => {
      console.error(`❌ Background clone setup failed for ${link.id}: ${err.message}`);
    });
  }

  return link;
}