Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
/**
* Ask Conversations
*
* Persistent storage for "Ask Yourself" conversations — the chat transcript
* the user has with their digital twin, with sources cited per turn.
*
* Each conversation is one JSON file under data/ask-conversations/<id>.json.
* Files older than 30 days are auto-pruned on list, unless the user has
* pinned them via setPromoted().
*
* Conversation shape:
* { id, title, mode, createdAt, updatedAt, promoted, turns[] }
* Turn shape:
* { id, role: 'user'|'assistant', content, sources?, mode?, providerId?, model?, createdAt }
*
* Conversation ids are sortable: `ask_<base36-ms-padded>_<hex>`. Lexically
* sorting filenames descending returns newest-first without reading any
* file, which lets `listConversations` JSON-parse only the head of the
* list to fill summaries. Beyond the page cutoff we `stat()` each file
* (mtime fast path) and only open the JSON for files whose mtime is older
* than the 30-day expiry — those need a JSON read to honour the `promoted`
* flag. Net cost: O(limit) JSON reads + O(N) stats for fresh tails, plus
* O(stale-files) JSON reads for the prune sweep.
*/
import { join } from 'path';
import { readdir, stat } from 'fs/promises';
import { randomUUID } from 'crypto';
import { PATHS, atomicWrite, ensureDir, readJSONFile, safeDate, unlinkGuarded } from '../lib/fileUtils.js';
export const ASK_DIR = join(PATHS.data, 'ask-conversations');
export const EXPIRY_DAYS = 30;
export const TITLE_MAX_LENGTH = 120;
// Owned here (the leaf storage module) and consumed by askService. Keeping
// this declaration on the storage side means the persistence layer has no
// upstream imports — a fresh test can mount this file alone without pulling
// the retrieval/provider stack via askService.
export const VALID_MODES = new Set(['ask', 'advise', 'draft']);
// Pad base36-ms to 9 chars (36^9 > 1e14, valid until year ~5100) so
// filename lexical order always matches chronological order — without
// padding, a digit rollover would make a newer id sort *before* an older
// one and the newest-first listing would silently flip. The hex suffix is
// the first 8 chars of a UUID. The regex is locked to those exact widths
// so non-canonical ids (which would land on disk with the wrong sort key)
// never pass validation at any layer.
const BASE36_TS_WIDTH = 9;
const HEX_SUFFIX_WIDTH = 8;
export const ID_RE = new RegExp(`^ask_[a-z0-9]{${BASE36_TS_WIDTH}}_[a-f0-9]{${HEX_SUFFIX_WIDTH}}$`);
function generateId() {
const ts = Date.now().toString(36).padStart(BASE36_TS_WIDTH, '0');
return `ask_${ts}_${randomUUID().split('-')[0]}`;
}
export function isValidId(id) {
return typeof id === 'string' && ID_RE.test(id);
}
function pathFor(id) {
if (!isValidId(id)) throw new Error(`Invalid conversation id: ${id}`);
return join(ASK_DIR, `${id}.json`);
}
function truncateTitle(text) {
if (typeof text !== 'string') return '';
const trimmed = text.trim().replace(/\s+/g, ' ');
if (trimmed.length <= TITLE_MAX_LENGTH) return trimmed;
return trimmed.slice(0, TITLE_MAX_LENGTH - 1) + '…';
}
async function readConversation(id, { strict = true } = {}) {
const data = await readJSONFile(pathFor(id), null, { logError: false, strict });
if (!data || typeof data !== 'object' || data.id !== id) return null;
return data;
}
export async function getConversation(id) {
if (!isValidId(id)) return null;
return readConversation(id);
}
export async function listConversations({ limit = 50 } = {}) {
await ensureDir(ASK_DIR);
const entries = await readdir(ASK_DIR).catch((err) => {
if (err.code === 'ENOENT') return [];
throw err;
});
// Filenames carry a sortable timestamp prefix (`ask_<base36-ms-padded>_…`)
// so a descending lexical sort gives newest-first ordering without
// reading any file. The two-tier sweep below uses this: JSON-read the
// head page for summaries, `stat()` everything else and only JSON-read
// tail files whose mtime puts them past the expiry window (those are
// pruning candidates and we still need the JSON to honour `promoted`).
const ids = entries
.filter((f) => f.endsWith('.json'))
.map((f) => f.slice(0, -5))
.filter(isValidId)
.sort((a, b) => b.localeCompare(a));
const summaries = [];
const now = Date.now();
const expiryMs = EXPIRY_DAYS * 24 * 60 * 60 * 1000;
// Two-tier sweep: filenames are newest-first, so we only read JSON for
// (a) the head of the list to fill `summaries`, and (b) anything whose
// mtime is older than the expiry window — those are pruning candidates
// and we still have to open them to honour the `promoted` flag. Fresh
// files past the `limit` cutoff are skipped entirely.
for (const id of ids) {
const headSlot = summaries.length < limit;
let conv = null;
let mtime = null;
if (headSlot) {
// Listing aggregates independent files and never rewrites a skipped
// record, so one unreadable conversation must not hide every healthy one.
conv = await readConversation(id, { strict: false });
if (!conv) continue;
} else {
// Cheap check first — only open the JSON if mtime indicates the file
// could be expired. atomicWrite touches mtime on every update.
mtime = await stat(pathFor(id)).then((s) => s.mtimeMs, () => null);
if (mtime === null) continue;
if ((now - mtime) <= expiryMs) continue;
// A failed read returns null and therefore cannot enter the prune path;
// preserve that file while the sweep continues over independent records.
conv = await readConversation(id, { strict: false });
if (!conv) continue;
}
// Fall back to file mtime when the JSON's own timestamps are missing or
// unparseable — a corrupted record with no usable updatedAt would
// otherwise live forever because `safeDate(undefined) === 0` made the
// expiry comparison vacuously false.
let effectiveTs = safeDate(conv.updatedAt) || safeDate(conv.createdAt);
if (!effectiveTs) {
if (mtime === null) {
mtime = await stat(pathFor(id)).then((s) => s.mtimeMs, () => null);
}
if (mtime !== null) effectiveTs = mtime;
}
if (!conv.promoted && effectiveTs && (now - effectiveTs) > expiryMs) {
// Single-user app — no concurrency races to worry about pruning here.
await unlinkGuarded(pathFor(id)).catch(() => {});
continue;
}
if (headSlot) {
summaries.push({
id: conv.id,
title: conv.title || '(untitled)',
mode: conv.mode,
createdAt: conv.createdAt,
updatedAt: conv.updatedAt,
turnCount: Array.isArray(conv.turns) ? conv.turns.length : 0,
// Count promotable assistant answers separately: a conversation can have
// turns (turnCount > 0) with no promotable answer when an Ask stream
// errors or the client disconnects — either before any assistant text is
// persisted (user turn only) OR after only whitespace deltas (a blank
// assistant turn). Promotion gates on a NON-EMPTY assistant turn (see
// latestAssistantTurn in askPromote.js, which `.trim()`s content), so this
// count must apply the same predicate or it would advertise a dead-end row.
assistantTurnCount: Array.isArray(conv.turns)
? conv.turns.filter((t) => t?.role === 'assistant' && (t.content || '').trim()).length
: 0,
promoted: !!conv.promoted,
});
}
}
return summaries;
}
export async function createConversation({ mode = 'ask', title = '' } = {}) {
if (!VALID_MODES.has(mode)) throw new Error(`Invalid mode: ${mode}`);
await ensureDir(ASK_DIR);
const now = new Date().toISOString();
const conv = {
id: generateId(),
title: truncateTitle(title) || '(new conversation)',
mode,
createdAt: now,
updatedAt: now,
promoted: false,
turns: [],
};
await atomicWrite(pathFor(conv.id), conv);
return conv;
}
export async function appendTurn(conversationId, turn) {
const conv = await readConversation(conversationId);
if (!conv) throw new Error(`Conversation not found: ${conversationId}`);
if (!turn || (turn.role !== 'user' && turn.role !== 'assistant')) {
throw new Error('Turn must include role of "user" or "assistant"');
}
// Storage owns the persistence schema, so an invalid `turn.mode` shouldn't
// be written to disk just because a future caller bypassed the route's
// zod validation. Mirror the gate `createConversation` already enforces.
if (turn.mode != null && !VALID_MODES.has(turn.mode)) {
throw new Error(`Invalid mode: ${turn.mode}`);
}
const stamped = {
id: turn.id || randomUUID(),
role: turn.role,
content: typeof turn.content === 'string' ? turn.content : '',
createdAt: turn.createdAt || new Date().toISOString(),
...(turn.sources ? { sources: turn.sources } : {}),
...(turn.mode ? { mode: turn.mode } : {}),
...(turn.providerId ? { providerId: turn.providerId } : {}),
...(turn.model ? { model: turn.model } : {}),
};
conv.turns = [...(conv.turns || []), stamped];
// First user turn becomes the conversation title — give the listing a
// useful label without a separate naming step. Title from `stamped.content`
// (the normalised, persisted value) so a non-string `turn.content` can't
// produce a title that disagrees with the stored turn body.
if (turn.role === 'user' && (!conv.title || conv.title === '(new conversation)')) {
conv.title = truncateTitle(stamped.content);
}
conv.updatedAt = stamped.createdAt;
await atomicWrite(pathFor(conversationId), conv);
return { conversation: conv, turn: stamped };
}
export async function deleteConversation(id) {
if (!isValidId(id)) return false;
const removed = await unlinkGuarded(pathFor(id)).then(() => true, (err) => {
if (err.code === 'ENOENT') return false;
throw err;
});
return removed;
}
export async function setPromoted(id, promoted) {
if (!isValidId(id)) return null;
const conv = await readConversation(id);
if (!conv) return null;
conv.promoted = !!promoted;
conv.updatedAt = new Date().toISOString();
await atomicWrite(pathFor(id), conv);
return conv;
}
// Test helper — exposed so tests can introspect/clean state without
// duplicating path math. Not part of the public API.
export const __test = { pathFor, generateId, ASK_DIR };