Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
import { PATHS } from '../lib/fileUtils.js';
import { createCollectionStore } from '../lib/collectionStore.js';
import { ServerError } from '../lib/errorHandler.js';
import { htmlToText } from '../lib/htmlToText.js';
import { fetchPublicText } from '../lib/safeUrlFetch.js';
import { countWords } from '../lib/textUtils.js';
import { v4 as uuidv4 } from '../lib/uuid.js';
export const RAPID_READER_LIBRARY_SCHEMA_VERSION = 1;
export const MAX_RAPID_READER_TEXT_BYTES = 2 * 1024 * 1024;
const MAX_TITLE_LENGTH = 200;
const now = () => new Date().toISOString();
const textBytes = (text) => Buffer.byteLength(text, 'utf8');
const normalizeText = (text) => typeof text === 'string' ? text.trim() : '';
const normalizeTitle = (title) => typeof title === 'string' ? title.trim().slice(0, MAX_TITLE_LENGTH) : '';
export const rapidReaderLibraryStore = createCollectionStore({
dir: PATHS.rapidReaderLibrary,
type: 'rapid-reader-library',
schemaVersion: RAPID_READER_LIBRARY_SCHEMA_VERSION,
sanitizeRecord: (record) => {
const text = normalizeText(record?.text);
const title = normalizeTitle(record?.title);
if (!record?.id || !text || !title || !['paste', 'fetch', 'accelerando'].includes(record.sourceType)) return null;
return { ...record, title, text, author: typeof record.author === 'string' ? record.author : null, sourceUrl: typeof record.sourceUrl === 'string' ? record.sourceUrl : null, wordCount: countWords(text) };
},
});
function assertText(text) {
if (!text) throw new ServerError('Text is required', { status: 400, code: 'VALIDATION_ERROR' });
if (textBytes(text) > MAX_RAPID_READER_TEXT_BYTES) throw new ServerError('Text exceeds the 2 MiB limit', { status: 400, code: 'VALIDATION_ERROR' });
}
function metadata(record) { const { text, ...entry } = record; return entry; }
function titleFromHtml(source) {
const match = /<title\b[^>]*>([\s\S]*?)<\/title>/i.exec(source);
return match ? htmlToText(match[1]).trim() : '';
}
function hostnameFor(url) { return new URL(url).hostname; }
export async function listRapidReaderLibrary() { return (await rapidReaderLibraryStore.loadAll()).map(metadata).sort((a, b) => b.updatedAt.localeCompare(a.updatedAt)); }
export async function getRapidReaderLibraryEntry(id) {
const entry = await rapidReaderLibraryStore.loadOne(id);
if (!entry) throw new ServerError('Shelf entry not found', { status: 404, code: 'NOT_FOUND' });
return entry;
}
// saveOne only writes {dir}/{id}/index.json — it never touches the type-level
// index, so a store whose records were all written by saveOne would report
// "no index.json (fresh install)" to the boot verifier forever. Stamp it once
// per process on the first write (same guard as creativeCommissions).
let typeIndexStamped = false;
async function ensureTypeIndex() {
if (typeIndexStamped) return;
typeIndexStamped = true;
await rapidReaderLibraryStore.saveTypeIndex({}).catch(() => { typeIndexStamped = false; });
}
async function saveEntry(entry) { await rapidReaderLibraryStore.saveOne(entry.id, entry); await ensureTypeIndex(); return entry; }
export async function createPastedRapidReaderEntry({ title, author, text }) {
const cleanText = normalizeText(text); const cleanTitle = normalizeTitle(title); assertText(cleanText);
if (!cleanTitle) throw new ServerError('Title is required', { status: 400, code: 'VALIDATION_ERROR' });
const timestamp = now();
return saveEntry({ id: uuidv4(), title: cleanTitle, author: normalizeTitle(author) || null, sourceUrl: null, sourceType: 'paste', text: cleanText, wordCount: countWords(cleanText), addedAt: timestamp, updatedAt: timestamp });
}
export async function fetchRapidReaderEntry({ url, title }) {
const source = await fetchPublicText(url, { blockPrivate: true, maxBytes: MAX_RAPID_READER_TEXT_BYTES, timeoutMs: 20_000 });
if (!source) throw new ServerError('Could not fetch the requested URL', { status: 502, code: 'FETCH_FAILED' });
const text = normalizeText(htmlToText(source)); assertText(text);
const timestamp = now(); const derivedTitle = normalizeTitle(title) || normalizeTitle(titleFromHtml(source)) || hostnameFor(url);
return saveEntry({ id: uuidv4(), title: derivedTitle, author: null, sourceUrl: url, sourceType: 'fetch', text, wordCount: countWords(text), addedAt: timestamp, updatedAt: timestamp });
}
export async function deleteRapidReaderLibraryEntry(id) { await getRapidReaderLibraryEntry(id); await rapidReaderLibraryStore.deleteOne(id); }
export async function upsertAccelerandoShelfEntry(book) {
const current = await rapidReaderLibraryStore.loadOne('accelerando'); const timestamp = now(); const text = normalizeText(book.text); assertText(text);
return saveEntry({ id: 'accelerando', title: book.title, author: book.author, sourceUrl: book.sourceUrl, sourceType: 'accelerando', text, wordCount: countWords(text), addedAt: current?.addedAt || timestamp, updatedAt: timestamp });
}