Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
#!/usr/bin/env node
import { existsSync, readdirSync, readFileSync, rmdirSync } from 'fs';
import { dirname, join } from 'path';
import { fileURLToPath } from 'url';
import { md5, buildPromptDriftTables } from './migrations/_lib.js';
import { seedDataDir } from './lib/seedDataDir.js';
import { mergeJsonStarterTargets } from './lib/mergeJsonStarter.js';
import { rewriteAppsPortosRoot } from './lib/rewriteAppsPortosRoot.js';
import { adoptLegacyAppliedList } from './lib/migrationLedger.js';
import { setupSpotlight } from './setup-spotlight.js';
import { setupTimeMachine } from './setup-timemachine.js';
const __dirname = dirname(fileURLToPath(import.meta.url));
const rootDir = join(__dirname, '..');
const dataDir = join(rootDir, 'data');
const referenceDir = join(rootDir, 'data.reference');
console.log('📁 Setting up data directory...');
// One-shot cleanup for installs that predate the migrations-out-of-data move:
// (1) relocate data/migrations/.applied.json → data/migrations.applied.json
// (the migration runner adopts it too, for a boot that skips setup-data);
// (2) drop the now-orphan data/migrations/ directory if empty. Guarded so we
// never clobber a fresh-layout applied file and never blow away a non-empty
// dir (in case a user has uncommitted local migration files).
const legacyMigrationsDir = join(dataDir, 'migrations');
if (await adoptLegacyAppliedList(rootDir)) {
console.log('🧹 Moved data/migrations/.applied.json → data/migrations.applied.json');
}
if (existsSync(legacyMigrationsDir) && readdirSync(legacyMigrationsDir).length === 0) {
rmdirSync(legacyMigrationsDir);
console.log('🧹 Removed orphan data/migrations/ directory');
}
/**
* Expand `__PORTOS_ROOT__` in data/apps.json to this checkout's absolute path.
* Idempotent: no-op when the file is missing or already expanded. Runs for both
* fresh seeds and existing data/ trees (partial copies / restored volumes can
* leave the literal token in place — that breaks Apps → Git checkout detection).
*/
const rewritePortosRootPlaceholder = () => {
const { rewritten, token, rootDir: expandedRoot } = rewriteAppsPortosRoot(dataDir, rootDir);
if (rewritten) {
console.log(`📍 Expanded ${token} in apps.json → ${expandedRoot}`);
}
};
const { created } = seedDataDir({ referenceDir, dataDir, log: console.log });
rewritePortosRootPlaceholder();
setupSpotlight(dataDir);
setupTimeMachine(rootDir);
console.log(created
? '✅ Data directory created'
: '✅ Data directory already exists, ensured subdirectories and files');
// Merge new top-level entries from data.reference's structured JSON files into
// the user's existing copies, leaving customized entries untouched. Without
// this, only file-level "missing → copy" propagation runs above, so a new
// prompt stage or shared variable added to data.reference/prompts/ never
// reaches an existing install whose stage-config.json or variables.json
// already exists. Each merge target points at one top-level dict-shaped key
// (`stages` for stage-config.json, `variables` for variables.json) and the
// merger adds entries the user is missing without overwriting anything they
// have. data/setup-data-seeded.json records which starter keys each target has
// already been offered, so a starter entry the user deleted stays deleted while
// an entry new in a later release still arrives. An install with no ledger entry
// for a target gets every missing key once (bootstrap), then the ledger is written.
const JSON_MERGE_TARGETS = [
{ relPath: 'prompts/stage-config.json', mergeKey: 'stages' },
{ relPath: 'prompts/variables.json', mergeKey: 'variables' },
{ relPath: 'providers.json', mergeKey: 'providers' },
];
mergeJsonStarterTargets({
targets: JSON_MERGE_TARGETS,
referenceDir,
dataDir,
ledgerPath: join(dataDir, 'setup-data-seeded.json'),
});
// Drift detection — warn when a data.reference/prompts/{stages,_partials}/*.md
// differs from the installed copy. Only fires on existing installs (fresh
// installs already got a full copy above). Prompt templates drift when a PortOS
// update adds new template variables (e.g. {{lengthTargets.*}}) that existing
// installs won't pick up because setup-data.js only copies *missing* files.
//
// The drift tables are NOT hand-mirrored here — they are swept from each
// migration's exported `ACCEPTED_OLD_MD5` / `NEW_SHIPPED_MD5` (+ optional
// `DRIFT_SUBDIRS`) constants, so the migration that actually performs the
// auto-update is the single source of truth. `buildPromptDriftTables` merges
// each file's lineage (union of accepted-old + intermediate-new hashes; the
// highest-numbered migration's new hash is the current shipped baseline) and
// returns per-subdir `{ oldMap, newMap, files }` tables. Only the files
// managed by a migration are checked, so prompts with no migration counterpart
// (e.g. cd-evaluate.md) never produce misleading warnings.
const driftTables = await buildPromptDriftTables(join(__dirname, 'migrations'));
const emptyDriftTable = () => ({ oldMap: {}, newMap: {}, files: [] });
const stageTable = driftTables.stages || emptyDriftTable();
const partialTable = driftTables._partials || emptyDriftTable();
// Walk one directory's worth of shipped prompt files against a hash table
// and partition them into auto-updatable (still on a known old hash) vs.
// customized (hash matches neither old nor new). Used twice — once for
// stage prompts, once for partial fragments.
const collectDrift = ({ sampleSubdir, dataSubdir, files, oldMap, newMap }) => {
const sampleSubpath = join(referenceDir, ...sampleSubdir);
const dataSubpath = join(dataDir, ...dataSubdir);
if (!existsSync(sampleSubpath) || !existsSync(dataSubpath)) {
return { autoUpdatable: [], customized: [] };
}
const autoUpdatable = [];
const customized = [];
for (const f of files) {
const dataPath = join(dataSubpath, f);
// No data.reference twin → the prompt was renamed/retired upstream (e.g.
// pipeline-tv-script.md → pipeline-teleplay.md); there's nothing to
// auto-update *to*, so skip rather than warn about a phantom drift.
if (!existsSync(join(sampleSubpath, f))) continue;
if (!existsSync(dataPath)) continue;
const dataMd5 = md5(readFileSync(dataPath, 'utf8'));
if (dataMd5 === newMap[f]) continue;
const oldExpected = oldMap[f];
const oldHashes = Array.isArray(oldExpected) ? oldExpected : [oldExpected];
if (oldHashes.includes(dataMd5)) {
autoUpdatable.push(f);
} else {
customized.push(f);
}
}
return { autoUpdatable, customized };
};
const stageDrift = collectDrift({
sampleSubdir: ['prompts', 'stages'],
dataSubdir: ['prompts', 'stages'],
files: stageTable.files,
oldMap: stageTable.oldMap,
newMap: stageTable.newMap,
});
const partialDrift = collectDrift({
sampleSubdir: ['prompts', '_partials'],
dataSubdir: ['prompts', '_partials'],
files: partialTable.files,
oldMap: partialTable.oldMap,
newMap: partialTable.newMap,
});
const autoUpdatable = [
...stageDrift.autoUpdatable.map((f) => ({ file: f, relDir: 'data/prompts/stages' })),
...partialDrift.autoUpdatable.map((f) => ({ file: f, relDir: 'data/prompts/_partials' })),
];
const customized = [
...stageDrift.customized.map((f) => ({ file: f, relDir: 'data/prompts/stages', referenceDir: 'data.reference/prompts/stages' })),
...partialDrift.customized.map((f) => ({ file: f, relDir: 'data/prompts/_partials', referenceDir: 'data.reference/prompts/_partials' })),
];
if (autoUpdatable.length > 0) {
console.warn(
`\n⚠️ ${autoUpdatable.length} pipeline prompt(s) have a pending migration — run \`npm run migrations\` to auto-update:\n` +
autoUpdatable.map(({ file, relDir }) => ` • ${relDir}/${file}`).join('\n') + '\n',
);
}
if (customized.length > 0) {
console.warn(
`\n⚠️ ${customized.length} pipeline prompt(s) are customized and cannot be auto-updated.\n` +
` Manually merge the new template variables into each file:\n` +
customized.map(({ file, relDir, referenceDir: sd }) =>
` • ${relDir}/${file}\n` +
` Compare with: ${sd}/${file}`,
).join('\n') + '\n',
);
}