Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
/**
* Per-record catalog payload schemaVersion migration.
*
* Every `catalog_ingredients` row carries `payload.schemaVersion` (stamped at
* create time by `catalogDB.createIngredient`). When a TYPE's payload shape
* evolves, the registry (`server/lib/catalogTypes.js`) bumps that type's
* `payloadSchemaVersion` and registers `payloadUpgraders` keyed by FROM
* version. This script walks every row whose stored `payload.schemaVersion`
* is below the registry-current for its type and applies the upgrader chain,
* persisting the upgraded payload (which re-stamps `schemaVersion` to current).
*
* Distinct from:
* - the cross-instance WIRE contract (`PORTOS_SCHEMA_VERSIONS.catalog`),
* which gates whether a peer push is even accepted; and
* - the storage-LAYOUT version (Postgres tables) handled by `ensureSchema`.
* This is the per-record payload-SHAPE version, mirroring the
* `data/{type}/index.json` `schemaVersion` convention documented in AGENTS.md.
*
* Idempotency: a marker in `data/catalog-payload.applied.json` records the
* highest registry payload version this install has fully migrated to, so the
* walk only runs when a new code version actually raised a type's
* `payloadSchemaVersion`. The per-row predicate (`stored < current`) is itself
* idempotent, so re-running is always safe.
*
* Invoked from server/index.js at boot, after `ensureSchema()` (it needs the
* tables) and alongside `migrateBibleToCatalog` — both are DB walks that can't
* ride the pre-DB `scripts/migrations/` file runner.
*/
import { readMarker, writeMarker } from '../lib/migrationMarker.js';
import { query } from '../lib/db.js';
import { CATALOG_TYPES, currentPayloadSchemaVersion, upgradePayload } from '../lib/catalogTypes.js';
import { toPgJson } from '../lib/pgJson.js';
const MARKER_FILENAME = 'catalog-payload.applied.json';
// The aggregate "fully-migrated-to" version is the max payloadSchemaVersion
// across all types. When code bumps any type past this, the marker is stale
// and the walk runs. (A per-type marker would be finer-grained but the walk's
// row predicate is already per-type, so a single high-water mark is enough to
// skip the no-op boot scan on an already-migrated install.)
const targetHighWater = () => Math.max(...CATALOG_TYPES.map((t) => t.payloadSchemaVersion));
/**
* Walk one type's below-current rows and apply the upgrader chain. Returns the
* number of rows upgraded. Rows whose stored `schemaVersion` is absent are
* treated as v1 (the original shape) by `upgradePayload`.
*
* Only types that declare at least one upgrader are walked — a type that
* bumped its version without registering an upgrader has nothing to apply, and
* `upgradePayload` would only re-stamp the marker (harmless but pointless work
* across potentially many rows), so we skip it.
*/
async function migrateType(type) {
const current = currentPayloadSchemaVersion(type.id);
if (current <= 1) return 0;
if (!type.payloadUpgraders || Object.keys(type.payloadUpgraders).length === 0) return 0;
// `payload->>'schemaVersion'` is text; a missing key yields NULL → treated as
// below-current. `COALESCE(...,'1')::int` lets the comparison run in SQL.
const result = await query(
`SELECT id, payload FROM catalog_ingredients
WHERE type = $1
AND deleted = false
AND COALESCE((payload->>'schemaVersion')::int, 1) < $2`,
[type.id, current],
);
let upgraded = 0;
for (const row of result.rows) {
const next = upgradePayload(type.id, row.payload || {});
await query(
`UPDATE catalog_ingredients SET payload = $2::jsonb WHERE id = $1`,
[row.id, toPgJson(next)],
);
upgraded++;
}
return upgraded;
}
/**
* Public entry point. No-ops when the marker shows this install is already at
* (or above) the current high-water version, unless `force` is set.
*/
export async function migrateCatalogPayload({ force = false } = {}) {
const highWater = targetHighWater();
const marker = await readMarker(MARKER_FILENAME);
if (!force && marker?.highWater >= highWater) {
return { skipped: true, marker };
}
console.log(`🧬 catalog payload migration: starting (target v${highWater})`);
const totals = { typesWalked: 0, upgraded: 0 };
for (const type of CATALOG_TYPES) {
const n = await migrateType(type);
if (n > 0) {
totals.typesWalked++;
totals.upgraded += n;
console.log(`🧬 ${type.id}: upgraded ${n} row(s) to v${currentPayloadSchemaVersion(type.id)}`);
}
}
const payload = { highWater, completedAt: new Date().toISOString(), stats: totals };
await writeMarker(MARKER_FILENAME, payload);
console.log(`🧬 catalog payload migration: ${totals.upgraded} row(s) upgraded across ${totals.typesWalked} type(s)`);
return { skipped: false, ...payload };
}