Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

10.8 KB
Raw Blame
/** Read-through federation of numeric app audit evidence; never exports prose or relays peers. */
import { PATHS } from '../lib/paths.js';
import { createHash } from 'node:crypto';
import { z } from 'zod';
import { query } from '../lib/db.js';
import { PORTOS_APP_ID } from '../lib/appIdentity.js';
import { AUDIT_TASK_TYPE_LIST, LEGACY_AUDIT_TASK_TYPE_ALIASES, normalizeAuditTaskType } from '../lib/auditCatalog.js';
import { auditQualityReportSchema, AUDIT_FRESHNESS_MS, numericQualityReport } from '../lib/auditQuality.js';
import { PORTOS_SCHEMA_VERSIONS } from '../lib/schemaVersions.js';
import { getOriginInfo } from '../lib/gitRemote.js';
import { peerBaseUrl } from '../lib/peerUrl.js';
import { readBodyCapped } from '../lib/safeUrlFetch.js';
import { peerFetch } from '../lib/peerHttpClient.js';
import { releaseTieId } from './appQualitySnapshotFormat.js';

const MAX_PAYLOAD_BYTES = 4 * 1024 * 1024;
const sharedSummary = 'Numeric assessment shared by a federated instance; evidence remains on the source machine.';
const hash = value => createHash('sha256').update(value).digest('hex');
// A stored v2 (slice-scored) report travels as the v1 numeric shape: the wire
// contract is unchanged, so a peer that predates v2 reads the run score it
// already expects, and the slice text never leaves this machine (#12076).
const numericReportSchema = z.preprocess(value => value && typeof value === 'object'
  ? { ...numericQualityReport(value), summary: sharedSummary } : value, auditQualityReportSchema)
  .transform(({ summary: _summary, ...report }) => report);
const measurementSchema = z.object({
  measurementId: z.string().regex(/^[a-f0-9]{64}$/),
  assessedAt: z.string().datetime(),
  report: numericReportSchema,
}).strict();
// Rows are validated one by one (below), so a peer on a newer catalog that
// sends a category this install does not know costs that row, not the peer.
// Everything this install can parse, sent with each request so a peer on a
// newer catalog withholds categories this one would reject.
const READABLE_CATEGORY_PARAM = encodeURIComponent([
  ...AUDIT_TASK_TYPE_LIST, ...Object.keys(LEGACY_AUDIT_TASK_TYPE_ALIASES),
].join(','));

const payloadSchema = z.object({
  schemaVersion: z.literal(PORTOS_SCHEMA_VERSIONS.appQuality),
  repository: z.string().regex(/^[a-f0-9]{64}$/),
  measurements: z.array(z.unknown()).max(10000),
}).strict();

/**
 * The categories every install parsed before the catalog added the service and
 * data-platform lenses. An install that predates the `categories` request
 * parameter validates the whole payload strictly, so one row in a category it
 * does not know would make it drop EVERY measurement from this install. Such a
 * requester is sent only these. Frozen: a later catalog addition is announced by
 * the requester's own `categories` list, never by editing this one.
 */
export const FEDERATION_LEGACY_CATEGORIES = Object.freeze([
  'security', 'code-quality', 'test-coverage', 'performance', 'accessibility', 'documentation',
  'ui-bugs', 'mobile-responsive', 'error-handling', 'typing', 'console-errors', 'ux', 'data-safety',
  'simplify', 'module-hygiene', 'api-contract', 'ui-lifecycle', 'observability', 'copy',
  'better-complexity', 'better-cognitive-load', 'better-structural-drift', 'better-runtime-safety',
  'better-dependency-freedom', 'better-test-quality',
]);

// PortOS's own app record always carries a repoPath; the bare `{ id: PORTOS_APP_ID }`
// form used for self-directed calls resolves to this install's checkout.
const appRepoPath = app => app.repoPath || (app.id === PORTOS_APP_ID ? PATHS.root : null);

// Match repositories without sharing remote URLs, names, credentials or local paths.
// Independent versions of the same repository intentionally contribute to one score.
async function repositoryKey(deps, app = { id: PORTOS_APP_ID }) {
  const repoPath = appRepoPath(app);
  if (!repoPath) return null;
  const origin = await (deps.getOriginInfo || getOriginInfo)(repoPath);
  return origin.host && origin.fullName ? hash(`${origin.host}/${origin.fullName}`.toLowerCase()) : null;
}

async function eligiblePeers(deps) {
  const { getPeers } = deps.getPeers ? deps : await import('./instances.js');
  const { peerAllowsOutbound } = await import('./sharing/peerSyncShared.js');
  return (await getPeers()).filter(peer => peerAllowsOutbound(peer));
}

export function qualityRecord(row) {
  return { category: normalizeAuditTaskType(row.category), agentId: row.agent_id,
    measurementId: hash(row.agent_id), assessedAt: new Date(row.assessed_at).toISOString(), report: row.report };
}

/** Daily category winners include the freshness lookback required for historical snapshots. */
export async function readQualityRecords(appId, days, now, deps = {}) {
  const start = new Date(now);
  start.setUTCHours(0, 0, 0, 0);
  start.setUTCDate(start.getUTCDate() - days + 1);
  const result = await (deps.query || query)(
    `SELECT DISTINCT ON (category, (assessed_at AT TIME ZONE 'UTC')::date)
       category, agent_id, assessed_at, report FROM app_quality_measurements
     WHERE app_id = $1 AND assessed_at >= $2 AND assessed_at <= $3
     ORDER BY category, (assessed_at AT TIME ZONE 'UTC')::date, assessed_at DESC, agent_id DESC`,
    [appId, new Date(start.getTime() - AUDIT_FRESHNESS_MS), new Date(now)]
  );
  return result.rows.map(qualityRecord);
}

/** New endpoint always enforces peer sharing consent, even without instance-password auth. */
export async function exportPortosQuality(callerId, days, deps = {}, requestedRepository, requesterCategories = FEDERATION_LEGACY_CATEGORIES) {
  const peers = await eligiblePeers(deps);
  if (!callerId || !peers.some(peer => peer.instanceId === callerId)) return null;
  let app = { id: PORTOS_APP_ID };
  if (requestedRepository) {
    const { getAllApps } = deps.getAllApps ? deps : await import('./apps.js');
    const candidates = await Promise.all((await getAllApps()).map(async candidate => ({
      app: candidate, repository: await repositoryKey(deps, candidate),
    })));
    app = candidates.find(candidate => candidate.repository === requestedRepository)?.app;
    if (!app) return null;
  }
  const snapshot = await buildQualitySnapshot(app, days, deps);
  if (!snapshot) return snapshot;
  const parseable = new Set(requesterCategories);
  return { ...snapshot, measurements: snapshot.measurements.filter(row => parseable.has(row.report.category)) };
}

/** Sanitized local evidence, also used by the explicit release snapshot command. */
export async function buildQualitySnapshot(app = { id: PORTOS_APP_ID }, days = 30, deps = {}) {
  const repository = await repositoryKey(deps, app);
  if (!repository) return null;
  const records = await readQualityRecords(app.id, days, deps.now ?? Date.now(), deps);
  const measurements = records.flatMap(record => {
    const parsed = measurementSchema.safeParse({ measurementId: record.measurementId,
      assessedAt: record.assessedAt, report: record.report });
    return parsed.success && parsed.data.report.category === record.category ? [parsed.data] : [];
  });
  return { schemaVersion: PORTOS_SCHEMA_VERSIONS.appQuality, repository, measurements };
}

/** No persistence or forwarding: disabled/offline peers cannot leave a hidden stale contribution. */
export async function collectPortosQuality(days, deps = {}) {
  return collectAppQuality({ id: PORTOS_APP_ID }, days, deps);
}

export async function collectAppQuality(app, days, deps = {}) {
  const peers = await eligiblePeers(deps);
  if (!peers.length) return { records: [], federation: { peers: 0, available: 0, unavailable: 0 } };
  const repository = await repositoryKey(deps, app);
  if (!repository) return { records: [], federation: { peers: 0, available: 0, unavailable: 0 } };
  const now = deps.now ?? Date.now();
  const results = await Promise.allSettled(peers.map(async peer => {
    const response = await (deps.peerFetch || peerFetch)(
      `${peerBaseUrl(peer)}/api/apps/quality-federation?days=${days}${app.id === PORTOS_APP_ID ? '' : `&repository=${repository}`}&categories=${READABLE_CATEGORY_PARAM}`,
      { signal: AbortSignal.timeout(3000), redirect: 'error', maxBytes: MAX_PAYLOAD_BYTES }, peer);
    if (!response.ok) {
      await response.body?.cancel?.();
      throw new Error('Peer quality unavailable');
    }
    const body = await readBodyCapped(response, MAX_PAYLOAD_BYTES);
    if (!body) {
      await response.body?.cancel?.();
      throw new Error('Peer quality payload too large');
    }
    const payload = payloadSchema.parse(JSON.parse(body.toString('utf8')));
    if (payload.repository !== repository) throw new Error('Different repository');
    const measurements = payload.measurements.flatMap(row => {
      const parsed = measurementSchema.safeParse(row);
      return parsed.success ? [parsed.data] : [];
    });
    return measurements.filter(row => Date.parse(row.assessedAt) <= now).map(row => ({
      ...row, category: row.report.category, sourcePeerId: peer.id, sourcePeerName: peer.name || peer.id,
      report: { ...row.report, summary: sharedSummary },
    }));
  }));
  return { records: results.flatMap(result => result.status === 'fulfilled' ? result.value : []),
    federation: { peers: peers.length, available: results.filter(r => r.status === 'fulfilled').length,
      unavailable: results.filter(r => r.status === 'rejected').length } };
}

/**
 * Every app — PortOS's own checkout included — reads `.quality.json` (or the
 * historical `quality-snapshot.json` when the canonical file is absent). v1 and
 * v2 both normalize. The read never writes. Repository match, future dates, and
 * the 4 MiB cap still apply. The peer wire schema is not used here.
 */
async function releasePayload(deps, app) {
  const repoPath = appRepoPath(app);
  if (!repoPath) return null;
  const { readStoredQualitySnapshot } = await import('./appQualitySnapshotFile.js');
  const stored = await readStoredQualitySnapshot(repoPath, deps);
  // 'future-categories': a newer install's snapshot — its known rows still count.
  return ['v1', 'v2', 'future-categories'].includes(stored.status) ? stored : null;
}

/** Shipped evidence is read-only and never re-exported as a local audit. */
export async function readReleaseQuality(deps = {}, app = { id: PORTOS_APP_ID }) {
  const payload = await releasePayload(deps, app);
  if (!payload || payload.repository !== await repositoryKey(deps, app)) return [];
  const now = deps.now ?? Date.now();
  return payload.records.filter(row => Date.parse(row.assessedAt) <= now).map(row => ({
    measurementId: releaseTieId(row),
    assessedAt: row.assessedAt,
    category: row.category,
    sourcePeerName: 'Release snapshot',
    report: {
      version: 1,
      category: row.category,
      score: row.score,
      worstSeverity: row.worstSeverity,
      coverage: row.coverage,
      confidence: row.confidence,
      summary: 'Published release assessment; original assessment date and freshness rules apply.',
      scannedFiles: row.scannedFiles,
      totalFiles: row.totalFiles,
    },
  }));
}