Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

20.1 KB
Raw Blame
/**
 * Observation-first discovery for the Eidoverse (#7457, epic #7453).
 *
 * SwarmWorld's finding is that reuse between agents starts with LOOKING at the
 * world, not with asking its author about it. PortOS's mind tools grew the
 * other way round: a mind could speak (`eidoverse.say`), travel
 * (`eidoverse.visit`), and chat (`eidoverse.chat`, `eidoverse.visit-chat`)
 * long before it could see what was already standing in its own world. The
 * playbook has told it to "move through the world and map what already exists"
 * since the continuous-play loop landed, naming places, projections, and
 * affordances it had no tool to read. This module is that missing read.
 *
 * It is deliberately PURE and takes already-collected snapshots. The service
 * beside it (`services/eidoverseObservationLedger.js`) owns the I/O, the
 * clock, and the visit marker; everything here is a total function of its
 * arguments, so the whole diff contract is testable without a world runtime,
 * a socket, or a file.
 *
 * **Absent is never empty, here or in the trail.** A section that failed to
 * collect arrives as `null` rather than `[]`, and an unavailable section
 * contributes no changes and carries the previous marker's ids forward. A
 * transient peer-read failure that presented as "no peers" would otherwise
 * report every peer departed, rewrite the marker without them, and then report
 * them all as new on the next observation.
 *
 * **Three things it must not do.** It must not require a running world — a
 * mind that cannot start the runtime still needs to know what is in its
 * Commons, so occupancy is derived from the live PortOS signals a projection
 * would place rather than from a live entity dump. It must not invent its own
 * idea of the world's shape: places come from the install's RESOLVED design
 * recipe, so a mind never names a district the world renamed (V3 calls it the
 * Federation Terminal, not the V2 Harbor) or one the user's overrides moved.
 * And it must not widen what a mind can see: every identity that reaches the
 * report is already one the existing mind tools return
 * (`summarizeFoundation`'s opaque `originInstanceId`, `eidoversePeerId`'s
 * digest), and nothing here reads a record body.
 */

import { asArray } from './arrayUtils.js';
import {
  EIDOVERSE_SCALAR_SOURCE_KEYS,
  EIDOVERSE_SOURCE_KEYS,
  EIDOVERSE_WORLD_DESIGN_V3,
} from './eidoverseWorldDesign.js';
import { EIDOVERSE_ATTENTION_REASON_CODES, eidoversePeerId } from './eidoverseWorldSignals.js';

/** Storage-layout version stamped on the persisted visit marker. */
export const EIDOVERSE_OBSERVATION_SCHEMA_VERSION = 1;

// Bounds. An observation is a tool result read by a language model, so every
// list is capped rather than trusted to stay small: a busy install has
// hundreds of foundations and an unbounded report would crowd out the turn it
// is supposed to inform. The caps are per-section so one crowded section
// cannot starve the others.
const MAX_PEERS = 32;
const MAX_INHERITED = 20;
const MAX_ATTENTION_CONTROLLERS = 10;
const MAX_CHANGE_ENTRIES = 20;
const MAX_ATTENTION_REASONS = 24;
const ATTENTION_REASON_CODES = new Set([...EIDOVERSE_ATTENTION_REASON_CODES, 'source_attention', 'source_error']);
// Only `foundationIds` needs this: the other two marker lists are derived from
// report sections that are already capped far below it. An install that
// somehow exceeds it simply stops distinguishing the oldest ids as "seen",
// which degrades to reporting them as new once, not to a broken file.
const MAX_MARKER_FOUNDATION_IDS = 2000;

const sortedUnique = (values) => [...new Set(asArray(values).filter(Boolean))].sort();

/**
 * Read one signal source for a district in a single pass.
 *
 * `count: null` means the source could not be read at all, which is NOT the
 * same as an empty source and must never collapse into `0` — an unreadable app
 * list would otherwise read as "the App Arcade is empty", exactly the
 * misreport #7458 fixed for district density. Which sources are a single
 * object rather than a list is `EIDOVERSE_SCALAR_SOURCE_KEYS`, the same
 * constant the projection's `sourceAvailable()` keys on, so the two cannot
 * drift into disagreeing about what an empty district looks like.
 *
 * A source the install's recipe has switched OFF is reported as disabled
 * rather than counted: the projection places nothing for it, so counting it
 * would show a mind density that does not exist in the world.
 */
function readSource(source, key, includes) {
  if (includes?.[key] !== true) return { count: null, disabled: true, contributors: [] };
  const value = source?.[key];
  const scalar = EIDOVERSE_SCALAR_SOURCE_KEYS.includes(key);
  if (scalar ? value === null || value === undefined : !Array.isArray(value)) {
    return { count: null, disabled: false, contributors: [] };
  }
  const rows = scalar ? [value] : value;
  const contributors = [];
  // Aggregate by source severity, never by record identity. Stable code ordering
  // and a closed vocabulary prevent private error text or row bags escaping.
  for (const severity of ['error', 'attention']) {
    const affectedRows = rows.filter((row) => row?.status === severity);
    if (!affectedRows.length || !EIDOVERSE_SOURCE_KEYS.includes(key)) continue;
    const reasons = new Map();
    for (const row of affectedRows) {
      const valid = asArray(row.attentionReasons).filter((reason) =>
        ATTENTION_REASON_CODES.has(reason?.code));
      const entries = valid.length ? valid : [{ code: `source_${severity}` }];
      for (const reason of entries) {
        const known = Number.isSafeInteger(reason.affectedCount) && reason.affectedCount >= 0;
        const previous = reasons.get(reason.code);
        reasons.set(reason.code, {
          code: reason.code,
          // Unknown counts stay unknown; a row is not necessarily one record.
          affectedCount: known && (!previous || previous.affectedCount !== null)
            ? Math.min(Number.MAX_SAFE_INTEGER, (previous?.affectedCount || 0) + reason.affectedCount) : null,
        });
      }
    }
    contributors.push({ source: key, severity, signalCount: affectedRows.length,
      reasons: [...reasons.values()].sort((left, right) => left.code.localeCompare(right.code)) });
  }
  return { count: rows.length, disabled: false, contributors,
    attention: rows.some((row) => row?.status === 'error' || row?.status === 'attention') };
}

/**
 * One district as something a mind can stand in and describe: what feeds it,
 * how much is currently there, and whether any of it wants attention.
 *
 * `signalCount` is what the district's enabled sources REPORT, not a count of
 * placed entities. The projection allocates a capped, round-robin sample of
 * those signals into the world (`buildProjectionPlan`), so the world may hold
 * fewer. Reporting the source count keeps this readable without a live
 * runtime; the field name and the tool description both say "signals" rather
 * than "entities" so the difference is never implied away.
 */
function observePlace(district, source, includes) {
  let signalCount = null;
  let attention = false;
  const unreadableSources = [];
  const disabledSources = [];
  const contributorsBySource = [];
  let attentionTruncated = false;
  let reasonCount = 0;
  for (const key of district.sources) {
    const { count, disabled, attention: wantsAttention, contributors } = readSource(source, key, includes);
    if (disabled) disabledSources.push(key);
    else if (count === null) unreadableSources.push(key);
    else signalCount = (signalCount ?? 0) + count;
    if (wantsAttention) attention = true;
    contributorsBySource.push(...contributors);
  }
  const attentionSignals = [];
  contributorsBySource.sort((left, right) => left.source.localeCompare(right.source)
    || (left.severity === right.severity ? 0 : left.severity === 'error' ? -1 : 1));
  for (const contributor of contributorsBySource) {
    const reasons = contributor.reasons.slice(0, MAX_ATTENTION_REASONS - reasonCount);
    if (reasons.length < contributor.reasons.length) attentionTruncated = true;
    reasonCount += reasons.length;
    if (reasons.length) attentionSignals.push({ ...contributor, reasons });
  }
  // Priority order, and `quiet` only when every enabled source read cleanly:
  // an unreadable source must never render as an empty district.
  const status = attention ? 'attention'
    : signalCount === null ? 'unknown'
      : signalCount > 0 ? 'active' : 'quiet';
  return {
    id: district.id,
    label: district.label,
    direction: district.direction,
    landmark: district.landmark,
    sources: district.sources,
    unreadableSources,
    disabledSources,
    signalCount,
    status,
    attentionSignals,
    attentionTruncated,
  };
}

/**
 * Inherited foundations, keyed back to the peer chamber they arrived through.
 *
 * `sourceInstanceId` (pulled-FROM) is hashed with `eidoversePeerId` so it lines
 * up with the opaque ids in `peers`; `originInstanceId` (authored-BY) stays as
 * it is because that is exactly what `summarizeFoundation` already hands the
 * mind, and a multi-hop re-share is only legible when the two are distinct.
 */
function observeInherited(inherited) {
  const peerIdByInstance = new Map();
  const peerIdFor = (instanceId) => {
    if (!instanceId) return null;
    if (!peerIdByInstance.has(instanceId)) peerIdByInstance.set(instanceId, eidoversePeerId({ instanceId }));
    return peerIdByInstance.get(instanceId);
  };
  const byPeer = new Map();
  for (const entry of inherited) {
    const peerId = peerIdFor(entry.inheritance?.sourceInstanceId);
    if (peerId) byPeer.set(peerId, (byPeer.get(peerId) || 0) + 1);
  }
  // ISO-8601 timestamps sort correctly as plain strings; `localeCompare` would
  // run full ICU collation over every inherited foundation to keep 20.
  const rows = inherited
    .map((entry) => ({ entry, at: entry.inheritance?.inheritedAt || '' }))
    .sort((left, right) => (left.at < right.at ? 1 : left.at > right.at ? -1 : 0))
    .slice(0, MAX_INHERITED)
    .map(({ entry }) => ({
      id: entry.id,
      kind: entry.kind,
      title: entry.title,
      originInstanceId: entry.inheritance?.originInstanceId ?? null,
      fromPeerId: peerIdFor(entry.inheritance?.sourceInstanceId),
      inheritedAt: entry.inheritance?.inheritedAt ?? null,
    }));
  return { rows, byPeer };
}

/**
 * The federated neighbours, as the world shows them. `peerId` is the same
 * opaque digest `eidoverse.destinations` returns, so a mind can carry one
 * straight into `eidoverse.visit` — and so nothing here is a hostname.
 * `travelAvailable` is read from the signal rather than re-probed: the source
 * collector already resolved it against the live destination list.
 *
 * `inheritedFoundations` is the observation-first payoff — it links a peer to
 * the foundations this install actually pulled from it, which is how a peer's
 * contribution becomes discoverable by touring the Commons instead of by
 * reading a repository.
 */
function observePeers(source, byPeer) {
  if (!Array.isArray(source?.peers)) return null;
  return source.peers.slice(0, MAX_PEERS).map((peer) => ({
    peerId: peer.id,
    status: peer.status,
    enabled: peer.enabled === true,
    fullSync: peer.fullSync === true,
    travelAvailable: peer.travelAvailable === true,
    inheritedFoundations: byPeer.get(peer.id) || 0,
  }));
}

/**
 * Controller health, filtered to what a maintenance wake would act on.
 *
 * Takes the SUMMARY projection (`summarizeControllerInstall`), never a raw
 * ledger record: the raw record carries `lastOutcome: { ok, reason }`, and
 * reading `lastTickOk` off one silently yields `undefined` for every install,
 * which would quietly disable the failed-tick clause below.
 *
 * `lastTickOk === null` means the controller has never stepped yet, which is
 * the normal state one interval after an install and deliberately NOT an
 * alarm. `lastDelivery` is the SEPARATE verdict #7628 added: a step can read
 * `ok: true` while every effect it produced was refused by the world, so a
 * clean step alone does not clear this filter — `lastDelivery.ok === false`
 * is checked too. `lastDelivery.ok === null` (delivery off, or nothing to
 * deliver this tick) is deliberately NOT an alarm either.
 */
function observeControllers(installs) {
  if (!Array.isArray(installs)) return null;
  return installs
    .filter((install) => install?.disarmedReason
      || (install?.consecutiveFailures ?? 0) > 0
      || install?.lastTickOk === false
      || install?.lastDelivery?.ok === false)
    .slice(0, MAX_ATTENTION_CONTROLLERS)
    .map((install) => ({
      id: install.id,
      controllerId: install.controllerId,
      armed: install.armed === true,
      lastTickOk: install.lastTickOk ?? null,
      lastTickReason: install.lastTickReason ?? null,
      consecutiveFailures: install.consecutiveFailures ?? 0,
      lastDelivery: install.lastDelivery ?? null,
      disarmedReason: install.disarmedReason ?? null,
    }));
}

const NO_CHANGES = Object.freeze({
  newFoundations: [],
  newPeers: [],
  departedPeers: [],
  placesChanged: [],
  controllersNeedingAttention: [],
});

/**
 * What changed since the marker this mind left last time.
 *
 * A first observation reports `firstObservation: true` and NO new items rather
 * than declaring the entire world new — a mind waking into an install that has
 * been running for months should be told it has not looked before, not handed
 * two hundred "new" foundations. That distinction is the absent-versus-empty
 * rule from AGENTS.md applied to the marker itself.
 *
 * Places report a change only when their STATUS flips, never on a count
 * change: signal counts move every wake as tasks and agents come and go, so
 * diffing them would make `changes` noise instead of signal. The live count
 * stays in `places` for a mind that wants it.
 */
function diffAgainstMarker({ marker, foundations, peers, controllers, placeStatus }) {
  if (!marker) return { firstObservation: true, since: null, ...NO_CHANGES };
  const seenFoundations = new Set(asArray(marker.foundationIds));
  const seenPeers = new Set(asArray(marker.peerIds));
  const seenAttention = new Set(asArray(marker.attentionControllerIds));
  const previousPlaceStatus = marker.placeStatus && typeof marker.placeStatus === 'object' ? marker.placeStatus : {};
  const currentPeers = new Set(peers.ids);
  return {
    firstObservation: false,
    since: marker.observedAt ?? null,
    newFoundations: foundations.available
      ? foundations.ids.filter((id) => !seenFoundations.has(id)).slice(0, MAX_CHANGE_ENTRIES)
      : [],
    newPeers: peers.available
      ? peers.ids.filter((id) => !seenPeers.has(id)).slice(0, MAX_CHANGE_ENTRIES)
      : [],
    departedPeers: peers.available
      ? [...seenPeers].filter((id) => !currentPeers.has(id)).sort().slice(0, MAX_CHANGE_ENTRIES)
      : [],
    placesChanged: Object.entries(placeStatus)
      .filter(([id, status]) => previousPlaceStatus[id] !== undefined && previousPlaceStatus[id] !== status)
      .map(([id, status]) => ({ id, was: previousPlaceStatus[id], now: status }))
      .slice(0, MAX_CHANGE_ENTRIES),
    // Only controllers that were NOT already wanting attention last time, so a
    // long-broken controller stops re-alarming every single wake.
    controllersNeedingAttention: controllers.available
      ? controllers.ids.filter((id) => !seenAttention.has(id)).slice(0, MAX_CHANGE_ENTRIES)
      : [],
  };
}

/**
 * An id list for one section, plus whether the section could be read at all.
 *
 * A section that FAILED to collect must not present as empty. Treating it as
 * empty would report every peer as departed and every foundation as gone on a
 * transient read failure, and — because the marker is rewritten from the same
 * lists — would then report all of them as NEW again on the next observation.
 * So an unavailable section contributes no changes and CARRIES FORWARD what
 * the previous marker held, which is the absent-versus-empty rule applied to
 * the trail itself.
 */
function section(ids, available, carriedForward) {
  return available ? { ids, available: true } : { ids: asArray(carriedForward), available: false };
}

const GUIDANCE = 'Observation before conversation: what you see here is this install\'s own world. '
  + 'Read `places` and `changes` first, act on what is already standing (eidoverse.foundations, '
  + 'eidoverse.controllers, eidoverse.augment), and only then reach for chat or a peer visit. '
  + 'A peer\'s inherited foundation is a durable contribution you can build on without asking its author.';

/**
 * Build one observation report plus the marker to persist for the next one.
 *
 * Every argument is an already-read snapshot: `districts`/`includes` come from
 * the install's RESOLVED design recipe (the service resolves it the same way
 * the projection does), `controllerInstalls` is the summarized projection, and
 * `marker` is the previously persisted marker (`null` on a first observation).
 * Returns `{ report, marker }`: the caller decides whether to commit the new
 * marker, which keeps the diff contract testable without a write.
 */
export function buildEidoverseObservation({
  source = {},
  districts = EIDOVERSE_WORLD_DESIGN_V3.districts,
  includes = EIDOVERSE_WORLD_DESIGN_V3.includes,
  foundations = null,
  foundationCounts = null,
  controllerInstalls = null,
  controllerCounts = null,
  marker = null,
  observedAt,
} = {}) {
  const places = districts.map((district) => observePlace(district, source, includes));
  const inherited = asArray(foundations).filter((entry) => entry?.inheritance);
  const { rows: inheritedRows, byPeer } = observeInherited(inherited);
  const peers = observePeers(source, byPeer);
  const attentionControllers = observeControllers(controllerInstalls);

  // `null` means the section could not be read; `[]` means it read as empty.
  // Collapsing the two here is what would make a transient failure flap — see
  // `section()`.
  const foundationSection = section(
    sortedUnique(asArray(foundations).map((entry) => entry?.id)),
    Array.isArray(foundations),
    marker?.foundationIds,
  );
  const peerSection = section(
    sortedUnique(asArray(peers).map((entry) => entry.peerId)),
    peers !== null,
    marker?.peerIds,
  );
  const controllerSection = section(
    sortedUnique(asArray(attentionControllers).map((entry) => entry.id)),
    attentionControllers !== null,
    marker?.attentionControllerIds,
  );
  // `signalCount === null` alone is not a failed read — a district whose
  // sources are all disabled by the recipe also lands there, legitimately.
  // Only a district that actually FAILED a source read (`unreadableSources`
  // non-empty) gets the section()-equivalent carry-forward treatment: the
  // report keeps `status: 'unknown'` (built into `places` above) so the mind
  // still sees the read failed, but the committed marker — and therefore the
  // diff below — keeps the PREVIOUS marker's status for that district
  // instead of overwriting it with `unknown`. Same rule `section()` applies
  // to the three id-list sections, applied per key because this section is a
  // map rather than a list.
  const previousPlaceStatus = marker?.placeStatus && typeof marker.placeStatus === 'object' ? marker.placeStatus : {};
  const placeStatus = Object.fromEntries(places.map((place) => {
    const unreadable = place.signalCount === null && place.unreadableSources.length > 0;
    const carried = previousPlaceStatus[place.id];
    const status = unreadable && carried !== undefined ? carried : place.status;
    return [place.id, status];
  }));

  return {
    report: {
      schemaVersion: EIDOVERSE_OBSERVATION_SCHEMA_VERSION,
      observedAt,
      places,
      peers,
      foundations: {
        counts: foundationCounts,
        inherited: inheritedRows,
        inheritedTruncated: inherited.length > inheritedRows.length,
      },
      controllers: {
        counts: controllerCounts,
        needsAttention: attentionControllers,
      },
      changes: diffAgainstMarker({
        marker,
        foundations: foundationSection,
        peers: peerSection,
        controllers: controllerSection,
        placeStatus,
      }),
      guidance: GUIDANCE,
    },
    marker: {
      schemaVersion: EIDOVERSE_OBSERVATION_SCHEMA_VERSION,
      observedAt,
      foundationIds: foundationSection.ids.slice(0, MAX_MARKER_FOUNDATION_IDS),
      peerIds: peerSection.ids,
      placeStatus,
      attentionControllerIds: controllerSection.ids,
    },
  };
}