Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

9.5 KB
Raw Blame
/** CDP accounting for the synthetic audit. Never retains URLs, headers or bodies. */
export function endpointFor(rawUrl) {
  const { pathname, protocol } = new URL(rawUrl);
  if (!['http:', 'https:', 'ws:', 'wss:'].includes(protocol)) return '/non-http';
  if (pathname.startsWith('/assets/')) return '/assets/:asset';
  if (pathname.startsWith('/data/')) return '/data/:asset';
  if (/^\/api\/video-gen\/history\/[^/]+$/.test(pathname)) return '/api/video-gen/history/:record';
  if (/^\/api\/messages\/[^/]+\/[^/]+$/.test(pathname)) return '/api/messages/:account/:message';
  // Collection endpoints and bootstrap paths contain no record identifiers;
  // arbitrary --route traffic may, so opaque numeric/hex segments collapse too.
  return pathname.replace(/\/(?:\d+|[a-f0-9]{16,})(?=\/|$)/gi, '/:id').replace(/[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}/gi, ':account')
    .replace(/synthetic-[^/]+/g, ':record');
}

const empty = () => ({ requests: 0, completed: 0, failed: 0, canceled: 0,
  duplicates: 0, overlapping: 0, httpErrors: 0,
  encodedBytes: 0, decodedBytes: 0, unavailable: 0, socketSentFrames: 0,
  socketReceivedFrames: 0, socketSentBytes: 0, socketReceivedBytes: 0 });

/**
 * `label` maps a raw URL (and method) to the endpoint pattern a report may
 * hold; the live audit passes a source-route redactor. Exact URLs are kept in
 * memory only, to count repeats within a phase (`duplicates`) and repeats
 * started while an identical request was still in flight (`overlapping`).
 */
export function createTrafficAccumulator({ label = endpointFor } = {}) {
  const phases = new Map();
  const requests = new Map();
  const sockets = new Map();
  const seen = new Map();
  const inFlight = new Map();
  let phase = 'cold';
  const bucket = (endpoint, target = phase) => {
    if (!phases.has(target)) phases.set(target, new Map());
    const entries = phases.get(target);
    if (!entries.has(endpoint)) entries.set(endpoint, empty());
    return entries.get(endpoint);
  };
  const finish = (id, encoded, failed, canceled) => {
    const request = requests.get(id);
    if (!request) return;
    const row = bucket(request.endpoint);
    // dataReceived encoded lengths are provisional. loadingFinished is the
    // authoritative total (including headers); add only the uncounted remainder.
    row.encodedBytes += Math.max(0, (encoded || 0) - request.encoded);
    row[failed ? 'failed' : 'completed']++;
    if (canceled) row.canceled++;
    const open = (inFlight.get(request.key) || 1) - 1;
    if (open) inFlight.set(request.key, open);
    else inFlight.delete(request.key);
    requests.delete(id);
  };
  return {
    setPhase(name) { phase = name; },
    event(name, event) {
      const id = event.requestId;
      if (name === 'requestWillBeSent') {
        if (event.redirectResponse) finish(id, event.redirectResponse.encodedDataLength, false, false);
        const method = event.request.method || 'GET';
        const endpoint = label(event.request.url, method);
        const key = method + ' ' + event.request.url;
        const row = bucket(endpoint);
        row.requests++;
        if (!seen.has(phase)) seen.set(phase, new Set());
        if (seen.get(phase).has(key)) row.duplicates++;
        else seen.get(phase).add(key);
        if (inFlight.has(key)) row.overlapping++;
        inFlight.set(key, (inFlight.get(key) || 0) + 1);
        requests.set(id, { endpoint, encoded: 0, key });
      } else if (name === 'responseReceived') {
        const request = requests.get(id);
        if (request) {
          const row = bucket(request.endpoint);
          if (event.response.status === 503) row.unavailable++;
          if (event.response.status >= 400) row.httpErrors++;
          const headers = Math.max(0, event.response.encodedDataLength || 0);
          row.encodedBytes += headers;
          request.encoded += headers;
        }
      } else if (name === 'dataReceived') {
        const request = requests.get(id);
        if (!request) return;
        const encoded = Math.max(0, event.encodedDataLength || 0);
        const row = bucket(request.endpoint);
        row.decodedBytes += Math.max(0, event.dataLength || 0);
        row.encodedBytes += encoded;
        request.encoded += encoded;
      } else if (name === 'loadingFinished') finish(id, event.encodedDataLength, false, false);
      else if (name === 'loadingFailed') finish(id, 0, true, event.canceled);
      else if (name === 'webSocketCreated') sockets.set(id, label(event.url, 'GET'));
      else if (name === 'webSocketClosed') sockets.delete(id);
      else if (name === 'webSocketFrameSent' || name === 'webSocketFrameReceived') {
        const row = bucket(sockets.get(id) || '/socket.io/');
        const direction = name === 'webSocketFrameSent' ? 'Sent' : 'Received';
        const { opcode, payloadData } = event.response;
        row['socket' + direction + 'Frames']++;
        row['socket' + direction + 'Bytes'] += opcode === 1
          ? Buffer.byteLength(payloadData, 'utf8') : Buffer.from(payloadData, 'base64').length;
      }
    },
    snapshot() {
      return Object.fromEntries([...phases].map(([name, endpoints]) => [name,
        Object.fromEntries([...endpoints].sort(([a], [b]) => a.localeCompare(b)).map(([key, value]) => [key, { ...value }]))]));
    },
    pending() { return requests.size; },
    pendingByEndpoint() {
      const counts = new Map();
      for (const { endpoint } of requests.values()) counts.set(endpoint, (counts.get(endpoint) || 0) + 1);
      return Object.fromEntries([...counts].sort(([a], [b]) => a.localeCompare(b)));
    },
  };
}

/** Transient payload inspection returns only counts and contract failures. */
export function inspectCollection(endpoint, query, payload, bytes, cardinalities) {
  const failures = [];
  const fail = reason => failures.push({ endpoint, reason, bytes });
  const media = endpoint === '/api/image-gen/gallery';
  const rows = media ? payload?.items : payload?.messages;
  const pageLimit = media ? 60 : 50;
  const total = media ? cardinalities.images + cardinalities.videos - cardinalities.hiddenImages - cardinalities.hiddenVideos : cardinalities.messages;
  if (!Array.isArray(rows)) { fail('unpaged-response'); return { count: 0, failures }; }
  if (query.get('summary') !== 'true') fail('missing-summary-projection');
  if (rows.length > pageLimit || payload.total !== total) fail('page-limit-or-total');
  if (bytes > pageLimit * 2048) fail('list-byte-budget');
  if (media && payload.hiddenTotal !== cardinalities.hiddenImages + cardinalities.hiddenVideos) fail('hidden-total');
  for (const item of rows) {
    const row = media ? item.data : item;
    if (!row || (media && !['image', 'video'].includes(item.kind))) { fail('invalid-row'); break; }
    if (media && (row.hidden || /synthetic-(image|video)-\d*0(?:\.|$)/.test(row.filename || ''))) { fail('hidden-row'); break; }
    if ((media && (row.prompt?.length > 512 || row.negativePrompt?.length > 512))
        || (!media && ('bodyText' in row || 'bodyHtml' in row || row.evaluation?.reasoning))) {
      fail('eager-detail-fields'); break;
    }
  }
  return { count: rows.length, failures };
}

export function trafficFailures(phases) {
  const failures = [];
  for (const [phase, endpoints] of Object.entries(phases)) {
    for (const [endpoint, row] of Object.entries(endpoints)) {
      if (['sibling', 'idle'].includes(phase) && endpoint === '/api/messages/inbox' && row.requests) {
        failures.push({ endpoint, reason: 'sibling-inbox-fetch', count: row.requests, bytes: row.encodedBytes });
      }
      if (phase === 'cold' && ['/api/messages/:account/:message', '/api/video-gen/history/:record', '/api/image-gen/:record/variants'].includes(endpoint) && row.requests) {
        failures.push({ endpoint, reason: 'eager-detail-request', count: row.requests, bytes: row.encodedBytes });
      }
      if (['/api/video-gen/history', '/api/image-gen/history'].includes(endpoint) && row.requests) {
        failures.push({ endpoint, reason: 'full-history-fetch', count: row.requests, bytes: row.encodedBytes });
      }
    }
  }
  return failures;
}

/** Idle HTTP/socket bytes scaled to one minute from the measured window. */
export function idleBytesPerMinute(idle, idleMs) {
  const entries = Object.values(idle || {});
  const total = key => entries.reduce((sum, value) => sum + value[key], 0);
  const scale = bytes => Math.round(bytes * 60000 / idleMs);
  return { encoded: scale(total('encodedBytes')), decoded: scale(total('decodedBytes')),
    socketPayload: scale(total('socketSentBytes') + total('socketReceivedBytes')) };
}

/**
 * CLI: [output] [--route=/path] [--sibling=/path] [--idle-ms=N]. Routes must be
 * same-origin absolute paths so the audit can never reach another host.
 */
export function parseAuditArgs(argv) {
  const out = { output: undefined, route: null, sibling: '/messages/contacts', idleMs: 60000 };
  const path = (name, value) => {
    if (!/^\/(?!\/)[^\s\\]*$/.test(value || '')) throw new Error('--' + name + ' must be an absolute path such as /');
    return value;
  };
  for (const arg of argv) {
    const [flag, ...rest] = arg.split('=');
    const value = rest.join('=');
    if (flag === '--route') out.route = path('route', value);
    else if (flag === '--sibling') out.sibling = path('sibling', value);
    else if (flag === '--idle-ms') {
      out.idleMs = Number(value);
      if (!Number.isInteger(out.idleMs) || out.idleMs < 1000 || out.idleMs > 300000) throw new Error('--idle-ms must be 1000-300000');
    } else if (!arg.startsWith('--') && out.output === undefined) out.output = arg;
    else throw new Error('Unknown argument: ' + flag);
  }
  return out;
}