Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

11.5 KB
Raw Blame
/**
 * Share Buckets — registry CRUD.
 *
 * A "bucket" is a local directory the user registers; PortOS reads/writes a
 * stable layout inside it (bucket.json + manifests/ + records/ + assets/),
 * while an external cloud-sync app (Google Drive, Dropbox, iCloud, Syncthing,
 * USB stick…) handles cross-network replication. PortOS is transport-agnostic.
 *
 * The registry itself lives in PortOS's local data/, NOT in the shared bucket —
 * otherwise every peer's settings would overwrite each other.
 */

import { randomUUID } from 'crypto';
import { join, basename } from 'path';
import { access, constants, stat } from 'fs/promises';
import { PATHS, atomicWrite, readJSONFile, ensureDir } from '../../lib/fileUtils.js';
import { SHARING_SCHEMA_VERSION } from './version.js';
import { isStr, trimTo } from '../../lib/textUtils.js';
import { createFileWriteQueue } from '../../lib/fileWriteQueue.js';
import { codedError } from '../../lib/codedError.js';

const REGISTRY_PATH = () => join(PATHS.data, 'sharing', 'buckets.json');

export const BUCKET_MODES = Object.freeze(['auto-merge', 'inbox']);

export const NAME_MAX = 120;
export const PATH_MAX = 2000;
export const DISPLAY_NAME_MAX = 120;
export const BIO_MAX = 2000;

export const ERR_NOT_FOUND = 'SHARING_BUCKET_NOT_FOUND';
export const ERR_VALIDATION = 'SHARING_BUCKET_VALIDATION';
export const ERR_PATH_UNUSABLE = 'SHARING_BUCKET_PATH_UNUSABLE';

// One tail for the whole registry file: every bucket shares it, so a per-id
// queue would still let two snapshots overwrite each other.
const queueRegistryWrite = createFileWriteQueue();


const sanitizeBucket = (raw) => {
  if (!raw || typeof raw !== 'object') return null;
  if (!isStr(raw.id) || !raw.id) return null;
  const name = trimTo(raw.name, NAME_MAX);
  const path = trimTo(raw.path, PATH_MAX);
  if (!name || !path) return null;
  const mode = BUCKET_MODES.includes(raw.mode) ? raw.mode : 'inbox';
  const displayNameOverride = trimTo(raw.displayNameOverride, DISPLAY_NAME_MAX) || null;
  const bioOverride = trimTo(raw.bioOverride, BIO_MAX) || null;
  const createdAt = isStr(raw.createdAt) ? raw.createdAt : new Date().toISOString();
  const updatedAt = isStr(raw.updatedAt) ? raw.updatedAt : createdAt;
  return { id: raw.id, name, path, mode, displayNameOverride, bioOverride, createdAt, updatedAt };
};

async function readRegistry() {
  await ensureDir(join(PATHS.data, 'sharing'));
  const raw = await readJSONFile(REGISTRY_PATH(), { buckets: [] }, { logError: false, strict: true });
  const buckets = Array.isArray(raw.buckets) ? raw.buckets.map(sanitizeBucket).filter(Boolean) : [];
  return { buckets };
}

async function writeRegistry(state) {
  await ensureDir(join(PATHS.data, 'sharing'));
  await atomicWrite(REGISTRY_PATH(), state);
}

/** Validate that the bucket path is a readable+writable directory. */
async function assertPathUsable(path) {
  let st;
  try {
    st = await stat(path);
  } catch (_err) {
    throw codedError(`Bucket path does not exist or is unreadable: ${path}`, ERR_PATH_UNUSABLE);
  }
  if (!st.isDirectory()) {
    throw codedError(`Bucket path is not a directory: ${path}`, ERR_PATH_UNUSABLE);
  }
  try {
    await access(path, constants.R_OK | constants.W_OK);
  } catch (_err) {
    throw codedError(`Bucket path is not writable: ${path}`, ERR_PATH_UNUSABLE);
  }
}

/**
 * Path helpers for the per-bucket content-addressed blob store. v2 manifests
 * (`SHARING_SCHEMA_VERSION >= 2`) reference assets via their SHA-256 hash;
 * blobs and their `.metadata.json` sidecars live alongside each other under
 * `assets/blobs/` so storage-layout changes only touch this file.
 *
 * Hashes from manifests are UNTRUSTED — a hostile peer could ship
 * `hash: '../../../../etc/hosts'` to traverse out of the blob dir on import.
 * `isHexHash` enforces the SHA-256 shape (64 lowercase hex chars) so
 * `bucketBlobPath(...)` can never produce a path outside `assets/blobs/`.
 * The path helpers throw on bad input; the importer also drops bad refs at
 * the manifest boundary so a single broken entry doesn't poison a batch.
 */
export const IMAGE_EXT_RE = /\.(png|jpe?g|webp)$/i;
const HEX_HASH_RE = /^[0-9a-f]{64}$/;
export function isHexHash(v) { return typeof v === 'string' && HEX_HASH_RE.test(v); }
export function bucketBlobsDir(bucketPath) { return join(bucketPath, 'assets', 'blobs'); }
export function bucketBlobPath(bucketPath, hash) {
  if (!isHexHash(hash)) throw codedError(`Invalid asset hash: ${hash}`, ERR_VALIDATION);
  return join(bucketBlobsDir(bucketPath), hash);
}
export function bucketBlobSidecarPath(bucketPath, hash) {
  if (!isHexHash(hash)) throw codedError(`Invalid asset hash: ${hash}`, ERR_VALIDATION);
  return join(bucketBlobsDir(bucketPath), `${hash}.metadata.json`);
}
// Sidecar mapping `<sourcePath>:<mtimeMs>:<size> → <hash>` so the exporter can
// skip `sha256File` + `copyFile` on re-export of unchanged assets.
export function bucketBlobIndexPath(bucketPath) { return join(bucketBlobsDir(bucketPath), '.index.json'); }
export function imageSidecarName(filename) { return filename.replace(IMAGE_EXT_RE, '') + '.metadata.json'; }

// Per-bucket record store — series/issues/universes/media/reviews/outlines JSON
// live under `records/<type>/<id>.json`. Unlike the blob helpers above, these do
// NOT validate `id`: every callsite supplies an id that's either locally
// authored (the exporter) or already screened by `isSafeRecordId` (the
// importer) before it reaches the path. Keeping them as plain joins preserves
// the prior hand-joined behavior exactly.
export function bucketRecordsDir(bucketPath, type) { return join(bucketPath, 'records', type); }
export function bucketRecordPath(bucketPath, type, id) { return join(bucketRecordsDir(bucketPath, type), `${id}.json`); }

/**
 * Returns the filename if it's safe to use as a path segment under an asset
 * directory, otherwise null. Rejects path separators, parent-directory
 * tokens, and any value that doesn't match its own basename — the canonical
 * traversal guard for inbound/peer-supplied asset filenames before they hit a
 * `join(dir, name)` FS op. Shared by the peer-sync diff/pull paths
 * (`peerSync.js`, `sidecarSync.js`) so every callsite scrubs identically.
 */
export function sanitizeAssetFilename(name) {
  if (typeof name !== 'string' || !name) return null;
  // Reject separators and exact parent-directory segments (`.` / `..`
  // as the whole basename). A basename like `my..render.png` is
  // legitimate (the gallery filename validator permits `..` inside a
  // basename) — only the path-segment forms are traversal.
  if (name.includes('/') || name.includes('\\')) return null;
  if (name === '.' || name === '..') return null;
  if (basename(name) !== name) return null;
  return name;
}

/** Lay out the canonical bucket structure (idempotent). */
export async function ensureBucketLayout(bucket) {
  const base = bucket.path;
  await ensureDir(base);
  await ensureDir(join(base, 'manifests'));
  await ensureDir(bucketRecordsDir(base, 'series'));
  await ensureDir(bucketRecordsDir(base, 'issues'));
  await ensureDir(bucketRecordsDir(base, 'universes'));
  await ensureDir(bucketRecordsDir(base, 'media'));
  await ensureDir(bucketRecordsDir(base, 'reviews'));
  await ensureDir(bucketRecordsDir(base, 'outlines'));
  await ensureDir(join(base, 'assets', 'images'));
  await ensureDir(join(base, 'assets', 'videos'));
  await ensureDir(join(base, 'assets', 'blobs'));
  const bucketJsonPath = join(base, 'bucket.json');
  const existing = await readJSONFile(bucketJsonPath, null, { logError: false, strict: true });
  if (!existing) {
    // bucket.json is the shared identity. Stamp the schema version at
    // creation time so peers can tell at a glance which protocol the bucket
    // is operating under. We do NOT bump an existing peer's bucket.json
    // unprompted — if it's there, leave it (the peer's writer follows its
    // local SHARING_SCHEMA_VERSION; the importer compares per-manifest).
    await atomicWrite(bucketJsonPath, {
      id: bucket.id,
      name: bucket.name,
      schemaVersion: SHARING_SCHEMA_VERSION,
      sharingSchemaVersion: SHARING_SCHEMA_VERSION,
      createdAt: bucket.createdAt,
    });
  }
}

/** Read the on-disk bucket.json so routes/UI can report producedBy + schema info. */
export async function readBucketJson(bucket) {
  const bucketJsonPath = join(bucket.path, 'bucket.json');
  // Read-only UI metadata; ensureBucketLayout uses its own strict creation guard.
  return readJSONFile(bucketJsonPath, null, { logError: false });
}

export async function listBuckets() {
  const { buckets } = await readRegistry();
  return [...buckets].sort((a, b) => (a.name || '').localeCompare(b.name || ''));
}

export async function getBucket(id) {
  const { buckets } = await readRegistry();
  const found = buckets.find((b) => b.id === id);
  if (!found) throw codedError(`Bucket not found: ${id}`, ERR_NOT_FOUND);
  return found;
}

export async function createBucket(input = {}) {
  const name = trimTo(input.name, NAME_MAX);
  if (!name) throw codedError(`Bucket name is required (1..${NAME_MAX} chars)`, ERR_VALIDATION);
  const path = trimTo(input.path, PATH_MAX);
  if (!path) throw codedError('Bucket path is required', ERR_VALIDATION);
  await assertPathUsable(path);
  return queueRegistryWrite(async () => {
    const state = await readRegistry();
    // Reject duplicate paths so two registry entries can't both attempt to
    // claim the same on-disk folder (they would fight over bucket.json + the
    // watcher would double-fire on every manifest).
    if (state.buckets.some((b) => b.path === path)) {
      throw codedError(`A bucket is already registered at: ${path}`, ERR_VALIDATION);
    }
    const now = new Date().toISOString();
    const bucket = sanitizeBucket({
      id: `bkt-${randomUUID()}`,
      name,
      path,
      mode: input.mode || 'inbox',
      displayNameOverride: input.displayNameOverride || null,
      bioOverride: input.bioOverride || null,
      createdAt: now,
      updatedAt: now,
    });
    state.buckets.push(bucket);
    // Validate/read the shared identity before publishing the local registration.
    await ensureBucketLayout(bucket);
    await writeRegistry(state);
    return bucket;
  });
}

export function updateBucket(id, patch = {}) {
  return queueRegistryWrite(async () => {
    const state = await readRegistry();
    const idx = state.buckets.findIndex((b) => b.id === id);
    if (idx < 0) throw codedError(`Bucket not found: ${id}`, ERR_NOT_FOUND);
    const cur = state.buckets[idx];
    const merged = sanitizeBucket({
      ...cur,
      ...('name' in patch ? { name: patch.name } : {}),
      ...('mode' in patch ? { mode: patch.mode } : {}),
      ...('displayNameOverride' in patch ? { displayNameOverride: patch.displayNameOverride } : {}),
      ...('bioOverride' in patch ? { bioOverride: patch.bioOverride } : {}),
      updatedAt: new Date().toISOString(),
    });
    if (!merged) throw codedError('Invalid bucket payload', ERR_VALIDATION);
    // `path` is intentionally NOT patchable — if the user wants to move a
    // bucket, they delete and re-register so the registry can re-validate and
    // re-layout the new path.
    state.buckets[idx] = merged;
    await writeRegistry(state);
    return merged;
  });
}

export function deleteBucket(id) {
  return queueRegistryWrite(async () => {
    const state = await readRegistry();
    const before = state.buckets.length;
    state.buckets = state.buckets.filter((b) => b.id !== id);
    if (state.buckets.length === before) throw codedError(`Bucket not found: ${id}`, ERR_NOT_FOUND);
    await writeRegistry(state);
    return { id };
  });
}