Skip to content

A local-first multi-agent app development platform, built on PenguinHarness

22.1 KB
Raw Blame
/**
 * Plugin registries: WHERE plugin index entries come from. A registry is one source
 * of `PluginIndexEntry` rows — the shared index format every registry speaks (see
 * api/types.ts; the schema follows typst/packages' `index.json`: a flat array of
 * per-version entries). Discovery only: a Project asks for an entry on the Plugins page
 * (http/routes/plugins-installed.ts), and nothing here imports plugin code.
 *
 * Two implementations, one contract:
 *   - the builtin registry serves the index embedded in this package (builtin-index.json —
 *     the curated entries, today the four sandbox backends) plus one entry per plugin package
 *     this build ships, each read from the package itself;
 *   - the HTTP registry fetches an `index.json` URL and runs it through the same
 *     validator, so a remote index is trusted no further than the embedded one.
 *
 * The deployment's registry list is fixed to the builtin one for now; additional
 * sources plug in as more `PluginRegistry` values.
 */
import { builtinPluginDir, loadLibraryPlugins } from "@adelie/core";
import type { PluginIndexEntry } from "../api/types.js";
import builtinIndex from "./builtin-index.json" with { type: "json" };
import fs from "node:fs/promises";
import path from "node:path";
import { discoverBuiltinPlugins, resolvePluginPackage } from "./loader.js";
import type { PluginBase } from "./loader.js";

/** One source of plugin index entries; `source` identifies it for display and errors. */
export interface PluginRegistry {
  readonly source: string;
  index(): Promise<PluginIndexEntry[]>;
  /**
   * Long-form documentation for one entry, or null when this source has none for it.
   *
   * Separate from `index` because the shapes differ: the index is a listing sent in full
   * on every page load, a readme is large and wanted only for the entry someone opened.
   */
  readme(name: string): Promise<string | null>;
}

function isStringArray(value: unknown): value is string[] {
  return Array.isArray(value) && value.every((v) => typeof v === "string");
}

/** Validates one raw entry; returns null instead of throwing so the caller can name the index position. */
function asIndexEntry(value: unknown): PluginIndexEntry | null {
  if (typeof value !== "object" || value === null) return null;
  const e = value as Record<string, unknown>;
  if (
    typeof e.name !== "string" ||
    typeof e.version !== "string" ||
    typeof e.description !== "string" ||
    !isStringArray(e.authors) ||
    typeof e.license !== "string"
  ) {
    return null;
  }
  for (const key of ["repository", "homepage"] as const) {
    if (e[key] !== undefined && typeof e[key] !== "string") return null;
  }
  for (const key of ["keywords", "categories"] as const) {
    if (e[key] !== undefined && !isStringArray(e[key])) return null;
  }
  if (e.updatedAt !== undefined && typeof e.updatedAt !== "number") return null;
  return value as PluginIndexEntry;
}

/**
 * Validates a whole index document. Strict, not per-entry-tolerant: an index is one
 * publisher's single artifact, so a malformed row means the artifact is broken —
 * unlike a Project's plugin list, whose entries are independent choices skipped one
 * by one.
 */
export function parsePluginIndex(data: unknown, source: string): PluginIndexEntry[] {
  if (!Array.isArray(data)) {
    throw new Error(`plugin index from ${source} is not an array`);
  }
  return data.map((raw, i) => {
    const entry = asIndexEntry(raw);
    if (entry === null) {
      throw new Error(`plugin index from ${source} has a malformed entry at index ${i}`);
    }
    return entry;
  });
}

export const BUILTIN_REGISTRY_SOURCE = "builtin";

/**
 * Who the entries this package publishes are attributed to: the workspace's own plugin
 * packages carry no `author` field, and the embedded index names the project itself — the
 * same identity the docs' footer carries.
 */
const PUBLISHER = "Prism Shadow";

/**
 * The scope every built-in plugin package is published under. Core names a built-in plugin by
 * its DIRECTORY, which is the dependency name it read out of the host package with this scope
 * stripped (see core's plugin roots) — so scoping the name back is how the rest of the system
 * (a Project's plugin list, an install, the market's shelf) names the same package.
 */
const PLUGIN_SCOPE = "@adelie/";

/**
 * The built-in library's plugin packages, by specifier: what this build carries because the
 * host package declares it — the same list the Plugins page's cards come from. Names only, and
 * nothing is read from the packages here.
 */
export function builtinLibraryNames(root: string): string[] {
  return loadLibraryPlugins(root)
    .filter((plugin) => plugin.source === "builtin")
    .map((plugin) => `${PLUGIN_SCOPE}${plugin.name}`);
}

/**
 * Every plugin package this build carries, by specifier: the built-in library's own packages
 * (the ones the host package declares — the same list the Plugins page's cards come from) plus
 * whatever a builtin prefix stages on top of them (the desktop app and a hot push stage theirs
 * beside `skills/` — see scripts/build-plugins.mjs). This is the MARKET's shelf: the packages
 * whose entries the index describes even when nobody wrote one by hand.
 *
 * It is not the deployment's answer to "may this Project ask for it": a Project's plugin list
 * takes MODULE plugins (packages with a code entry the server can load), which is
 * `discoverBuiltinPlugins` over the builtin prefixes and nothing else — a skills/hooks package
 * has no entry file, so listing one there would only produce a row that fails to load.
 */
export async function shippedPluginNames(
  root: string,
  bases: readonly PluginBase[],
): Promise<string[]> {
  const names = new Set(builtinLibraryNames(root));
  for (const name of await discoverBuiltinPlugins(bases)) names.add(name);
  return [...names].sort();
}

/**
 * Where one plugin package's own files are: its directory, or null when this machine does not
 * have it. Two kinds of package are on the shelf and they are found differently — a LIBRARY
 * package (the ones core carries, `@adelie/<directory>`) through core, which resolves them
 * from its own `node_modules` chain and knows a checkout's `plugins/<name>/` is the copy to
 * read; a staged MODULE plugin (a sandbox backend, seeded by a push) from the bases, the same
 * lookup a Project's install goes through. Nothing is assumed about either: whatever the
 * resolution finds is the package.
 */
function packageDir(specifier: string, bases: readonly PluginBase[]): string | null {
  if (specifier.startsWith(PLUGIN_SCOPE)) {
    const dir = builtinPluginDir(specifier.slice(PLUGIN_SCOPE.length));
    if (dir !== undefined) return dir;
  }
  return resolvePluginPackage(specifier, bases)?.dir ?? null;
}

/**
 * One catalogue entry for a plugin package on this machine, read from the package itself. A
 * plugin package carries two manifests and they answer different questions: `plugin.json` is
 * the PLUGIN's (its version, its category — the same version the Plugins page's card shows),
 * `package.json` is the npm package's (its description, license, keywords). Both are read, and
 * the entry takes each field from where it belongs; a code-only backend has no plugin.json and
 * falls back to the package's own version. Null when the package is not here, or when what it
 * does carry cannot name it — a catalogue entry invented around a package nobody can read would
 * be worse than the omission.
 *
 * What is deliberately left out: `repository` and `homepage` (a package's own `repository` is
 * a git URL with a directory, not a page the index could link, and guessing the page would put
 * a URL nobody verified in front of the reader), and `updatedAt` (nothing here knows when the
 * package was published).
 */
async function packageIndexEntry(
  name: string,
  bases: readonly PluginBase[],
): Promise<PluginIndexEntry | null> {
  const dir = packageDir(name, bases);
  if (dir === null) return null;
  let manifest: Record<string, unknown>;
  try {
    manifest = JSON.parse(await fs.readFile(path.join(dir, "package.json"), "utf8")) as Record<
      string,
      unknown
    >;
  } catch {
    return null;
  }
  const plugin = await pluginManifest(dir);
  const version = typeof plugin?.version === "string" ? plugin.version : manifest.version;
  const description = manifest.description;
  const license = manifest.license;
  if (
    typeof version !== "string" ||
    typeof description !== "string" ||
    typeof license !== "string"
  ) {
    return null;
  }
  const category = typeof plugin?.category === "string" ? plugin.category : null;
  const keywords = manifest.keywords;
  return {
    name,
    version,
    description,
    authors: [PUBLISHER],
    license,
    ...(category !== null && category !== "" ? { categories: [category] } : {}),
    ...(Array.isArray(keywords) && keywords.every((k) => typeof k === "string")
      ? { keywords: keywords as string[] }
      : {}),
  };
}

/** A plugin package's own manifest (`plugin.json` beside its `package.json`), or null when it has none — a code-only backend carries only the npm one. */
async function pluginManifest(dir: string): Promise<Record<string, unknown> | null> {
  try {
    const raw: unknown = JSON.parse(await fs.readFile(path.join(dir, "plugin.json"), "utf8"));
    return raw !== null && typeof raw === "object" ? (raw as Record<string, unknown>) : null;
  } catch {
    return null;
  }
}

/** What the built-in registry needs: where packages resolve from, and which names are this build's own. */
export interface BuiltinRegistryDeps {
  /** Where a package is looked up (the shipped prefix, the data root's, the installation). */
  bases?: () => readonly PluginBase[];
  /**
   * The plugin packages this build ships (see shippedPluginNames). The entries the embedded
   * index already carries are left as they are — it is the curated listing; this describes the
   * rest of the shelf from the packages themselves, so a plugin that ships is on the market
   * without anyone writing a second copy of its name, version and description by hand.
   */
  shipped?: () => Promise<readonly string[]>;
}

/**
 * The registry embedded in this package: the workspace's own plugin packages. The index is the
 * curated entries plus one entry per shipped package they do not already carry. A readme is the
 * package's own README.md, read from the copy on this machine (see packageDir) — the file the
 * package ships, never a second copy. A listed package that is not on this machine has none to
 * show, and one whose package ships no README serves none.
 */
export function builtinPluginRegistry(deps: BuiltinRegistryDeps = {}): PluginRegistry {
  const { bases = () => [], shipped = async () => [] } = deps;
  return {
    source: BUILTIN_REGISTRY_SOURCE,
    // Validated like any other source: a broken embedded index should fail loudly
    // in tests rather than serve garbage.
    index: async () => {
      const curated = parsePluginIndex(builtinIndex, BUILTIN_REGISTRY_SOURCE);
      const listed = new Set(curated.map((entry) => entry.name));
      const derived: PluginIndexEntry[] = [];
      for (const name of await shipped()) {
        if (listed.has(name)) continue;
        listed.add(name);
        const entry = await packageIndexEntry(name, bases());
        if (entry !== null) derived.push(entry);
      }
      return [...curated, ...derived];
    },
    readme: async (name) => {
      const dir = packageDir(name, bases());
      if (dir === null) return null;
      try {
        return await fs.readFile(path.join(dir, "README.md"), "utf8");
      } catch {
        return null;
      }
    },
  };
}

/** A registry behind an `index.json` URL; `fetchImpl` is injectable for tests. */
export function httpPluginRegistry(
  indexUrl: string,
  fetchImpl: typeof fetch = fetch,
): PluginRegistry {
  return {
    source: indexUrl,
    index: async () => {
      const res = await fetchImpl(indexUrl);
      if (!res.ok) {
        throw new Error(`plugin index from ${indexUrl} answered HTTP ${res.status}`);
      }
      let data: unknown;
      try {
        data = await res.json();
      } catch (err) {
        throw new Error(
          `plugin index from ${indexUrl} is not valid JSON: ${err instanceof Error ? err.message : String(err)}`,
        );
      }
      return parsePluginIndex(data, indexUrl);
    },
    // The shared index format carries no readme location, so a remote source has none to
    // offer yet. Null rather than a guessed URL: inventing one would have the Web App
    // render whatever answered it.
    readme: () => Promise.resolve(null),
  };
}

// ---------------------------------------------------------------------------
// The published side of the shelf
// ---------------------------------------------------------------------------

/** The npm registry this reads: npm's own, the same constant services/plugin-download.ts names. */
const NPM_REGISTRY = "https://registry.npmjs.org";

/**
 * The keyword a plugin package publishes itself under. It is how the market finds a plugin
 * that is NOT part of a build — a package the repository never carried, or one an older build
 * predates — without anyone curating a list: publish `@adelie/<name>` with this keyword and its
 * next version appears on the shelf.
 */
export const PLUGIN_KEYWORD = "adelie-plugin";

/**
 * The scopes the market lists from npm. One entry today: Adelie's own packages, which is what
 * makes the keyword safe to read — a keyword alone would let anyone put their package in front
 * of every user's Install button, while nobody outside the project can publish under this scope.
 * Widening this list is the deliberate act of letting another publisher onto the shelf.
 */
const MARKET_SCOPES: readonly string[] = ["@adelie/"];

/** How long a successful listing of the published side is reused. */
export const NPM_LIST_TTL_MS = 10 * 60 * 1000;
/** How long a failed listing is reused (short: a transient failure should heal on its own). */
export const NPM_LIST_FAILURE_TTL_MS = 2 * 60 * 1000;
/** How long one npm request may take. Bounded so an unreachable registry cannot pin a page load. */
const NPM_TIMEOUT_MS = 10_000;

/** What the npm registry reads through; a test stands in a canned fetch and a clock. */
export interface NpmRegistryOptions {
  fetchImpl?: typeof fetch;
  timeoutMs?: number;
  /** How long a successful listing is served from cache. */
  ttlMs?: number;
  /** How long a failed listing is served from cache. */
  failureTtlMs?: number;
  now?: () => number;
}

/** The npm search document's package object, as much of it as the index entry needs. */
interface NpmSearchPackage {
  name?: unknown;
  version?: unknown;
  description?: unknown;
  license?: unknown;
  keywords?: unknown;
  date?: unknown;
  publisher?: { username?: unknown };
  author?: { name?: unknown };
  maintainers?: Array<{ username?: unknown }>;
  links?: { npm?: unknown; repository?: unknown };
}

function isNonEmptyString(value: unknown): value is string {
  return typeof value === "string" && value !== "";
}

function stringArray(value: unknown): string[] {
  return Array.isArray(value) ? value.filter(isNonEmptyString) : [];
}

/**
 * A repository link the registry hands out, turned into a page a reader can open: npm writes
 * `git+https://…/x.git`, and the `git+` prefix and the `.git` tail are both for `git clone`,
 * not for a browser. Anything that is not http(s) after that is dropped — the index's links
 * are rendered as links, so a `git://` or a bare path would be a dead one.
 */
function browsableRepository(raw: unknown): string | undefined {
  if (typeof raw !== "string") return undefined;
  let value = raw.trim();
  if (value.startsWith("git+")) value = value.slice(4);
  if (value.endsWith(".git")) value = value.slice(0, -4);
  try {
    const url = new URL(value);
    return url.protocol === "http:" || url.protocol === "https:" ? url.toString() : undefined;
  } catch {
    return undefined;
  }
}

/**
 * One search result as an index entry, or null when this package cannot be described by one.
 *
 * Every field the entry needs must come from the package: an entry invented around a package
 * the registry described incompletely would put a nameless or licenseless row in front of the
 * reader, which is worse than leaving the package off (the same call the built-in registry's
 * own reader makes). `homepage` is the package's npm page — the one address the registry knows
 * is a page — and no `categories` are set: a category is the plugin's own (`plugin.json`), which
 * lives in the tarball, not in a search result.
 */
function searchEntry(raw: unknown): PluginIndexEntry | null {
  if (typeof raw !== "object" || raw === null) return null;
  const { package: pkg } = raw as { package?: unknown };
  if (typeof pkg !== "object" || pkg === null) return null;
  const p = pkg as NpmSearchPackage;
  const { name, version, description, license } = p;
  if (
    !isNonEmptyString(name) ||
    !isNonEmptyString(version) ||
    !isNonEmptyString(description) ||
    !isNonEmptyString(license)
  ) {
    return null;
  }
  if (!MARKET_SCOPES.some((scope) => name.startsWith(scope))) return null;
  const authors = [
    ...new Set(
      [
        p.publisher?.username,
        p.author?.name,
        ...(p.maintainers ?? []).map((m) => m?.username),
      ].filter(isNonEmptyString),
    ),
  ];
  if (authors.length === 0) return null;
  const keywords = stringArray(p.keywords);
  const date = typeof p.date === "string" ? Date.parse(p.date) : Number.NaN;
  const repository = browsableRepository(p.links?.repository);
  const homepage = typeof p.links?.npm === "string" ? p.links.npm : undefined;
  return {
    name,
    version,
    description,
    authors,
    license,
    ...(repository === undefined ? {} : { repository }),
    ...(homepage === undefined ? {} : { homepage }),
    ...(keywords.length === 0 ? {} : { keywords }),
    ...(Number.isFinite(date) ? { updatedAt: Math.floor(date / 1000) } : {}),
  };
}

/**
 * The published side of the market: `@adelie` packages that carry the plugin keyword, read from
 * npm's search API. This is what lets a plugin reach the shelf without a release — the built-in
 * registry can only describe what a build carries.
 *
 * Strictly fail-soft, like the update check: `index()` never throws. A failed lookup answers
 * with whatever the last successful one found (nothing, the first time) and is retried after a
 * short TTL, because a plugin page that cannot reach npm must still show the library. One
 * listing is shared by every caller and served from cache between lookups, so opening the page
 * repeatedly is one request per TTL.
 *
 * `readme` reads the package's own README from its packument — the file the package publishes,
 * the same one the built-in side reads from a local copy — and caches per name, including a
 * miss. It is a second request, made only when somebody opens an entry.
 */
export function npmPluginRegistry(options: NpmRegistryOptions = {}): PluginRegistry {
  const {
    fetchImpl = fetch,
    timeoutMs = NPM_TIMEOUT_MS,
    ttlMs = NPM_LIST_TTL_MS,
    failureTtlMs = NPM_LIST_FAILURE_TTL_MS,
    now = Date.now,
  } = options;
  const listing = `${NPM_REGISTRY}/-/v1/search?text=${encodeURIComponent(`keywords:${PLUGIN_KEYWORD}`)}&size=250`;
  let cached: { entries: PluginIndexEntry[]; expiresAt: number } | null = null;
  let inflight: Promise<PluginIndexEntry[]> | null = null;
  const readmes = new Map<string, string | null>();

  const search = async (): Promise<PluginIndexEntry[]> => {
    const res = await fetchImpl(listing, {
      headers: { accept: "application/json", "user-agent": "adelie-server" },
      signal: AbortSignal.timeout(timeoutMs),
    });
    if (!res.ok) throw new Error(`npm search answered HTTP ${res.status}`);
    const body = (await res.json()) as { objects?: unknown };
    const objects = Array.isArray(body.objects) ? body.objects : [];
    return objects.map(searchEntry).filter((e): e is PluginIndexEntry => e !== null);
  };

  return {
    source: "npm",
    index: async () => {
      if (cached !== null && now() < cached.expiresAt) return cached.entries;
      inflight ??= search().then(
        (entries) => {
          cached = { entries, expiresAt: now() + ttlMs };
          return entries;
        },
        (err: unknown) => {
          // The stale listing is better than an empty shelf; with none, the page simply shows
          // the library. One line, not a thrown error: this runs on a page load.
          console.warn(
            `plugin market: npm listing failed (${err instanceof Error ? err.message : String(err)}${cached === null ? "" : "; serving the previous listing"})`,
          );
          const entries = cached?.entries ?? [];
          cached = { entries, expiresAt: now() + failureTtlMs };
          return entries;
        },
      );
      try {
        return await inflight;
      } finally {
        inflight = null;
      }
    },
    readme: async (name) => {
      const hit = readmes.get(name);
      if (hit !== undefined) return hit;
      let found: string | null = null;
      try {
        const res = await fetchImpl(`${NPM_REGISTRY}/${encodeURIComponent(name)}`, {
          headers: { accept: "application/json", "user-agent": "adelie-server" },
          signal: AbortSignal.timeout(timeoutMs),
        });
        if (res.ok) {
          const body = (await res.json()) as { readme?: unknown };
          found = isNonEmptyString(body.readme) ? body.readme : null;
        }
      } catch {
        // A readme nobody can fetch is a missing readme, not a failed page: it is not cached,
        // so the next visit tries again.
        return null;
      }
      readmes.set(name, found);
      return found;
    },
  };
}

/**
 * The published side as this process is configured, or null when the deployment has opted out.
 *
 * `ADELIE_PLUGIN_MARKET=off` is for a machine that must reach nobody (an air-gapped install, a
 * locked-down network): the market then lists only what the build carries, exactly as it did
 * before this side existed. Reading it here rather than inside `npmPluginRegistry` keeps the
 * switch with the decision it makes, and out of the registry, which is one cached lookup whether
 * or not anybody uses it.
 */
export function configuredPluginMarket(
  env: Record<string, string | undefined>,
): PluginRegistry | null {
  return env["ADELIE_PLUGIN_MARKET"] === "off" ? null : npmPluginRegistry();
}