Skip to content

Beautiful, AI-native markdown IDE and LLM wiki

20.5 KB
Raw Blame
#!/usr/bin/env -S npx tsx
/**
 * Shared-content composer for the two OK skill bundles
 *
 * Source of truth (git-tracked):
 *   packages/server/assets/skills/discovery/SKILL.md
 *   packages/server/assets/skills/project/SKILL.md
 *   packages/server/assets/skills/_shared/<name>.md   — prose used by ≥2 bundles
 *
 * Source SKILL.md files MAY contain `{{> _shared/<name>.md }}` placeholders.
 * This build step resolves each placeholder against `_shared/` and writes the
 * composed result to a gitignored dist location:
 *
 *   packages/server/dist/assets/skills/<bundle>/SKILL.md
 *
 * A built `resolveBundledSkillDir()` (the CLI bundle, the server build the
 * desktop app loads) probes that dist path before the source path, so built
 * artifacts install the composed (placeholder-free) bundle; run from source it
 * reads the uncomposed source directly, which the source-fallback tripwire in
 * build-skill-bundles.test.ts keeps safe. At v1
 * `_shared/` carries no shared prose yet — neither bundle references a
 * placeholder, so composition is an identity transform — but the mechanism +
 * the byte-equality guard exist now so shared prose can land without drift
 * the moment two bundles overlap.
 *
 * Build-time, NOT runtime: composing at runtime would add install-time
 * complexity; this matches the standard build-step include pattern.
 */

import {
  cpSync,
  existsSync,
  lstatSync,
  mkdirSync,
  mkdtempSync,
  readdirSync,
  readFileSync,
  renameSync,
  rmSync,
  writeFileSync,
} from 'node:fs';
import { basename, dirname, join, relative } from 'node:path';
import { fileURLToPath } from 'node:url';
import { BUNDLE_IDS, BUNDLE_SKILL_NAME, type BundleId } from '../src/skill-bundles.ts';
import { enumeratePackSkills } from '../src/skill-pack-sources.ts';

// Re-export the canonical bundle-id list (single source: `skill-bundles.ts`) so
// existing consumers that import `BUNDLE_IDS` from this script keep working. The
// composer builds EVERY id in this set — adding a bundle in `skill-bundles.ts`
// now flows into the dist composition automatically, instead of the old
// hand-maintained literal here that silently omitted `write-skill`.
export { BUNDLE_IDS };

const SCRIPT_DIR = dirname(fileURLToPath(import.meta.url));
const PKG_ROOT = join(SCRIPT_DIR, '..');

/** Default on-disk layout. Overridable so unit tests can point at a fixture. */
export interface SkillBundlePaths {
  /** Directory holding `<bundle>/SKILL.md` + `_shared/`. */
  readonly skillsDir: string;
  /** Directory the composed `<bundle>/SKILL.md` files are written under. */
  readonly distDir: string;
}

export function defaultPaths(): SkillBundlePaths {
  return {
    skillsDir: join(PKG_ROOT, 'assets', 'skills'),
    distDir: join(PKG_ROOT, 'dist', 'assets', 'skills'),
  };
}

/**
 * The ONLY supported placeholder form (no conditional
 * includes, no variable substitution): `{{> _shared/<name>.md }}`. `<name>`
 * is a single path segment (the `.md` is part of the captured name).
 */
const PLACEHOLDER_RE = /\{\{>\s*_shared\/([A-Za-z0-9._-]+)\s*\}\}/g;

/**
 * Resolve every `{{> _shared/<name>.md }}` placeholder in `source` via the
 * `resolveShared` callback. Pure — no fs — so it is trivially unit-testable.
 * Returns the composed text plus the de-duplicated list of placeholder names
 * referenced (in first-seen order).
 */
export function composeSkill(
  source: string,
  resolveShared: (name: string) => string,
): { composed: string; placeholders: string[] } {
  const placeholders: string[] = [];
  const composed = source.replace(PLACEHOLDER_RE, (_match, name: string) => {
    if (!placeholders.includes(name)) placeholders.push(name);
    return resolveShared(name);
  });
  return { composed, placeholders };
}

function sharedResolver(skillsDir: string): (name: string) => string {
  const sharedDir = join(skillsDir, '_shared');
  return (name: string) => {
    const sharedPath = join(sharedDir, name);
    if (!existsSync(sharedPath)) {
      throw new Error(
        `Skill bundle references {{> _shared/${name} }} but ${sharedPath} does not exist.`,
      );
    }
    return readFileSync(sharedPath, 'utf-8');
  };
}

interface ComposedBundle {
  readonly bundle: BundleId;
  /** Composed (placeholder-free) SKILL.md text. */
  readonly composed: string;
  /** Placeholder names this bundle referenced. */
  readonly placeholders: string[];
  /** Where the composed file was (or would be) written. */
  readonly outputPath: string;
}

const REMOVAL_RETRIES = 3;

const STAGING_ROOT_PREFIX = '.ok-skill-publish-';

const KEPT_TREE_PREFIX = `${STAGING_ROOT_PREFIX}kept-`;

const STALE_STAGING_AGE_MS = 30_000;

const DESTINATION_OCCUPIED_CODES = new Set(['ENOTEMPTY', 'EEXIST']);

function discardTree(path: string): void {
  try {
    rmSync(path, { recursive: true, force: true, maxRetries: REMOVAL_RETRIES });
  } catch (err) {
    const reason = (err as NodeJS.ErrnoException).code ?? (err as Error).message;
    console.warn(`[build-skill-bundles] could not remove ${path}: ${reason}`);
  }
}

function reinstateTree(superseded: string, dest: string): boolean {
  try {
    renameSync(superseded, dest);
    return true;
  } catch (err) {
    const reason = (err as NodeJS.ErrnoException).code ?? (err as Error).message;
    console.error(
      `[build-skill-bundles] could not reinstate ${dest} from ${superseded}: ${reason}`,
    );
    return false;
  }
}

function destinationIsPublished(dest: string): boolean {
  try {
    return readdirSync(dest).length > 0;
  } catch {
    return false;
  }
}

function peerPublished(dest: string, err: unknown): boolean {
  const code =
    typeof err === 'object' && err !== null ? (err as NodeJS.ErrnoException).code : undefined;
  if (code !== undefined && DESTINATION_OCCUPIED_CODES.has(code)) return true;
  return destinationIsPublished(dest);
}

function stagingParentFor(distDir: string): string {
  let current = distDir;
  while (current !== dirname(current)) {
    if (basename(current) === 'dist') return dirname(current);
    current = dirname(current);
  }
  return dirname(distDir);
}

type TreeInspection =
  | { readonly ok: true; readonly newest: number }
  | { readonly ok: false; readonly failed: string };

function inspectTree(path: string): TreeInspection {
  let newest = 0;
  let failed: string | null = null;
  const visit = (current: string): void => {
    if (failed !== null) return;
    let isDirectory: boolean;
    try {
      const entry = lstatSync(current);
      if (entry.mtimeMs > newest) newest = entry.mtimeMs;
      isDirectory = entry.isDirectory();
    } catch (err) {
      failed = (err as NodeJS.ErrnoException).code ?? (err as Error).message;
      return;
    }
    if (!isDirectory) return;
    let children: string[];
    try {
      children = readdirSync(current);
    } catch (err) {
      failed = (err as NodeJS.ErrnoException).code ?? (err as Error).message;
      return;
    }
    for (const child of children) visit(join(current, child));
  };
  visit(path);
  return failed === null ? { ok: true, newest } : { ok: false, failed };
}

function keepDisplacedTree(holder: string): { readonly kept: boolean; readonly root: string } {
  const kept = join(
    dirname(holder),
    `${KEPT_TREE_PREFIX}${basename(holder).slice(STAGING_ROOT_PREFIX.length)}`,
  );
  try {
    renameSync(holder, kept);
    return { kept: true, root: kept };
  } catch (err) {
    const reason = (err as NodeJS.ErrnoException).code ?? (err as Error).message;
    console.error(
      `[build-skill-bundles] could not move ${holder} out of reclaimable scratch: ${reason}`,
    );
    return { kept: false, root: holder };
  }
}

const sweepNotices = new Set<string>();

function noticeOnce(path: string, note: string): void {
  if (sweepNotices.has(path)) return;
  sweepNotices.add(path);
  console.warn(`[build-skill-bundles] ${path}: ${note}`);
}

let sweepRuns = 0;

function reapStaleStagingRoots(stagingParent: string): boolean {
  sweepRuns += 1;
  const cutoff = Date.now() - STALE_STAGING_AGE_MS;
  let entries: string[];
  try {
    entries = readdirSync(stagingParent);
  } catch (err) {
    const reason = (err as NodeJS.ErrnoException).code ?? (err as Error).message;
    noticeOnce(stagingParent, `could not be swept for reclaimable scratch (${reason})`);
    return false;
  }
  for (const name of entries) {
    if (!name.startsWith(STAGING_ROOT_PREFIX)) continue;
    const full = join(stagingParent, name);
    if (name.startsWith(KEPT_TREE_PREFIX)) {
      noticeOnce(
        full,
        'not reclaimed; it holds the tree a failed publish kept, which nothing removes automatically',
      );
      continue;
    }
    const age = inspectTree(full);
    if (!age.ok) {
      if (age.failed !== 'ENOENT') {
        noticeOnce(full, `not reclaimed; its age could not be established (${age.failed})`);
      }
      continue;
    }
    if (age.newest >= cutoff) continue;
    discardTree(full);
  }
  return true;
}

const sweptStagingParents = new Set<string>();

function openStagingRoot(distDir: string): string {
  const stagingParent = stagingParentFor(distDir);
  mkdirSync(stagingParent, { recursive: true });
  if (!sweptStagingParents.has(stagingParent) && reapStaleStagingRoots(stagingParent)) {
    sweptStagingParents.add(stagingParent);
  }
  return mkdtempSync(join(stagingParent, STAGING_ROOT_PREFIX));
}

/* WARN: `packages/app/scripts/copy-excalidraw-assets.mjs` publishes a directory by the same
   stage-then-displace-then-rename shape; its failure semantics are its own. */
function publishTree(stagingRoot: string, staged: string, dest: string): void {
  mkdirSync(dirname(dest), { recursive: true });
  const holder = mkdtempSync(`${stagingRoot}-superseded-`);
  const superseded = join(holder, basename(dest));
  let displaced = false;
  let stranded = false;
  try {
    try {
      renameSync(dest, superseded);
      displaced = true;
    } catch (err) {
      const code = (err as NodeJS.ErrnoException).code;
      if (code !== 'ENOENT' && existsSync(dest)) throw err;
      if (code !== 'ENOENT') {
        console.warn(
          `[build-skill-bundles] could not displace ${dest} before publishing: ${code ?? (err as Error).message}`,
        );
      }
    }
    try {
      renameSync(staged, dest);
    } catch (err) {
      if (peerPublished(dest, err)) {
        console.warn(
          `[build-skill-bundles] yielded ${dest} to a peer that published it first; discarding this run's tree for it`,
        );
        return;
      }
      if (!displaced) throw err;
      if (reinstateTree(superseded, dest)) throw err;
      stranded = true;
      const reason = (err as NodeJS.ErrnoException).code ?? (err as Error).message;
      const outcome = keepDisplacedTree(holder);
      const copy = join(outcome.root, basename(dest));
      const fate = outcome.kept
        ? `kept at ${copy}`
        : `left at ${copy}, which a later build may reclaim`;
      throw new Error(
        `[build-skill-bundles] could not publish ${dest} (${reason}) and could not put back the tree it displaced: ${dest} is now absent and that tree is its only copy, ${fate}`,
      );
    }
  } finally {
    if (!stranded) discardTree(holder);
  }
}

function stageComposedBundle(
  sourceDir: string,
  stageDir: string,
  resolve: (name: string) => string,
): { composed: string; placeholders: string[] } {
  const source = readFileSync(join(sourceDir, 'SKILL.md'), 'utf-8');
  const { composed, placeholders } = composeSkill(source, resolve);
  cpSync(sourceDir, stageDir, { recursive: true });
  writeFileSync(join(stageDir, 'SKILL.md'), composed, 'utf-8');
  return { composed, placeholders };
}

/*
 * WARN: `packages/cli/scripts/build-skill-assets.ts` calls `buildSkillBundles`
 * and `buildPackSkills` by relative path and depends on both signatures, so a
 * change to either one has to land in that consumer in the same edit.
 */
/**
 * Compose every bundle and write the result to `distDir/<bundle>/SKILL.md`.
 * Returns one entry per bundle.
 */
export function buildSkillBundles(paths: SkillBundlePaths = defaultPaths()): ComposedBundle[] {
  const resolve = sharedResolver(paths.skillsDir);
  const results: ComposedBundle[] = [];
  const stagingRoot = openStagingRoot(paths.distDir);
  try {
    for (const bundle of BUNDLE_IDS) {
      const stageDir = join(stagingRoot, bundle);
      const { composed, placeholders } = stageComposedBundle(
        join(paths.skillsDir, bundle),
        stageDir,
        resolve,
      );
      const outDir = join(paths.distDir, bundle);
      publishTree(stagingRoot, stageDir, outDir);
      results.push({ bundle, composed, placeholders, outputPath: join(outDir, 'SKILL.md') });
    }
  } finally {
    discardTree(stagingRoot);
  }
  return results;
}

/**
 * Compose every per-pack skill into the dist tree the same way the named bundles
 * are composed. Pack skills carry no `_shared` placeholders today, so composition
 * is an identity transform — but routing them through `composeSkill` keeps them
 * on the same mechanism, and (critically) gets them INTO
 * `dist/assets/skills/packs/<id>/` so the published CLI + desktop bundles actually
 * ship them. Without this, `resolveBundledSkillDir('packs/<id>')` only resolves
 * against the source tree (dev), and `ok seed`'s pack-skill install silently
 * no-ops in any built artifact.
 *
 * A decomposed pack holds several `SKILL.md` files (root + one per member
 * scenario dir); the dist tree keeps that nested layout verbatim, since the
 * installer and the mirror both enumerate it. Returns the skill names built.
 */
export function buildPackSkills(paths: SkillBundlePaths = defaultPaths()): string[] {
  const packsSrc = join(paths.skillsDir, 'packs');
  if (!existsSync(packsSrc)) return [];
  const resolve = sharedResolver(paths.skillsDir);
  const built: string[] = [];
  const stagingRoot = openStagingRoot(paths.distDir);
  try {
    for (const entry of readdirSync(packsSrc, { withFileTypes: true })) {
      if (!entry.isDirectory()) continue;
      const sourceDir = join(packsSrc, entry.name);
      const skills = enumeratePackSkills(sourceDir);
      if (skills.length === 0) continue;
      const composedSkills = skills.map((skill) => ({
        name: skill.name,
        rel: relative(sourceDir, skill.sourceDir),
        composed: composeSkill(readFileSync(join(skill.sourceDir, 'SKILL.md'), 'utf-8'), resolve)
          .composed,
      }));
      const stageDir = join(stagingRoot, 'packs', entry.name);
      cpSync(sourceDir, stageDir, { recursive: true });
      for (const skill of composedSkills) {
        writeFileSync(join(stageDir, skill.rel, 'SKILL.md'), skill.composed, 'utf-8');
        built.push(skill.name);
      }
      publishTree(stagingRoot, stageDir, join(paths.distDir, 'packs', entry.name));
    }
  } finally {
    discardTree(stagingRoot);
  }
  return built;
}

/**
 * Compose the Agent Plugins (agent-plugins.org) view of the built-ins: a
 * conformant plugin directory —
 * `dist/assets/agent-plugin/{plugin.json, skills/<real skill name>/…}`.
 *
 * Derived, not a second source: the internal `assets/skills/<id>` layout stays
 * the single source of truth (its id→name indirection and probe chain are
 * load-bearing across installed apps), and this artifact re-materializes from
 * it on every build, so it cannot drift. The `skills/` children carry the
 * skills' REAL names, per the standard's install convention.
 */
export function buildAgentPluginArtifact(paths: SkillBundlePaths = defaultPaths()): string {
  const outRoot = join(paths.distDir, '..', 'agent-plugin');
  const resolve = sharedResolver(paths.skillsDir);
  const stagingRoot = openStagingRoot(paths.distDir);
  const stageRoot = join(stagingRoot, 'agent-plugin');
  try {
    for (const bundle of BUNDLE_IDS) {
      stageComposedBundle(
        join(paths.skillsDir, bundle),
        join(stageRoot, 'skills', BUNDLE_SKILL_NAME[bundle]),
        resolve,
      );
    }
    const manifest = {
      $schema: 'https://agent-plugins.org/schemas/1.0.0/plugin.schema.json',
      name: 'open-knowledge',
      description:
        'OpenKnowledge built-in skills: project workflow, discovery, and skill authoring',
      author: { name: 'Inkeep' },
      repository: 'https://github.com/inkeep/open-knowledge',
      keywords: ['openknowledge', 'knowledge-base'],
    };
    writeFileSync(
      join(stageRoot, 'plugin.json'),
      `${JSON.stringify(manifest, null, 2)}\n`,
      'utf-8',
    );
    publishTree(stagingRoot, stageRoot, outRoot);
  } finally {
    discardTree(stagingRoot);
  }
  return outRoot;
}

interface ByteEqualityResult {
  readonly ok: boolean;
  readonly violations: string[];
}

/**
 * Byte-equality guard. For every `{{> _shared/<name>.md }}`
 * placeholder referenced by any bundle, assert (a) the `_shared/<name>` file
 * exists and (b) the composed output of EVERY referencing bundle contains the
 * shared file's exact bytes. Because all bundles resolve the same `_shared/`
 * file, "byte-identical content across bundles" reduces to: every referencing
 * bundle's composed output embeds that one file verbatim.
 *
 * No fs writes — recomposes in memory. At v1 (`_shared/` empty, no
 * placeholders) this passes trivially.
 */
export function checkSharedContentByteEquality(
  paths: SkillBundlePaths = defaultPaths(),
): ByteEqualityResult {
  const violations: string[] = [];
  const sharedDir = join(paths.skillsDir, '_shared');
  const sharedCache = new Map<string, string>();
  const readShared = (name: string): string => {
    const cached = sharedCache.get(name);
    if (cached !== undefined) return cached;
    const text = readFileSync(join(sharedDir, name), 'utf-8');
    sharedCache.set(name, text);
    return text;
  };

  const composed = new Map<BundleId, { text: string; placeholders: string[] }>();
  for (const bundle of BUNDLE_IDS) {
    const source = readFileSync(join(paths.skillsDir, bundle, 'SKILL.md'), 'utf-8');
    // Collect placeholders without resolving — a missing file is reported as a
    // violation rather than thrown, so the guard surfaces every problem at once.
    const placeholders: string[] = [];
    composeSkill(source, (name) => {
      if (!placeholders.includes(name)) placeholders.push(name);
      return '';
    });
    for (const name of placeholders) {
      if (!existsSync(join(sharedDir, name))) {
        violations.push(`bundle '${bundle}' references {{> _shared/${name} }} — file is missing`);
      }
    }
    composed.set(bundle, { text: '', placeholders });
  }

  // Second pass: now that missing-file violations are recorded, recompose the
  // bundles whose placeholders all resolve and assert the embedded bytes.
  for (const bundle of BUNDLE_IDS) {
    const entry = composed.get(bundle);
    if (!entry) continue;
    const resolvable = entry.placeholders.every((name) => existsSync(join(sharedDir, name)));
    if (!resolvable) continue;
    const source = readFileSync(join(paths.skillsDir, bundle, 'SKILL.md'), 'utf-8');
    const { composed: text } = composeSkill(source, readShared);
    for (const name of entry.placeholders) {
      if (!text.includes(readShared(name))) {
        violations.push(
          `bundle '${bundle}' composed output is not byte-identical to _shared/${name}`,
        );
      }
    }
  }

  return { ok: violations.length === 0, violations };
}

if (import.meta.main) {
  const check = process.argv.includes('--check');
  if (check) {
    const result = checkSharedContentByteEquality();
    if (result.ok) {
      console.log('[build-skill-bundles] shared-content byte-equality check passed.');
    } else {
      console.error('[build-skill-bundles] shared-content byte-equality check FAILED:');
      for (const v of result.violations) console.error(`  - ${v}`);
      process.exitCode = 1;
    }
  } else {
    let stage = 'composing the skill bundles';
    try {
      const built = buildSkillBundles();
      for (const b of built) {
        const note =
          b.placeholders.length > 0
            ? ` (resolved ${b.placeholders.length} placeholder(s): ${b.placeholders.join(', ')})`
            : ' (no placeholders)';
        console.log(`[build-skill-bundles] composed ${b.bundle} → ${b.outputPath}${note}`);
      }
      stage = 'composing the pack skills';
      const packs = buildPackSkills();
      if (packs.length > 0) {
        console.log(
          `[build-skill-bundles] composed ${packs.length} pack skill(s): ${packs.join(', ')}`,
        );
      }
      stage = 'composing the Agent Plugins artifact';
      const pluginRoot = buildAgentPluginArtifact();
      console.log(`[build-skill-bundles] composed Agent Plugins artifact → ${pluginRoot}`);
    } catch (err) {
      console.error(`[build-skill-bundles] ${stage} failed:`);
      console.error(err);
      process.exitCode = 1;
    }
  }
}

export const __testing = { peerPublished, keepDisplacedTree, sweepRuns: () => sweepRuns };