Skip to content

Never stop coding. Free MIT AI gateway: one endpoint, 375 providers (150+ free), 1,400+ models Kimi, Claude, GPT, Gemini, GLM, DeepSeek, MiniMax. Works with Claude Code, Codex, Cursor, OpenCode, Cline & Copilot. Quota-aware auto-fallback, RTK+Caveman compression saves 15-95% tokens, MCP/A2A, Desktop/PWA. Built by 900+ contributors

16.7 KB
Raw Blame
/**
 * generator.ts — idempotent SKILL.md generator for all 45 agent skills.
 *
 * Usage (library):
 *   import { generateAgentSkills, buildSkillMarkdown } from "@/lib/agentSkills/generator";
 *
 * Usage (CLI):
 *   node scripts/skills/generate-agent-skills.mjs [--apply] [--prune] [--only=id1,id2] [--json]
 *
 * Safety contract:
 *  - Default: dryRun=true, prune=false (no filesystem writes).
 *  - dryRun=false is required for any actual writes.
 *  - prune=true AND dryRun=false is required to delete orphan directories.
 *  - F9 is responsible for running apply; F3 only implements + tests the generator.
 */

import fs from "node:fs";
import path from "node:path";
import { getCatalog, refreshCatalog } from "./catalog";
import { parseOpenapi } from "./openapiParser";
import { parseCliRegistry } from "./cliRegistryParser";
import { buildApiOperationExample } from "./apiOperationExample";
import type { AgentSkill, GeneratorOptions, GeneratorReport } from "./types";
import type { OpenapiPath, ParsedOpenapi } from "./openapiParser";
import type { ParsedCliRegistry } from "./cliRegistryParser";

// ── Constants ─────────────────────────────────────────────────────────────────

const GENERATED_COMMENT =
  "<!-- generated by src/lib/agentSkills/generator.ts; manual edits will be overwritten -->";

const CUSTOM_START_MARKER = "<!-- skill:custom-start -->";
const CUSTOM_END_MARKER = "<!-- skill:custom-end -->";

// Runnable examples for commands that need concrete arguments or safety-critical flags.
const CLI_PREVIEW_EXAMPLES: Record<string, string> = {
  "sync pull [outPath]": "sync pull omniroute-export.tar.gz",
  "models test-add <model>":
    "models test-add example-model --provider example-provider --connection example-connection --dry-run",
};

const MAX_SKILL_BODY_LINES = 500;
const REFERENCE_TOC_THRESHOLD_LINES = 100;

// ── Types ─────────────────────────────────────────────────────────────────────

interface BuildSources {
  openapi: ParsedOpenapi;
  cliRegistry: ParsedCliRegistry;
}

interface GeneratedSkillReference {
  path: string;
  content: string;
}

interface ApiSkillDocuments {
  body: string;
  references: GeneratedSkillReference[];
}

// ── Frontmatter helpers ───────────────────────────────────────────────────────

/**
 * Serializes frontmatter to YAML front-matter block.
 * Handles descriptions with colons or special characters safely.
 */
function serializeFrontmatter(fm: { name: string; description: string }): string {
  // Use block scalar for description if it contains colons, quotes, or newlines
  const needsQuote = (v: string): boolean => /[:\n"']/.test(v) || v.startsWith(" ");

  // Escape order matters for double-quoted YAML scalars: backslash FIRST (so the
  // escapes we add below are not themselves re-escaped), then the double-quote,
  // then collapse real newlines into the \n escape sequence.
  const escDq = (v: string): string => v.replace(/\\/g, "\\\\").replace(/"/g, '\\"');
  const nameStr = needsQuote(fm.name) ? `"${escDq(fm.name)}"` : fm.name;
  const descStr = needsQuote(fm.description)
    ? `"${escDq(fm.description).replace(/\n/g, "\\n")}"`
    : fm.description;

  return `---\nname: ${nameStr}\ndescription: ${descStr}\n---\n`;
}

/**
 * Extracts the custom block content from an existing SKILL.md string.
 * Returns null if no custom block is found.
 */
function extractCustomBlock(content: string): string | null {
  const startIdx = content.indexOf(CUSTOM_START_MARKER);
  const endIdx = content.indexOf(CUSTOM_END_MARKER);
  if (startIdx === -1 || endIdx === -1 || endIdx <= startIdx) return null;
  return content.slice(startIdx, endIdx + CUSTOM_END_MARKER.length);
}

// ── Body builders ─────────────────────────────────────────────────────────────

function endpointBlockLines(op: OpenapiPath, usesDashboardSession: boolean): string[] {
  const lines: string[] = [];

  lines.push(`### ${op.method} ${op.path}\n`);
  if (op.summary) {
    lines.push(`${op.summary}`);
    lines.push("");
  }
  if (op.description) {
    lines.push(op.description);
    lines.push("");
  }

  // Minimal curl example. Only omni-auth establishes and consumes a dashboard
  // session; generic API skills use independently usable Bearer examples.
  lines.push("```bash");
  lines.push(...buildApiOperationExample(op, usesDashboardSession));
  lines.push("```");
  lines.push("");

  return lines;
}

function markdownAnchor(value: string): string {
  return value
    .toLowerCase()
    .replace(/[^a-z0-9 -]/g, "")
    .replace(/ /g, "-");
}

function buildApiDocuments(skill: AgentSkill, sources: BuildSources): ApiSkillDocuments {
  const areaMap = sources.openapi.areas;
  const ops = areaMap.get(skill.area as Parameters<typeof areaMap.get>[0]) ?? [];
  const usesDashboardSession = skill.id === "omni-auth";

  const lines: string[] = [];

  lines.push("## Overview\n");
  lines.push(skill.description);
  lines.push("");

  lines.push("## Authentication\n");
  if (usesDashboardSession) {
    lines.push(
      "Remote API requests use a Bearer credential. Dashboard login is different: " +
        "`POST /api/auth/login` accepts a management password and returns an `auth_token` session cookie."
    );
  } else {
    lines.push(
      "All requests require a valid Bearer token or session cookie. " +
        "Obtain a token via `POST /api/auth/login` or configure `REQUIRE_API_KEY=false` for local development."
    );
  }
  lines.push("");

  const endpointBlocks = ops.map((op) => ({
    title: `${op.method} ${op.path}`,
    lines: endpointBlockLines(op, usesDashboardSession),
  }));
  const endpointLines =
    endpointBlocks.length === 0
      ? ["_No endpoints mapped for this area yet._"]
      : endpointBlocks.flatMap((block) => block.lines);
  const payloadLines = [
    "## Payloads\n",
    "See the full OpenAPI specification at `GET /api/openapi/spec` or " +
      "`docs/openapi.yaml` for detailed request/response schemas.",
    "",
  ];

  const inlineLines = [...lines, "## Endpoints\n", ...endpointLines, ...payloadLines];
  const inlineBody = inlineLines.join("\n");
  if (inlineLines.length <= MAX_SKILL_BODY_LINES) {
    return { body: inlineBody, references: [] };
  }

  const endpointIndex = endpointBlocks.map(
    (block) => `- [\`${block.title}\`](references/endpoints.md#${markdownAnchor(block.title)})`
  );
  const indexedLines = [...lines, "## Endpoints\n", ...endpointIndex, ...payloadLines];

  const referencePreamble = [GENERATED_COMMENT, "", "# Endpoint reference", ""];
  const tableOfContents = endpointBlocks.map(
    (block) => `- [\`${block.title}\`](#${markdownAnchor(block.title)})`
  );
  const needsTableOfContents =
    referencePreamble.length + tableOfContents.length + 2 + endpointLines.length >
    REFERENCE_TOC_THRESHOLD_LINES;
  const referenceLines = [...referencePreamble];
  if (needsTableOfContents) {
    referenceLines.push("## Table of contents", "", ...tableOfContents, "");
  }
  referenceLines.push(...endpointLines);
  const referenceContent = referenceLines.join("\n");

  return {
    body: indexedLines.join("\n"),
    references: [{ path: "references/endpoints.md", content: referenceContent }],
  };
}

function buildCliBody(skill: AgentSkill, sources: BuildSources): string {
  const familyMap = sources.cliRegistry.families;
  const cmds = familyMap.get(skill.area as Parameters<typeof familyMap.get>[0]) ?? [];

  const lines: string[] = [];

  lines.push("## Overview\n");
  lines.push(skill.description);
  lines.push("");

  lines.push("## Quick install\n");
  lines.push("```bash");
  lines.push("npm install -g omniroute   # or: npx omniroute");
  lines.push("omniroute --version");
  lines.push("```");
  lines.push("");

  lines.push("## Subcommands\n");

  if (cmds.length === 0) {
    lines.push("_No CLI subcommands mapped for this family yet._");
    lines.push("");
  } else {
    for (const cmd of cmds) {
      lines.push(`### \`${cmd.name}\`\n`);
      if (cmd.description) {
        lines.push(cmd.description);
        lines.push("");
      }

      if (cmd.flags.length > 0) {
        lines.push("**Flags:**\n");
        for (const flag of cmd.flags) {
          lines.push(`- \`${flag}\``);
        }
        lines.push("");
      }

      lines.push("**Example:**\n");
      lines.push("```bash");
      lines.push(`omniroute ${CLI_PREVIEW_EXAMPLES[cmd.name] ?? cmd.name}`);
      lines.push("```");
      lines.push("");
    }
  }

  return lines.join("\n");
}

// ── Core builder ─────────────────────────────────────────────────────────────

/**
 * Generates the full SKILL.md string (frontmatter + comment + body) for a given skill.
 *
 * If `existingContent` is provided, any `<!-- skill:custom-start --> ... <!-- skill:custom-end -->`
 * block found in it is re-injected after the generated body (marker preservation / D24).
 */
export function buildSkillMarkdown(
  skillId: string,
  sources: BuildSources,
  existingContent?: string
): {
  frontmatter: { name: string; description: string };
  body: string;
  references: GeneratedSkillReference[];
} {
  const skill = getCatalog().find((s) => s.id === skillId);
  if (!skill) {
    throw new Error(`buildSkillMarkdown: skill "${skillId}" not found in catalog`);
  }

  const fm = {
    name: skill.id,
    description: skill.description.slice(0, 2000),
  };

  const apiDocuments = skill.category === "api" ? buildApiDocuments(skill, sources) : undefined;
  const bodyLines =
    skill.category === "api"
      ? apiDocuments.body
      : skill.category === "external"
        ? "" // external: content lives in the custom block below
        : buildCliBody(skill, sources); // cli + config share the CLI-shaped body

  // Re-inject custom block if present in existing content
  let customBlock = "";
  if (existingContent) {
    const extracted = extractCustomBlock(existingContent);
    if (extracted) {
      customBlock = `\n${extracted}\n`;
    }
  }

  const body = GENERATED_COMMENT + "\n\n" + bodyLines + customBlock;

  return { frontmatter: fm, body, references: apiDocuments?.references ?? [] };
}

/**
 * Assembles the full file content string from frontmatter + body.
 */
function assembleFileContent(fm: { name: string; description: string }, body: string): string {
  return serializeFrontmatter(fm) + body;
}

function readReferenceFile(skillDir: string, referencePath: string): string | undefined {
  try {
    return fs.readFileSync(path.join(skillDir, referencePath), "utf-8");
  } catch {
    return undefined;
  }
}

// ── Main generator ─────────────────────────────────────────────────────────────

/**
 * Runs the full generator. Idempotent.
 *
 * - dryRun=true (default): returns report but writes nothing to disk.
 * - dryRun=false: writes/updates SKILL.md files as needed.
 * - prune=true: detects orphan directories in `skills/` not in the catalog.
 *   - dryRun=true + prune=true: lists orphans, no deletions.
 *   - dryRun=false + prune=true: deletes orphan directories.
 */
export async function generateAgentSkills(opts: GeneratorOptions): Promise<GeneratorReport> {
  const { dryRun = true, prune = false, outputDir = "skills", onlyIds } = opts;

  // Anchor the base path with a literal so Turbopack's static analyzer can resolve
  // it without falling back to tracing the entire project root. (#6329)
  // Honor an absolute outputDir (e.g. a tmp dir in tests) — path.join(cwd, "/abs")
  // would mangle it into cwd/abs, so guard with isAbsolute while keeping the
  // Turbopack-friendly join form for the common relative case ("skills"). (#6366 regression)
  const outputBase = path.isAbsolute(outputDir) ? outputDir : path.join(process.cwd(), outputDir);

  const catalog = getCatalog();
  const catalogIds = new Set(catalog.map((s) => s.id));

  // Filter to onlyIds if provided
  const skillsToProcess = onlyIds ? catalog.filter((s) => onlyIds.includes(s.id)) : catalog;

  // Lazily parse sources (only once per generator run)
  let _openapi: ParsedOpenapi | null = null;
  let _cliRegistry: ParsedCliRegistry | null = null;

  function getSources(): BuildSources {
    if (!_openapi) {
      try {
        _openapi = parseOpenapi();
      } catch {
        _openapi = { paths: new Map(), areas: new Map() };
      }
    }
    if (!_cliRegistry) {
      try {
        _cliRegistry = parseCliRegistry();
      } catch {
        _cliRegistry = { commands: new Map(), families: new Map() };
      }
    }
    return { openapi: _openapi, cliRegistry: _cliRegistry };
  }

  const report: GeneratorReport = {
    generated: [],
    unchanged: [],
    pruned: [],
    orphansDetected: [],
    errors: [],
  };

  // ── Detect orphans ──────────────────────────────────────────────────────────
  if (prune) {
    let dirs: string[] = [];
    try {
      dirs = fs
        .readdirSync(outputBase, { withFileTypes: true })
        .filter((e) => e.isDirectory())
        .map((e) => e.name);
    } catch {
      // outputDir doesn't exist yet — no orphans
    }

    for (const dir of dirs) {
      if (!catalogIds.has(dir)) {
        report.orphansDetected.push(dir);
        if (!dryRun) {
          try {
            fs.rmSync(path.join(outputBase, dir), { recursive: true, force: true });
            report.pruned.push(dir);
          } catch (err) {
            report.errors.push({
              id: dir,
              error: `Failed to prune: ${err instanceof Error ? err.message : String(err)}`,
            });
          }
        }
      }
    }
  }

  // ── Generate / compare each skill ──────────────────────────────────────────
  const sources = getSources();

  for (const skill of skillsToProcess) {
    try {
      const skillDir = path.join(outputBase, skill.id);
      const skillFile = path.join(skillDir, "SKILL.md");

      // Read existing content for marker preservation
      let existingContent: string | undefined;
      try {
        existingContent = fs.readFileSync(skillFile, "utf-8");
      } catch {
        existingContent = undefined;
      }

      const { frontmatter, body, references } = buildSkillMarkdown(
        skill.id,
        sources,
        existingContent
      );
      const newContent = assembleFileContent(frontmatter, body);

      const staleReferencePath = references.length === 0 ? "references/endpoints.md" : undefined;
      const referenceChanges = new Set(
        references
          .filter((reference) => readReferenceFile(skillDir, reference.path) !== reference.content)
          .map((reference) => reference.path)
      );
      if (staleReferencePath && readReferenceFile(skillDir, staleReferencePath) !== undefined) {
        referenceChanges.add(staleReferencePath);
      }
      const hasChanges = existingContent !== newContent || referenceChanges.size > 0;

      // Compare with existing to detect actual changes
      if (!hasChanges) {
        report.unchanged.push(skill.id);
        continue;
      }

      if (dryRun) {
        // In dry-run: classify as generated (would write/remove) but don't touch disk
        report.generated.push(skill.id);
      } else {
        // Apply: ensure directory + write file
        fs.mkdirSync(skillDir, { recursive: true });
        fs.writeFileSync(skillFile, newContent, "utf-8");
        for (const reference of references) {
          fs.mkdirSync(path.dirname(path.join(skillDir, reference.path)), { recursive: true });
          fs.writeFileSync(path.join(skillDir, reference.path), reference.content, "utf-8");
        }
        if (staleReferencePath) {
          fs.rmSync(path.join(skillDir, staleReferencePath), { force: true });
        }
        report.generated.push(skill.id);
      }
    } catch (err) {
      report.errors.push({
        id: skill.id,
        error: err instanceof Error ? err.message : String(err),
      });
    }
  }

  // After writing, refresh catalog cache so subsequent calls reflect new files
  if (!dryRun && report.generated.length > 0) {
    refreshCatalog();
  }

  return report;
}

// Exported for unit testing only — keep the internal helpers reachable without
// widening the public surface.
export const __testing = { serializeFrontmatter };