Skip to content

An open source design system that's fully customizable and agent ready

TypeScript
22.9 KB
Raw Blame
// Copyright (c) Meta Platforms, Inc. and affiliates.

/**
 * @file The doc syntax for commands a reader runs: `{@command …}` for an
 * Astryx command, `{@pm …}` for a package-manager step, `{@pm-only <manager>
 * "<reason>" …}` for a step only one package manager can do, and `{@@…}` to
 * show a token as written. Each token is rendered for the reader: an Astryx
 * command with the reader's runner, a package-manager step in the reader's
 * package manager.
 *
 * @input Doc text, the reader's CLI invocation and package manager.
 * @output The text with each token rendered; each token's parse, for doctor;
 *   the token a plain-text command should be written as, for doctor's fix.
 * @position foundation/doc-compiler — pure; the env lookup lives in
 *   foundation/env/package-manager.mjs.
 */

/**
 * The package managers a `{@pm}` token renders for. `yarn-classic` is Yarn 1;
 * `yarn` is Yarn 2 and newer.
 * @typedef {'npm' | 'pnpm' | 'yarn' | 'yarn-classic' | 'bun'} DocPackageManager
 */

/** Every package manager a token renders for, in the order docs list them. */
export const DOC_PACKAGE_MANAGERS = /** @type {const} */ ([
  'npm',
  'pnpm',
  'yarn',
  'yarn-classic',
  'bun',
]);

/** The binary each package manager runs as. */
const BINARY = {
  npm: 'npm',
  pnpm: 'pnpm',
  yarn: 'yarn',
  'yarn-classic': 'yarn',
  bun: 'bun',
};

/** The verbs `{@pm}` takes, each portable across every package manager. */
export const PM_VERBS = /** @type {const} */ ([
  'install',
  'remove',
  'update',
  'run',
  'exec',
  'dlx',
  'create',
]);

/** The flags `{@pm}` takes, and the verbs each applies to. */
export const PM_FLAGS = {
  '--dev': ['install'],
  '--exact': ['install'],
  '--global': ['install', 'remove'],
  '--latest': ['update'],
};

/**
 * A token: `{@name body}`, or one written with more `@`s, which shows with one
 * `@` fewer (`{@@pm …}` shows as `{@pm …}`). The body may hold quoted text
 * and balanced braces (`pkg set 'exports={}'`), but never a line break or a
 * backtick: a token sits on one line of one code span or block, and one that
 * does not is an unclosed opening doctor names.
 */
const TOKEN =
  /\{@(@*)([a-z][\w-]*)(?:[ \t]+((?:'[^'\n`]*'|"[^"\n`]*"|\{[^{}\n`]*\}|[^{}'"\n`])*?))?[ \t]*\}/g;

/** The token names this module renders (`{@link}` resolves elsewhere). */
const COMMAND_TOKENS = new Set(['command', 'pm', 'pm-only']);

/** A version range: anything after `@` that is not one exact version. */
const RANGE = /[\^~<>*| ]|(?:^|\.)x(?:\.|$)|^latest$|^next$/i;

/**
 * Split a token body into words, keeping a double-quoted run as one word.
 * @param {string} body
 * @returns {string[]}
 */
export function splitWords(body) {
  const words = [];
  const pattern = /"([^"]*)"|(\S+)/g;
  for (const match of String(body).matchAll(pattern)) {
    words.push(match[1] !== undefined ? `"${match[1]}"` : match[2]);
  }
  return words;
}

/**
 * @typedef {object} PmStep
 * @property {'pm'} kind
 * @property {typeof PM_VERBS[number]} verb
 * @property {string[]} args packages, a script or binary and its arguments
 * @property {Set<string>} flags
 */

/**
 * @typedef {object} PmOnlyStep
 * @property {'pm-only'} kind
 * @property {DocPackageManager} manager
 * @property {string} reason
 * @property {string[]} args the command after the manager's name
 */

/**
 * @typedef {object} CliStep
 * @property {'command'} kind
 * @property {string[]} args the command after `astryx`
 */

/**
 * @typedef {PmStep | PmOnlyStep | CliStep} DocStep
 */

/**
 * Read one token. Returns the step, or `{error}` saying what is wrong and how
 * to write it.
 * @param {string} name the token name, `pm` in `{@pm install x}`
 * @param {string} body what follows the name
 * @returns {DocStep | {error: string}}
 */
export function parseCommandToken(name, body = '') {
  const words = splitWords(body);
  if (name === 'command') {
    if (words.length === 0) {
      return {error: '`{@command}` needs a command: `{@command docs theme}`'};
    }
    if (words[0] === 'astryx') {
      return {
        error: `\`{@command}\` takes the command without \`astryx\`: \`{@command ${words.slice(1).join(' ')}}\``,
      };
    }
    return {kind: 'command', args: words};
  }
  if (name === 'pm-only') {
    const [manager, reason, ...args] = words;
    if (!DOC_PACKAGE_MANAGERS.includes(/** @type {any} */ (manager))) {
      return {
        error: `\`{@pm-only}\` starts with the package manager, one of ${DOC_PACKAGE_MANAGERS.join(', ')}: \`{@pm-only npm "<why only npm>" pkg set name=x}\``,
      };
    }
    if (!reason || !/^".+"$/.test(reason)) {
      return {
        error: `\`{@pm-only ${manager}}\` needs a reason in double quotes after the package manager, which readers see: \`{@pm-only ${manager} "<why only ${manager}>" ${args.join(' ') || '<command>'}}\``,
      };
    }
    if (args.length === 0) {
      return {error: `\`{@pm-only ${manager} ${reason}}\` needs a command`};
    }
    return {
      kind: 'pm-only',
      manager: /** @type {DocPackageManager} */ (manager),
      reason: reason.slice(1, -1),
      args,
    };
  }
  if (name === 'pm') {
    const [verb, ...rest] = words;
    if (!PM_VERBS.includes(/** @type {any} */ (verb))) {
      return {
        error: `\`{@pm}\` starts with a verb, one of ${PM_VERBS.join(', ')}: \`{@pm install @acme/widgets}\``,
      };
    }
    const flags = new Set();
    const args = [];
    for (const word of rest) {
      if (
        word.startsWith('--') &&
        word in PM_FLAGS &&
        verb !== 'run' &&
        verb !== 'exec' &&
        verb !== 'dlx'
      ) {
        const allowed = /** @type {Record<string, readonly string[]>} */ (
          PM_FLAGS
        )[word];
        if (!allowed.includes(verb)) {
          return {
            error: `\`${word}\` applies to ${allowed.map(each => `\`{@pm ${each}}\``).join(' and ')}, not \`{@pm ${verb}}\``,
          };
        }
        flags.add(word);
      } else if (
        word.startsWith('-') &&
        verb !== 'run' &&
        verb !== 'exec' &&
        verb !== 'dlx' &&
        verb !== 'create'
      ) {
        return {
          error: `\`{@pm ${verb}}\` takes the flags ${
            Object.keys(PM_FLAGS)
              .filter(flag =>
                /** @type {Record<string, readonly string[]>} */ (PM_FLAGS)[
                  flag
                ].includes(verb),
              )
              .join(', ') || 'none'
          }, not \`${word}\`, so it reads the same in every package manager`,
        };
      } else {
        args.push(word);
      }
    }
    if (['run', 'exec', 'dlx', 'create'].includes(verb) && args.length === 0) {
      return {
        error: `\`{@pm ${verb}}\` needs a name: \`{@pm ${verb} <name>}\``,
      };
    }
    if (verb === 'remove' && args.length === 0) {
      return {error: '`{@pm remove}` needs a package'};
    }
    if (flags.has('--exact')) {
      const ranged = args.find(arg => {
        const at = arg.lastIndexOf('@');
        return at > 0 && RANGE.test(arg.slice(at + 1));
      });
      if (ranged) {
        return {
          error: `\`--exact\` with a range (\`${ranged}\`) keeps the range in every package manager but npm: write one version, or drop \`--exact\``,
        };
      }
    }
    if (flags.has('--latest') && args.length === 0) {
      return {error: '`{@pm update --latest}` needs the packages to move'};
    }
    return {
      kind: 'pm',
      verb: /** @type {PmStep['verb']} */ (verb),
      args,
      flags,
    };
  }
  return {error: `\`{@${name}}\` is not a doc token`};
}

/**
 * The command a package-manager step runs in one package manager.
 * @param {PmStep} step
 * @param {DocPackageManager} pm
 * @returns {string}
 */
export function renderPmStep(step, pm) {
  const {verb, args, flags} = step;
  const join = (/** @type {Array<string | string[]>} */ ...parts) =>
    parts.flat().filter(Boolean).join(' ');
  const global = flags.has('--global');
  // Yarn 2 and newer has no global install: npm, which every Node carries.
  const manager = global && pm === 'yarn' ? 'npm' : pm;
  switch (verb) {
    case 'install': {
      if (args.length === 0 && !global) return join(BINARY[manager], 'install');
      const dev = flags.has('--dev') ? (manager === 'bun' ? '-d' : '-D') : '';
      const exact = flags.has('--exact')
        ? manager === 'bun'
          ? '--exact'
          : '-E'
        : '';
      if (global) {
        return {
          npm: join('npm install -g', exact, args),
          pnpm: join('pnpm add -g', exact, args),
          yarn: join('npm install -g', exact, args),
          'yarn-classic': join('yarn global add', exact, args),
          bun: join('bun add -g', exact, args),
        }[manager];
      }
      const add = manager === 'npm' ? 'npm install' : `${BINARY[manager]} add`;
      return join(add, dev, exact, args);
    }
    case 'remove': {
      if (global) {
        return {
          npm: join('npm uninstall -g', args),
          pnpm: join('pnpm remove -g', args),
          yarn: join('npm uninstall -g', args),
          'yarn-classic': join('yarn global remove', args),
          bun: join('bun remove -g', args),
        }[manager];
      }
      return join(
        manager === 'npm' ? 'npm uninstall' : `${BINARY[manager]} remove`,
        args,
      );
    }
    case 'update': {
      if (flags.has('--latest')) {
        return {
          npm: join(
            'npm install',
            args.map(arg => `${arg}@latest`),
          ),
          pnpm: join('pnpm update --latest', args),
          yarn: join('yarn up', args),
          'yarn-classic': join('yarn upgrade --latest', args),
          bun: join('bun update --latest', args),
        }[pm];
      }
      return {
        npm: join('npm update', args),
        pnpm: join('pnpm update', args),
        // `yarn up` moves to the newest release; -R stays in the range.
        yarn: join('yarn up -R', args),
        'yarn-classic': join('yarn upgrade', args),
        bun: join('bun update', args),
      }[pm];
    }
    case 'run': {
      const [script, ...rest] = args;
      // npm passes arguments to a script after `--`; pnpm and Yarn 2 pass a
      // `--` on to the script itself, so only npm writes one.
      if (pm === 'npm') {
        return join('npm run', script, rest.length > 0 ? ['--', ...rest] : []);
      }
      return join(BINARY[pm], 'run', script, rest);
    }
    case 'exec':
      return join(
        {
          npm: 'npx',
          pnpm: 'pnpm exec',
          yarn: 'yarn run',
          'yarn-classic': 'yarn run',
          bun: 'bunx',
        }[pm],
        args,
      );
    case 'dlx':
      return join(
        {
          npm: 'npx',
          pnpm: 'pnpm dlx',
          yarn: 'yarn dlx',
          // Yarn 1 has no dlx: npx, which every Node carries.
          'yarn-classic': 'npx',
          bun: 'bunx',
        }[pm],
        args,
      );
    case 'create': {
      // npm passes flags to the initializer only after `--`.
      const flagAt = args.findIndex(arg => arg.startsWith('-'));
      if (pm === 'npm' && flagAt > 0) {
        return join('npm create', [
          ...args.slice(0, flagAt),
          '--',
          ...args.slice(flagAt),
        ]);
      }
      return join(BINARY[pm], 'create', args);
    }
    default:
      return join(BINARY[pm], verb, args);
  }
}

/**
 * @typedef {object} RenderOptions
 * @property {string} stem how the reader runs the CLI, e.g. `pnpm exec astryx`
 * @property {DocPackageManager} pm the reader's package manager
 */

/**
 * One step as the reader runs it, and, for a one-manager step, the line
 * readers see beside it.
 * @param {DocStep} step
 * @param {RenderOptions} options
 * @returns {{command: string, note: string | null}}
 */
export function renderStep(step, {stem, pm}) {
  if (step.kind === 'command') {
    return {command: [stem, ...step.args].join(' '), note: null};
  }
  if (step.kind === 'pm-only') {
    return {
      command: [BINARY[step.manager], ...step.args].join(' '),
      note: `only with ${managerName(step.manager)}: ${step.reason}`,
    };
  }
  return {command: renderPmStep(step, pm), note: null};
}

/**
 * A package manager's name as readers read it.
 * @param {DocPackageManager} pm
 * @returns {string}
 */
export function managerName(pm) {
  return {
    npm: 'npm',
    pnpm: 'pnpm',
    yarn: 'Yarn',
    'yarn-classic': 'Yarn 1',
    bun: 'Bun',
  }[pm];
}

/** Code-fence languages whose lines are shell commands. */
const SHELL_LANGS = new Set(['', 'bash', 'sh', 'shell', 'zsh', 'console']);

/**
 * Text with each `{@command}`, `{@pm}` and `{@pm-only}` token rendered for the
 * reader, and each `{@@…}` shown as `{@…}`. A token in prose prints as a code
 * span; in a code span or code block, as the bare command. A one-manager
 * step's reason follows it: in prose right after the command, after a code
 * span, in parentheses; in a shell block as a comment. A Markdown fence in
 * prose is a code block in its language. A token that does not parse prints
 * as written, and doctor names it.
 * @param {string} text
 * @param {RenderOptions} options
 * @param {string | null} [lang] the code block's language when `text` is one
 *   block's body; omit for prose
 * @returns {string}
 */
export function renderCommandTokens(text, options, lang = null) {
  const source = String(text);
  if (!source.includes('{@')) return source;
  const inBlock = lang != null;
  const shell = inBlock && SHELL_LANGS.has(lang.toLowerCase());
  /**
   * @param {string} part
   * @param {string[]} notes where a one-manager step's reason goes
   * @param {boolean} wrap print a rendered command as a code span
   */
  const render = (part, notes, wrap) =>
    part.replace(TOKEN, (whole, escaped, name, body = '') => {
      if (escaped) return whole.replace('{@@', '{@');
      if (!COMMAND_TOKENS.has(name)) return whole;
      const step = parseCommandToken(name, body);
      if ('error' in step) return whole;
      const {command, note} = renderStep(step, options);
      // In prose the reason follows its own command, inside the sentence.
      if (note != null && wrap) return `\`${command}\` (${note})`;
      if (note != null) notes.push(note);
      return wrap ? `\`${command}\`` : command;
    });
  if (inBlock) {
    return source
      .split('\n')
      .map(line => {
        /** @type {string[]} */
        const notes = [];
        const out = render(line, notes, false);
        if (notes.length === 0) return out;
        return shell
          ? `${out}  # ${notes.join('; ')}`
          : `${out} (${notes.join('; ')})`;
      })
      .join('\n');
  }
  /** @param {string} line */
  const prose = line =>
    line
      .split(/(`[^`\n]*`)/)
      .map(part => {
        /** @type {string[]} */
        const notes = [];
        const isSpan =
          part.length > 1 && part.startsWith('`') && part.endsWith('`');
        const out = render(part, notes, !isSpan);
        return notes.length === 0 ? out : `${out} (${notes.join('; ')})`;
      })
      .join('');
  // A fence inside prose is a code block: its lines render as one, in the
  // fence's language.
  /** @type {{marker: string, lang: string} | null} */
  let fence = null;
  return source
    .split('\n')
    .map(line => {
      const marker = /^\s*(`{3,}|~{3,})\s*([\w-]*)/.exec(line);
      if (marker) {
        if (fence == null) {
          fence = {marker: marker[1], lang: marker[2].toLowerCase()};
        } else if (marker[1].startsWith(fence.marker)) {
          fence = null;
        }
        return line;
      }
      return fence == null
        ? prose(line)
        : renderCommandTokens(line, options, fence.lang);
    })
    .join('\n');
}

/**
 * Every command token in a text, with its parse, for doctor.
 * @param {string} text
 * @returns {Array<{token: string, name: string, step: DocStep | {error: string}}>}
 */
export function commandTokensIn(text) {
  const found = [];
  for (const match of String(text).matchAll(TOKEN)) {
    if (match[1] || !COMMAND_TOKENS.has(match[2])) continue;
    found.push({
      token: match[0],
      name: match[2],
      step: parseCommandToken(match[2], match[3] ?? ''),
    });
  }
  return found;
}

/** The token names a doc may write: these and `{@link}`. */
export const DOC_TOKEN_NAMES = new Set([...COMMAND_TOKENS, 'link']);

/**
 * Every `{@name …}` whose name is not a doc token, for doctor.
 * @param {string} text
 * @returns {string[]}
 */
export function unknownTokensIn(text) {
  const found = [];
  for (const match of String(text).matchAll(TOKEN)) {
    if (!match[1] && !DOC_TOKEN_NAMES.has(match[2])) found.push(match[0]);
  }
  return found;
}

/**
 * Each `{@command`, `{@pm` or `{@pm-only` opening (escaped with more `@`s
 * too) that no whole token starts at: one with no closing brace, or with a
 * quote it never closes. It prints as written, so doctor names it.
 * @param {string} text
 * @returns {string[]} the opening and the rest of its line
 */
export function unclosedTokensIn(text) {
  const source = String(text);
  const starts = new Set(
    [...source.matchAll(TOKEN)].map(
      match => /** @type {number} */ (match.index),
    ),
  );
  const found = [];
  for (const match of source.matchAll(/\{@@*(?:command|pm-only|pm)\b/g)) {
    const at = /** @type {number} */ (match.index);
    if (starts.has(at)) continue;
    const end = source.indexOf('\n', at);
    found.push(source.slice(at, end < 0 ? undefined : end).slice(0, 80));
  }
  return found;
}

/** Package-manager binaries a plain-text command may start with. */
const PM_BINARIES = new Set(['npm', 'npx', 'pnpm', 'yarn', 'bun', 'bunx']);

/**
 * The token a plain-text package-manager command should be written as, for
 * doctor's fix: `{@pm …}` when every package manager can run it, `{@pm-only
 * <manager> "<why>" …}` when only that one can, or null when the text is not
 * a package-manager command. A runner in front of `astryx` is an Astryx
 * command: `{@command …}`.
 * @param {string} command one command, such as `npm install -D @acme/x`
 * @returns {string | null}
 */
export function tokenForCommand(command) {
  const words = String(command).trim().split(/\s+/);
  const [bin, ...rest] = words;
  if (bin === 'astryx')
    return `{@command ${rest.join(' ')}}`.replace(' }', '}');
  if (!PM_BINARIES.has(bin)) return null;
  // A runner in front of astryx.
  const astryxAt = rest.indexOf('astryx');
  const runnerWords =
    bin === 'npx' || bin === 'bunx'
      ? 0
      : bin === 'pnpm' && (rest[0] === 'exec' || rest[0] === 'dlx')
        ? 1
        : bin === 'yarn' &&
            (rest[0] === 'run' || rest[0] === 'dlx' || rest[0] === 'exec')
          ? 1
          : bin === 'npm' && (rest[0] === 'exec' || rest[0] === 'x')
            ? 1
            : bin === 'bun' && rest[0] === 'x'
              ? 1
              : 0;
  if (
    astryxAt === runnerWords ||
    (astryxAt >= 0 &&
      rest.slice(runnerWords, astryxAt).every(w => w.startsWith('-')))
  ) {
    const after = rest.slice(astryxAt + 1).filter(w => w !== '--');
    return after.length > 0 ? `{@command ${after.join(' ')}}` : null;
  }
  if (bin === 'npm' && rest[0] === 'run' && rest[1] === 'astryx') {
    const after = rest.slice(2).filter(w => w !== '--');
    return after.length > 0 ? `{@command ${after.join(' ')}}` : null;
  }
  const flags = [];
  const args = [];
  const pmOnly = (/** @type {string} */ manager) =>
    `{@pm-only ${manager} "<why only ${manager}>" ${rest.join(' ')}}`;
  if (bin === 'npx' || bin === 'bunx') {
    const plain = rest.filter(w => w !== '-y' && w !== '--yes');
    // npx runs the project's own binary when it has one: an unscoped,
    // unversioned name such as `tsc` is that, so it is `exec`. A scoped or
    // versioned package, or `--yes`, downloads one: `dlx`.
    const target = plain.find(w => !w.startsWith('-')) ?? '';
    const downloads =
      plain.length < rest.length ||
      target.startsWith('@') ||
      /^[^@]+@/.test(target);
    return `{@pm ${downloads ? 'dlx' : 'exec'} ${plain.join(' ')}}`;
  }
  let verb = rest[0];
  let tail = rest.slice(1);
  if (bin === 'yarn' && verb === 'global') {
    if (tail[0] === 'add' || tail[0] === 'remove') {
      flags.push('--global');
      verb = tail[0];
      tail = tail.slice(1);
    } else {
      return pmOnly('yarn-classic');
    }
  }
  if (verb === undefined) return '{@pm install}';
  const VERBS = {
    install: 'install',
    i: 'install',
    add: 'install',
    uninstall: 'remove',
    un: 'remove',
    rm: 'remove',
    remove: 'remove',
    update: 'update',
    up: 'update',
    upgrade: 'update',
    run: 'run',
    'run-script': 'run',
    exec: 'exec',
    dlx: 'dlx',
    create: 'create',
    x: 'exec',
  };
  const mapped = /** @type {Record<string, string>} */ (VERBS)[verb];
  // npm's script shortcuts run a script every package manager can run.
  if (
    !mapped &&
    bin === 'npm' &&
    ['test', 'start', 'stop', 'restart'].includes(verb)
  ) {
    // The token puts npm's `--` back itself, as for `run`.
    return `{@pm run ${[verb, ...tail.filter(word => word !== '--')].join(' ')}}`;
  }
  if (!mapped) {
    // `pnpm build`, `yarn test`: a script run without `run`.
    if (
      bin !== 'npm' &&
      bin !== 'bun' &&
      /^[\w:.-]+$/.test(verb) &&
      !NPM_TOOLS.has(verb)
    ) {
      return `{@pm run ${[verb, ...tail].join(' ')}}`;
    }
    return pmOnly(bin === 'yarn' ? 'yarn' : /** @type {string} */ (bin));
  }
  // A script, a binary or an initializer takes every word after it as its
  // own, flags included; the token puts back any `--` npm needs.
  const passesThrough = ['run', 'exec', 'dlx', 'create'].includes(mapped);
  for (const word of tail) {
    if (passesThrough) {
      if (word !== '--') args.push(word);
      continue;
    }
    if (['-D', '-d', '--save-dev', '--dev', '-DE', '-ED'].includes(word))
      flags.push('--dev');
    if (['-E', '--exact', '--save-exact', '-DE', '-ED'].includes(word))
      flags.push('--exact');
    if (['-g', '--global'].includes(word)) flags.push('--global');
    if (word === '--latest' || word === '-L') flags.push('--latest');
    if (!word.startsWith('-')) args.push(word);
  }
  const unique = [...new Set(flags)];
  // A flag no other package manager shares: one manager only.
  const known = new Set([
    '-D',
    '-d',
    '--save-dev',
    '--dev',
    '-E',
    '--exact',
    '--save-exact',
    '-g',
    '--global',
    '--latest',
    '-L',
    '-DE',
    '-ED',
    '--',
  ]);
  if (
    mapped !== 'run' &&
    mapped !== 'exec' &&
    mapped !== 'dlx' &&
    mapped !== 'create' &&
    tail.some(w => w.startsWith('-') && !known.has(w))
  ) {
    return pmOnly(bin === 'yarn' ? 'yarn' : /** @type {string} */ (bin));
  }
  const token = `{@pm ${[mapped, ...args, ...unique].join(' ')}}`;
  const parsed = parseCommandToken(
    'pm',
    [mapped, ...args, ...unique].join(' '),
  );
  return 'error' in parsed
    ? pmOnly(bin === 'yarn' ? 'yarn' : /** @type {string} */ (bin))
    : token;
}

/** Commands only npm itself runs: `npm pkg`, `npm pack` and the rest. */
const NPM_TOOLS = new Set([
  'pkg',
  'pack',
  'publish',
  'version',
  'view',
  'info',
  'config',
  'prefix',
  'setup',
  'link',
  'why',
  'ls',
  'list',
  'outdated',
  'cache',
  'init',
  'dist-tag',
  'login',
  'whoami',
  'audit',
  'ci',
  'store',
  'global',
  'dedupe',
]);