Skip to content

armgit / dbx

Public Imported from GitHub
0

25 MB lightweight cross-platform database client for 100+ databases, including MySQL, PostgreSQL, SQLite, Redis, MongoDB, DuckDB, SQL Server, and Dameng. Built-in AI, MCP Server, CLI, desktop and Docker. | 轻量级跨平台数据库管理工具,支持 MySQL、PostgreSQL、SQLite、Redis、MongoDB、达梦等 100+ 数据库,提供桌面端、Docker、CLI、内置 AI 助手和 MCP。

8.5 KB
Raw Blame
import fs from "node:fs/promises";
import path from "node:path";

const DEFAULT_INPUT = "docs/content/docs";
const DEFAULT_OUTPUT = "outputs/knowledge-base/DBX_knowledge_base_cn.md";
const DEFAULT_LANG = "cn";

await main();

async function main() {
  const args = parseArgs(process.argv.slice(2));
  const inputDir = path.resolve(args.input ?? DEFAULT_INPUT);
  const outputFile = path.resolve(args.output ?? DEFAULT_OUTPUT);
  const lang = args.lang ?? DEFAULT_LANG;
  const fileSuffix = lang ? `.${lang}.mdx` : ".mdx";

  const files = (await fs.readdir(inputDir)).filter((name) => name.endsWith(fileSuffix)).sort((a, b) => a.localeCompare(b));

  if (files.length === 0) {
    throw new Error(`No ${fileSuffix} files found in ${inputDir}`);
  }

  const documents = [];
  let entryCount = 0;

  for (const file of files) {
    const sourcePath = path.join(inputDir, file);
    const displayPath = path.relative(process.cwd(), sourcePath).replaceAll(path.sep, "/");
    const raw = await fs.readFile(sourcePath, "utf8");
    const { fields, body } = parseFrontmatter(raw);
    const title = fields.title || file.replace(/\.mdx$/, "");
    const description = fields.description || "";
    const sections = [];

    for (const section of splitSections(body)) {
      const clean = normalizeMdx(section.lines.join("\n"));
      if (!clean || clean.length < 20) continue;

      const chunks = chunkText(clean).map((chunk, index, all) => {
        const heading = all.length > 1 ? `${section.heading}(${index + 1}/${all.length})` : section.heading;
        return {
          heading,
          question: questionFor(title, section.heading, chunk),
          keywords: keywordsFor(title, section.heading, description, chunk),
          answer: chunk,
        };
      });

      entryCount += chunks.length;
      sections.push(...chunks);
    }

    documents.push({ source: displayPath, title, description, sections });
  }

  await fs.mkdir(path.dirname(outputFile), { recursive: true });
  await fs.writeFile(outputFile, renderMarkdown(documents), "utf8");

  console.log(`Knowledge base written: ${path.relative(process.cwd(), outputFile)}`);
  console.log(`Sources: ${files.length}`);
  console.log(`Entries: ${entryCount}`);
}

function parseArgs(argv) {
  const parsed = {};
  for (let index = 0; index < argv.length; index += 1) {
    const arg = argv[index];
    if (arg === "--help" || arg === "-h") {
      printHelp();
      process.exit(0);
    }
    if (!arg.startsWith("--")) {
      throw new Error(`Unexpected argument: ${arg}`);
    }
    const key = arg.slice(2);
    const value = argv[index + 1];
    if (!value || value.startsWith("--")) {
      throw new Error(`Missing value for ${arg}`);
    }
    parsed[key] = value;
    index += 1;
  }
  return parsed;
}

function printHelp() {
  console.log(`Usage: node scripts/build-knowledge-base.mjs [options]

Options:
  --input <dir>      MDX docs directory. Default: ${DEFAULT_INPUT}
  --output <file>    Output Markdown path. Default: ${DEFAULT_OUTPUT}
  --lang <suffix>    File language suffix, e.g. cn for *.cn.mdx. Default: ${DEFAULT_LANG}
`);
}

function parseFrontmatter(source) {
  const match = source.match(/^---\n([\s\S]*?)\n---\n?/);
  const fields = {};
  if (!match) return { fields, body: source };

  for (const line of match[1].split("\n")) {
    const separator = line.indexOf(":");
    if (separator === -1) continue;
    const key = line.slice(0, separator).trim();
    const value = line
      .slice(separator + 1)
      .trim()
      .replace(/^["']|["']$/g, "");
    fields[key] = value;
  }

  return { fields, body: source.slice(match[0].length) };
}

function splitSections(body) {
  const sections = [];
  let current = { heading: "概览", lines: [] };

  for (const line of body.split("\n")) {
    const heading = line.match(/^(#{2,3})\s+(.+?)\s*$/);
    if (heading) {
      if (current.lines.join("\n").trim()) sections.push(current);
      current = { heading: heading[2].replace(/`/g, "").trim(), lines: [] };
      continue;
    }
    current.lines.push(line);
  }

  if (current.lines.join("\n").trim()) sections.push(current);
  return sections;
}

function normalizeMdx(text) {
  return text
    .replace(/<Card\s+([^>]*)>([\s\S]*?)<\/Card>/g, (_, attrs, inner) => {
      const title = attrs.match(/title=["']([^"']+)["']/)?.[1] ?? "";
      const href = attrs.match(/href=["']([^"']+)["']/)?.[1] ?? "";
      return [title && `**${title}**`, normalizeMdx(inner), href && `参考链接:${href}`].filter(Boolean).join("\n\n");
    })
    .replace(/<Callout[^>]*>([\s\S]*?)<\/Callout>/g, (_, inner) => normalizeMdx(inner))
    .replace(/<Tab[^>]*>([\s\S]*?)<\/Tab>/g, (_, inner) => normalizeMdx(inner))
    .replace(/<Step[^>]*>([\s\S]*?)<\/Step>/g, (_, inner) => normalizeMdx(inner))
    .replace(/<[^>\n]+>/g, "")
    .replace(/\{\/\*[\s\S]*?\*\/\}/g, "")
    .replace(/\[([^\]]+)\]\(([^)]+)\)/g, "$1($2)")
    .replace(/^\s{0,6}#{1,6}\s+(.+)$/gm, "$1")
    .replace(/^\s*import\s+.*$/gm, "")
    .replace(/^\s*export\s+.*$/gm, "")
    .replace(/[ \t]+\n/g, "\n")
    .replace(/\n{3,}/g, "\n\n")
    .trim();
}

function chunkText(text, maxLength = 2200) {
  if (text.length <= maxLength) return [text];

  const chunks = [];
  let current = "";
  for (const paragraph of text.split(/\n{2,}/)) {
    const next = [current, paragraph].filter(Boolean).join("\n\n");
    if (next.length > maxLength && current) {
      chunks.push(current);
      current = paragraph;
    } else {
      current = next;
    }
  }
  if (current) chunks.push(current);

  return chunks.flatMap((chunk) => {
    if (chunk.length <= maxLength) return [chunk];
    const pieces = [];
    for (let index = 0; index < chunk.length; index += maxLength) {
      pieces.push(chunk.slice(index, index + maxLength));
    }
    return pieces;
  });
}

function questionFor(category, heading, text) {
  if (heading === "概览") return `DBX 的${category}是什么?`;
  if (/安装|配置|导入|导出|创建|连接|使用流程|快速开始|启动|构建|下载|打开|执行|添加|删除|更新/.test(heading)) {
    return `DBX 中如何${heading.replace(/^(第一步|第二步|第三步|第四步|第五步):/, "")}?`;
  }
  if (/常见|问题|故障|错误|诊断|冲突|无法/.test(heading) || /失败|报错|检查/.test(text)) {
    return `DBX ${heading}时应该检查什么?`;
  }
  if (/支持|覆盖|矩阵|类型|数据库|文件|供应商|快捷键|命令|端口/.test(heading)) {
    return `DBX ${heading}支持哪些内容?`;
  }
  return `DBX 的${heading}是什么?`;
}

function keywordsFor(category, heading, description, text) {
  const pool = `${category} ${heading} ${description} ${text}`;
  const candidates = [
    "DBX",
    "AI",
    "SQL",
    "MCP",
    "JDBC",
    "SSH",
    "Docker",
    "CLI",
    "MongoDB",
    "Redis",
    "MySQL",
    "PostgreSQL",
    "SQLite",
    "DuckDB",
    "ClickHouse",
    "Oracle",
    "SQL Server",
    "Schema",
    "ER",
    "导入",
    "导出",
    "连接",
    "表",
    "字段",
    "查询",
    "快捷键",
    "插件",
    "驱动",
    "配置",
    "安全",
    "数据传输",
    "格式化",
    "代码片段",
    "隧道",
    "生产环境",
  ];
  const found = candidates.filter((word) => pool.includes(word));
  return [...new Set([category, heading, ...found])].slice(0, 14).join(", ");
}

function renderMarkdown(documents) {
  const lines = ["# DBX 中文知识库", "", "> 从 `docs/content/docs/*.cn.mdx` 生成的纯 Markdown 知识库内容。", "", "## 来源索引", "", "| 来源文件 | 标题 | 描述 | 知识条数 |", "| --- | --- | --- | ---: |"];

  for (const document of documents) {
    lines.push(`| ${escapeTableCell(document.source)} | ${escapeTableCell(document.title)} | ${escapeTableCell(document.description)} | ${document.sections.length} |`);
  }

  for (const document of documents) {
    lines.push("", `## ${document.title}`, "");
    if (document.description) lines.push(document.description, "");
    lines.push(`来源:\`${document.source}\``, "");

    for (const section of document.sections) {
      lines.push(`### ${section.heading}`, "");
      lines.push(`问题:${section.question}`, "");
      lines.push(`关键词:${section.keywords}`, "");
      lines.push("答案:", "");
      lines.push(section.answer, "");
      lines.push("---", "");
    }
  }

  return `${lines
    .join("\n")
    .replace(/\n{4,}/g, "\n\n\n")
    .trim()}\n`;
}

function escapeTableCell(value) {
  return String(value ?? "")
    .replaceAll("\\", "\\\\")
    .replaceAll("|", "\\|")
    .replace(/\s+/g, " ")
    .trim();
}