The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Add GitHub actions to generate llms.txt and llms-embedding-full.txt at repo root during release (#67737)
Phoomparin Mano65d34e4e372e
diff --git a/.github/scripts/generate-llms-txt.js b/.github/scripts/generate-llms-txt.jsnew file mode 100644index 00000000000..5da1f97594f--- /dev/null+++ b/.github/scripts/generate-llms-txt.js@@ -0,0 +1,317 @@+const fs = require("fs");+const path = require("path");++const REPO = "metabase/metabase";+const OUTPUT_FILE = "llms.txt";++// Sections to generate llms-<name>-full.txt for.+// Each section can define a preamble (introductory content before the main docs).+const LLMS_FULL_TO_GENERATE = [+ {+ section: "embedding",+ getPreamble: (branch) =>+ aboveVersion(branch, 57) ? getModularEmbeddingGotchaNotes() : "",+ },+];++/**+ * List directory recursively to get markdown files+ *+ * @param {string} docsPath - full path to docs directory+ * @returns {Array} list of markdown file objects with fullPath and relativePath+ */+function getMarkdownFilesList(docsPath) {+ if (!fs.existsSync(docsPath)) {+ console.error(`Error: Docs directory '${docsPath}' does not exist`);+ process.exit(1);+ }++ const entries = fs.readdirSync(docsPath, {+ recursive: true,+ withFileTypes: true,+ });++ return entries+ .filter((entry) => {+ return (+ entry.isFile() &&+ // Only markdown files+ entry.name.endsWith(".md") &&+ // Ignore README.md+ entry.name !== "README.md" &&+ // Ignore node_modules and hidden directories+ !entry.parentPath+ .split(path.sep)+ .some((dir) => dir === "node_modules" || dir.startsWith("."))+ );+ })+ .map((entry) => {+ const fullPath = path+ .join(entry.parentPath, entry.name)+ .replace(/\\/g, "/");++ const relativePath = path+ .relative(docsPath, fullPath)+ .replace(/\\/g, "/");++ return { fullPath, relativePath };+ });+}++/**+ * Generate llms.txt file with links to all markdown files in docs/+ *+ * @param {string} branch - branch name (e.g. 'master' or 'release-x.58.x')+ * @param {Array} files - list of all markdown file objects+ * @param {Array<Object>} sectionsConfig - list of section config objects to generate complete references for+ */+function generateLlmsTxt(branch, files, sectionsConfig) {+ const baseUrl = `https://raw.githubusercontent.com/${REPO}/refs/heads/${branch}`;++ const filesToExtract = files+ // Files in embedding/sdk/api/snippets does not exist in the repo.+ .filter(+ (file) => !file.relativePath.startsWith("embedding/sdk/api/snippets"),+ );++ const entries = filesToExtract.map((file) => {+ const content = fs.readFileSync(file.fullPath, "utf8");+ const title = extractTitle(content, file.relativePath);+ const url = `${baseUrl}/docs/${file.relativePath}`;++ return { title, url, path: file.relativePath };+ });++ // Generate table of contents entries+ const tableOfContents = entries.map(+ (entry) => `- [${entry.title}](${entry.url})`,+ );++ // Generate complete references entries+ const completeReferences = sectionsConfig+ .filter((config) =>+ files.some((file) => file.relativePath.startsWith(`${config.section}/`)),+ )+ .map((config) => {+ const capitalized =+ config.section.charAt(0).toUpperCase() + config.section.slice(1);+ return `- [${capitalized} - Complete Reference](${baseUrl}/llms-${config.section}-full.txt)`;+ });++ // Generate llms.txt content using template literal+ const content = `# Metabase Documentation+Metabase - The simplest, fastest way to get business intelligence and analytics to everyone in your company.++## Table of Contents++${tableOfContents.join("\n")}++## Complete References++These files are very large and are around 90,000 tokens. Do not use by default unless the context window is huge or RAG is supported in your editor.++${completeReferences.join("\n")}+`;++ fs.writeFileSync(OUTPUT_FILE, content, "utf8");++ console.log(+ `✅ Generated ${OUTPUT_FILE} with ${entries.length} markdown files`,+ );+ console.log(` Branch: ${branch}`);+ console.log(` Base URL: ${baseUrl}`);+}++/**+ * Generate llms-full.txt files for specified sections+ *+ * @param {string} branch - branch name (e.g., 'master', 'release-x.58.x')+ * @param {Array} files - list of all markdown file objects+ * @param {Array<Object>} sectionsConfig - list of section config objects to generate full files for+ */+function generateLlmsFullFiles(branch, files, sectionsConfig) {+ const baseUrl = `https://raw.githubusercontent.com/${REPO}/refs/heads/${branch}`;++ sectionsConfig.forEach((config) => {+ generateLlmsFullTxt(branch, config, files, baseUrl);+ });+}++/**+ * LLMs are likely to pay attention to the very first lines.+ *+ * We add the most important context for LLMs to avoid+ * confusion and pitfalls like out-of-date APIs in trained data.+ *+ * @returns {string} Markdown formatted gotcha notes+ */+function getModularEmbeddingGotchaNotes() {+ return `> **Important Version Notes**+>+> Watch out for these deprecated props and gotchas for Metabase 57 onwards, for modular embedding:+>+> 1. \`config\` prop on MetabaseProvider no longer exist - it is replaced by \`authConfig\`.+> 2. \`authProviderUri\` field no longer exist.+> 3. \`jwtProviderUri\` optional field only exists in v58+. This is used to make JWT auth faster by skipping the \`GET /auth/sso\` discovery request. Not needed for initial implementation.+> 4. \`fetchRequestToken\` is not needed by default. This is only used to customize how the SDK fetches the request token.+> 5. Numeric IDs must be integers not strings, e.g. \`dashboardId={1}\`. When the ID is retrieved from the URL and it is numeric, convert it to an integer via \`parseInt\` before passing it to the SDK. IDs can also be strings for entity ids, so you should not parse all IDs as numbers if entity ids are also to be expected.+`;+}++/**+ * Generate llms-full.txt file for a specific section+ * @param {string} branch - The branch name+ * @param {Object} config - Section config object with section name and optional getPreamble function+ * @param {Array} files - Array of all markdown file objects+ * @param {string} baseUrl - Base URL for raw GitHub links+ */+function generateLlmsFullTxt(branch, config, files, baseUrl) {+ const { section, getPreamble } = config;++ // Filter docs for this section+ const sectionFiles = files.filter((file) =>+ file.relativePath.startsWith(`${section}/`),+ );++ if (sectionFiles.length === 0) {+ console.log(` Skipping llms-full.txt for ${section} (no files found)`);+ return;+ }++ // Build content+ const sectionCapitalized = section.charAt(0).toUpperCase() + section.slice(1);+ const header = `# Metabase ${sectionCapitalized} - Complete Reference for AI agents++> Table of contents: ${baseUrl}/${OUTPUT_FILE}++`;++ // Add preamble if provided (e.g., version-specific notes, warnings)+ const preamble = getPreamble ? getPreamble(branch) : "";++ // Concatenate all documents+ const docs = sectionFiles+ .map((file) => readAndCleanMarkdown(file.fullPath))+ .join("\n");++ // Write llms-full.txt file at repo root+ const content = header + preamble + "\n" + docs;+ const llmsFullPath = `llms-${section}-full.txt`;+ fs.writeFileSync(llmsFullPath, content, "utf8");++ console.log(+ `✅ Generated ${llmsFullPath} with ${sectionFiles.length} documents`,+ );+}++/**+ * Extract title from markdown file frontmatter or filename.+ *+ * We try to extract the title in this order: 1) YAML frontmatter+ * 2) first H1 heading, and 3) the Markdown file name.+ *+ * Example Markdown and frontmatter format:+ * ```markdown+ * ---+ * title: "Getting Started"+ * description: "A guide"+ * ---+ * # Getting Started+ * ```+ *+ * @param {string} content - file content+ * @param {string} filePath - relative file path+ * @returns {string} title of content+ */+function extractTitle(content, filePath) {+ // Try to extract title from YAML frontmatter+ // First, match against the YAML frontmatter delimiter `---`+ const frontmatterMatch = content.match(/^---\s*\n([\s\S]*?)\n---\s*\n/);++ if (frontmatterMatch) {+ // Then, match against the title property+ // Example: `title: "My Page"`+ const frontmatter = frontmatterMatch[1];+ const titleMatch = frontmatter.match(/^title:\s*(.+)$/m);++ // Remove quotes if present+ if (titleMatch) {+ return titleMatch[1].trim().replace(/^["']|["']$/g, "");+ }+ }++ // Try to extract title from first H1 heading+ // Example: `# My Page`+ const h1Match = content.match(/^#\s+(.+)$/m);++ if (h1Match) {+ return h1Match[1].trim();+ }++ // Fallback to filename, e.g. `my-page.md`+ const filename = path.basename(filePath, ".md");++ // Convert kebab-case or snake_case to Title Case+ // `my-page.md` would become "My Page"+ return filename+ .split(/[-_]/)+ .map((word) => word.charAt(0).toUpperCase() + word.slice(1))+ .join(" ");+}++/**+ * Read and clean markdown file content+ *+ * @param {string} filePath - full path to the markdown file+ * @returns {string} cleaned markdown content with separator+ */+function readAndCleanMarkdown(filePath) {+ let content = fs.readFileSync(filePath, "utf8");++ // Strip YAML frontmatter+ content = content.replace(/^---\s*\n[\s\S]*?\n---\s*\n/, "");++ // Strip Jekyll/Liquid template syntax+ content = content.replace(/\{%.*?%\}/g, "");+ content = content.replace(/\{\{.*?\}\}/g, "");++ return `${content.trim()}\n\n---\n`;+}++/**+ * Check if branch version is above target version+ * @param {string} branch - Branch name (e.g., 'master', 'release-x.58.x')+ * @param {number} targetVersion - Target version number+ * @returns {boolean}+ */+function aboveVersion(branch, targetVersion) {+ if (branch === "master") {+ return true;+ }++ // Parse version from release branch like "release-x.58.x" -> 58+ const match = branch.match(/release-x\.(\d+)\.x/);+ if (!match) {+ return false;+ }++ const version = parseInt(match[1], 10);++ return version >= targetVersion;+}++// Main execution+if (require.main === module) {+ const args = process.argv.slice(2);+ const docsPath = args[0] || "docs";+ const branch = args[1] || "master";++ const resolvedDocsPath = path.resolve(docsPath);+ const markdownFiles = getMarkdownFilesList(resolvedDocsPath);++ // Sort all files upfront for consistent ordering across all generated files+ markdownFiles.sort((a, b) => a.relativePath.localeCompare(b.relativePath));++ generateLlmsTxt(branch, markdownFiles, LLMS_FULL_TO_GENERATE);+ generateLlmsFullFiles(branch, markdownFiles, LLMS_FULL_TO_GENERATE);+}diff --git a/.github/workflows/llms-txt-generate.yml b/.github/workflows/llms-txt-generate.ymlnew file mode 100644index 00000000000..23387815fd5--- /dev/null+++ b/.github/workflows/llms-txt-generate.yml@@ -0,0 +1,193 @@+name: Generate llms.txt file++on:+ workflow_call:+ inputs:+ version:+ description: 'Metabase version (e.g. v0.57.0)'+ type: string+ required: true+ branch:+ description: 'Target branch (e.g. release-x.57.x)'+ type: string+ required: true+ workflow_dispatch:+ inputs:+ version:+ description: 'Metabase version (e.g. v0.57.0)'+ type: string+ required: true+ branch:+ description: 'Target branch (e.g. release-x.57.x)'+ type: string+ required: true++permissions:+ contents: write+ pull-requests: write++jobs:+ check-version:+ name: Check if minor or major release+ runs-on: ubuntu-22.04+ timeout-minutes: 5+ outputs:+ should_generate: ${{ steps.check.outputs.should_generate }}+ target_branch: ${{ steps.check.outputs.target_branch }}+ version: ${{ steps.check.outputs.version }}+ steps:+ - name: Check version+ id: check+ run: |+ VERSION="${{ inputs.version }}"+ BRANCH="${{ inputs.branch }}"++ # Extract version number (e.g., v0.57.0 -> 0.57.0)+ VERSION_NUM="${VERSION#v}"++ # Extract patch version (last number)+ PATCH_VERSION=$(echo "$VERSION_NUM" | cut -d. -f3)++ # Only proceed if this is a minor or major release (patch version is 0)+ # Note: This is a safety check even though the workflow should only be called+ # for minor/major releases, just in case the workflow is dispatched manually.+ if [ "$PATCH_VERSION" != "0" ]; then+ echo "⏭️ This is a patch release ($VERSION). Skipping llms.txt generation."+ echo "should_generate=false" >> $GITHUB_OUTPUT+ else+ echo "✅ This is a minor or major release ($VERSION). Proceeding with llms.txt generation for branch: $BRANCH"+ echo "should_generate=true" >> $GITHUB_OUTPUT+ fi++ echo "target_branch=$BRANCH" >> $GITHUB_OUTPUT+ echo "version=$VERSION" >> $GITHUB_OUTPUT++ generate-llms-txt:+ name: Generate llms.txt+ needs: check-version+ if: needs.check-version.outputs.should_generate == 'true'+ runs-on: ubuntu-22.04+ timeout-minutes: 15+ steps:++ - name: Get the generate-llms-txt.js node script via sparse checkout+ uses: actions/checkout@v4+ with:+ token: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}+ ref: ${{ github.ref }}+ sparse-checkout: |+ .github/scripts+ path: workflow-scripts++ - name: Checkout target branch for docs+ uses: actions/checkout@v4+ with:+ token: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}+ ref: ${{ needs.check-version.outputs.target_branch }}+ path: metabase++ - name: Setup Node.js+ uses: actions/setup-node@v4+ with:+ node-version: 22++ - name: Generate llms.txt+ id: generate+ run: |+ node workflow-scripts/.github/scripts/generate-llms-txt.js metabase/docs ${{ needs.check-version.outputs.target_branch }}++ - name: Sanity check generated files+ run: |+ if [ ! -f "llms.txt" ]; then+ echo "❌ Error: llms.txt was not generated"+ exit 1+ fi++ if [ ! -f "llms-embedding-full.txt" ]; then+ echo "❌ Error: llms-embedding-full.txt was not generated"+ exit 1+ fi++ # Check that llms.txt contains URLs with the expected GitHub raw URL prefix+ URL_COUNT=$(grep -o "https://raw.githubusercontent.com/metabase/metabase/refs/heads/${{ needs.check-version.outputs.target_branch }}/docs" llms.txt | wc -l)+ if [ "$URL_COUNT" -lt 10 ]; then+ echo "❌ Error: llms.txt appears to be malformed. Found only $URL_COUNT URLs (expected at least 10)"+ exit 1+ fi++ echo "✅ Both llms.txt and llms-embedding-full.txt were generated successfully"+ echo "✅ llms.txt contains $URL_COUNT documentation URLs"++ - name: Set up git configuration+ working-directory: metabase+ run: |+ git config user.name "github-actions[bot]"+ git config user.email "41898282+github-actions[bot]@users.noreply.github.com"+ BRANCH_NAME="llms-txt/auto-update-${{ needs.check-version.outputs.target_branch }}-$(date +%Y%m%d-%H%M%S)"+ echo "BRANCH_NAME=$BRANCH_NAME" >> $GITHUB_ENV+ echo "Branch name will be: $BRANCH_NAME"++ - name: Copy generated files to metabase repo+ run: |+ cp llms.txt metabase/+ cp llms-embedding-full.txt metabase/++ - name: Create Pull Request+ id: create_pr+ uses: peter-evans/create-pull-request@v6+ with:+ token: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}+ path: metabase+ branch: ${{ env.BRANCH_NAME }}+ base: ${{ needs.check-version.outputs.target_branch }}+ commit-message: |+ Update llms.txt and llms-full.txt (${{ needs.check-version.outputs.target_branch }})++ Regenerated llms.txt to include all markdown files from the docs/ directory.+ Also generated llms-full.txt files for the embedding section.++ 🤖 Generated automatically by llms-txt-generate workflow++ Co-Authored-By: GitHub Actions <[email protected]>+ committer: "github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>"+ author: "github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>"+ title: "[chore] Update llms.txt for ${{ needs.check-version.outputs.target_branch }}"+ body: |+ ## Summary+ This PR contains automatically generated updates to `llms.txt` and `llms-embedding-full.txt` files, triggered by a minor or major release.++ ## Changes+ - The `llms.txt` file has been regenerated to include all markdown files from the `docs/` directory.+ - The `llms-embedding-full.txt` file has been generated for the embedding section (concatenated documentation for RAG).++ ## Generation+ These files were generated using:+ - Script: `.github/scripts/generate-llms-txt.js`+ - Branch: `${{ needs.check-version.outputs.target_branch }}`+ - Base URL: `https://raw.githubusercontent.com/metabase/metabase/refs/heads/${{ needs.check-version.outputs.target_branch }}`++ Please review the changes and merge if they look correct.++ 🤖 Generated automatically by the `llms-txt-generate` workflow.++ Co-Authored-By: GitHub Actions <[email protected]>+ labels: |+ Type:Documentation+ no-backport+ add-paths: |+ llms.txt+ llms-embedding-full.txt++ - name: Auto approve PR+ if: steps.create_pr.outputs.pull-request-number != ''+ env:+ GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}+ run: |+ gh pr review "${{ steps.create_pr.outputs.pull-request-number }}" --approve --body "Auto-approved by llms-txt-generate workflow"++ - name: Auto merge PR+ if: steps.create_pr.outputs.pull-request-number != ''+ env:+ GH_TOKEN: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}+ run: |+ gh pr merge "${{ steps.create_pr.outputs.pull-request-number }}" --auto --squashdiff --git a/.github/workflows/release.yml b/.github/workflows/release.ymlindex 3f9a1efd9bb..67e501ca133 100644--- a/.github/workflows/release.yml+++ b/.github/workflows/release.yml@@ -424,6 +424,19 @@ jobs:}});+ # updates the llms.txt asynchronously.+ # release does not need to wait for this PR to be merged,+ # because the llms.txt is in the main git repo.+ trigger-update-llms-txt:+ # do not trigger llms.txt update on automatic patch releases+ if: ${{ !inputs.auto }}+ needs: add-extra-tags+ uses: ./.github/workflows/llms-txt-generate.yml+ secrets: inherit+ with:+ version: ${{ inputs.version }}+ branch: ${{ github.ref_name }}+draft-release-notes:if: ${{ !inputs.auto }}needs: add-extra-tags