🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(mcp): expose bundled Langfuse skills (#19552)
tobi_klbf81cdbec7d8
diff --git a/.github/workflows/langfuse-skills-sync.yml b/.github/workflows/langfuse-skills-sync.ymlindex 7adefb8fae..ca2c152a28 100644--- a/.github/workflows/langfuse-skills-sync.yml+++ b/.github/workflows/langfuse-skills-sync.yml@@ -15,6 +15,7 @@ concurrency:env:BOT_BRANCH: langfuse-skills-sync-botGENERATED_FILE: packages/langfuse-skills/src/generated/skills.js+ MCP_GENERATED_FILE: packages/langfuse-skills/src/generated/mcp-skills.jsjobs:sync:@@ -74,14 +75,17 @@ jobs:exit 0fi- if [ "${#changed_files[@]}" -ne 1 ] || [ "${changed_files[0]}" != "$GENERATED_FILE" ]; then- printf "::error::Skill sync changed unexpected files: %s\n" "${changed_files[*]}"- exit 1- fi+ for file in "${changed_files[@]}"; do+ if [ "$file" != "$GENERATED_FILE" ] && [ "$file" != "$MCP_GENERATED_FILE" ]; then+ printf "::error::Skill sync changed unexpected file: %s\n" "$file"+ exit 1+ fi+ donepnpm --filter @repo/langfuse-skills run sync:checknode --check "$GENERATED_FILE"- git diff --check -- "$GENERATED_FILE"+ node --check "$MCP_GENERATED_FILE"+ git diff --check -- "$GENERATED_FILE" "$MCP_GENERATED_FILE"echo "has_changes=true" >> "$GITHUB_OUTPUT"- name: Create or update pull request@@ -93,7 +97,9 @@ jobs:token: ${{ secrets.GH_ACCESS_TOKEN }}branch: ${{ env.BOT_BRANCH }}base: main- add-paths: ${{ env.GENERATED_FILE }}+ add-paths: |+ ${{ env.GENERATED_FILE }}+ ${{ env.MCP_GENERATED_FILE }}commit-message: "chore(assistant): refresh embedded Langfuse skills"title: "chore(assistant): refresh embedded Langfuse skills"body: |@@ -105,4 +111,5 @@ jobs:- `pnpm --filter @repo/langfuse-skills run sync:check`- `node --check packages/langfuse-skills/src/generated/skills.js`+ - `node --check packages/langfuse-skills/src/generated/mcp-skills.js`- `git diff --check`diff --git a/packages/langfuse-skills/package.json b/packages/langfuse-skills/package.jsonindex d573ff17f5..d557168d99 100644--- a/packages/langfuse-skills/package.json+++ b/packages/langfuse-skills/package.json@@ -12,6 +12,11 @@"types": "./src/index.d.ts","import": "./src/index.js","require": "./src/index.js"+ },+ "./mcp": {+ "types": "./src/mcp.d.ts",+ "import": "./src/mcp.js",+ "require": "./src/mcp.js"}},"scripts": {@@ -19,6 +24,7 @@"sync:check": "node scripts/sync-skills.mjs --check"},"devDependencies": {- "prettier": "^3.8.3"+ "prettier": "^3.8.3",+ "yaml": "^2.9.0"}}diff --git a/packages/langfuse-skills/scripts/sync-skills.mjs b/packages/langfuse-skills/scripts/sync-skills.mjsindex 0f6f88230f..05b6300158 100644--- a/packages/langfuse-skills/scripts/sync-skills.mjs+++ b/packages/langfuse-skills/scripts/sync-skills.mjs@@ -2,14 +2,15 @@import { mkdirSync, readFileSync, writeFileSync } from "node:fs";import { resolve } from "node:path";+import { createHash } from "node:crypto";+import { parseDocument } from "yaml";import process from "node:process";import { format as formatWithPrettier } from "prettier";const packageRoot = resolve(new URL("..", import.meta.url).pathname);const generatedPath = resolve(packageRoot, "src/generated/skills.js");+const mcpGeneratedPath = resolve(packageRoot, "src/generated/mcp-skills.js");const sourceRef = process.env.LANGFUSE_SKILLS_REF ?? "main";-const sourceApiUrl =- `https://api.github.com/repos/langfuse/skills/contents/skills/langfuse/references?ref=${encodeURIComponent(sourceRef)}`;const isCheckMode = process.argv.includes("--check");const getGitHubHeaders = () => ({@@ -30,25 +31,6 @@ const fetchResponse = async (url) => {return response;};-const listRemoteMarkdownFiles = async () => {- const response = await fetchResponse(sourceApiUrl);- const entries = await response.json();-- if (!Array.isArray(entries)) {- throw new Error("Unexpected GitHub API response while listing remote skills");- }-- return entries- .filter(- (entry) =>- entry.type === "file" &&- typeof entry.name === "string" &&- entry.name.endsWith(".md") &&- typeof entry.download_url === "string",- )- .sort((left, right) => left.name.localeCompare(right.name));-};-const parseSkill = (fileName, markdown) => {const frontmatterMatch = markdown.match(/^---\n([\s\S]*?)\n---\n?/);if (!frontmatterMatch) {@@ -97,18 +79,64 @@ const parseSkill = (fileName, markdown) => {};};+const fetchTree = async () => {+ const response = await fetchResponse(+ `https://api.github.com/repos/langfuse/skills/git/trees/${encodeURIComponent(sourceRef)}?recursive=1`,+ );+ const tree = await response.json();+ if (tree.truncated) throw new Error("Incomplete upstream tree");+ return tree;+};++const findSkillRoots = (tree) => {+ const roots = tree.tree+ .filter((entry) => /^skills\/[^/]+\/SKILL\.md$/.test(entry.path))+ .map((entry) => entry.path.slice(0, -9))+ .sort();+ if (roots.length === 0) throw new Error("No upstream skills found");+ return roots;+};++const downloadFile = async (sha, root, entry) => {+ const response = await fetchResponse(+ `https://raw.githubusercontent.com/langfuse/skills/${sha}/${entry.path}`,+ );+ return {+ path: entry.path.slice(root.length + 1),+ bytes: Buffer.from(await response.arrayBuffer()),+ };+};++const downloadBundle = async (tree, root) => {+ const entries = tree.tree+ .filter(+ (entry) => entry.type === "blob" && entry.path.startsWith(root + "/"),+ )+ .sort((a, b) => a.path.localeCompare(b.path));+ const files = await Promise.all(+ entries.map((entry) => downloadFile(tree.sha, root, entry)),+ );+ return { name: root.split("/").at(-1), files };+};+const getExpectedSkills = async () => {- const remoteFiles = await listRemoteMarkdownFiles();- return Promise.all(- remoteFiles.map(async ({ name, download_url: downloadUrl }) => {- const markdown = await (await fetchResponse(downloadUrl)).text();- return parseSkill(name, markdown);- }),+ const tree = await fetchTree();+ const roots = findSkillRoots(tree);+ const bundles = await Promise.all(+ roots.map((root) => downloadBundle(tree, root)),);+ return { source: `langfuse/skills@${tree.sha}`, bundles };};-const renderGeneratedModule = async (skills) =>- formatWithPrettier(+const renderGeneratedModule = async ({ bundles }) => {+ const references = bundles.find(+ (bundle) => bundle.name === "langfuse",+ )?.files;+ if (!references) throw new Error("No Langfuse reference skills found");+ const skills = references+ .filter((file) => /^references\/[^/]+\.md$/.test(file.path))+ .map((file) => parseSkill(file.path, file.bytes.toString("utf8")));+ return formatWithPrettier(`// This file is generated by packages/langfuse-skills/scripts/sync-skills.mjs.// Do not edit it manually.@@ -123,16 +151,91 @@ ${skills`,{ parser: "babel" },);+};++const parseFrontmatter = ({ name, files }) => {+ const markdown = files+ .find((file) => file.path === "SKILL.md")+ .bytes.toString("utf8");+ const yaml = /^---\r?\n([\s\S]*?)\r?\n---(?:\r?\n|$)/.exec(markdown)?.[1];+ if (yaml === undefined) throw new Error(`Missing frontmatter: ${name}`);+ const document = parseDocument(yaml);+ if (document.errors.length > 0)+ throw new Error(`Invalid frontmatter: ${name}`);+ const frontmatter = document.toJS({ maxAliasCount: 0 });+ if (+ frontmatter?.name !== name ||+ !/^[a-z0-9]+(?:-[a-z0-9]+)*$/.test(name) ||+ name.length > 64 ||+ typeof frontmatter.description !== "string"+ ) {+ throw new Error(`Invalid skill metadata: ${name}`);+ }+ return frontmatter;+};++const toResourceEntry = (root, { path, bytes }) => {+ const uri = `${root}/${path.split("/").map(encodeURIComponent).join("/")}`;+ const content = path.endsWith(".md")+ ? { uri, mimeType: "text/markdown", text: bytes.toString("utf8") }+ : {+ uri,+ mimeType: "application/octet-stream",+ blob: bytes.toString("base64"),+ };+ return {+ manifest: {+ uri,+ digest: `sha256:${createHash("sha256").update(bytes).digest("hex")}`,+ size: bytes.length,+ },+ content,+ };+};++const toSkillEntry = (bundle) => {+ const root = `skill://langfuse/${bundle.name}`;+ const frontmatter = parseFrontmatter(bundle);+ const entries = bundle.files.map((file) => toResourceEntry(root, file));+ return {+ skill: {+ uri: `${root}/SKILL.md`,+ frontmatter,+ resources: entries.map((entry) => entry.manifest),+ },+ resources: entries.map((entry) => entry.content),+ };+};++const renderMcpModule = async ({ source, bundles }) => {+ const entries = bundles.map(toSkillEntry);+ const skills = entries.map((entry) => entry.skill);+ const resources = entries.flatMap((entry) => entry.resources);+ return formatWithPrettier(+ `// This file is generated by packages/langfuse-skills/scripts/sync-skills.mjs.+// Do not edit it manually.+// Source: ${source}++exports.LANGFUSE_MCP_SKILLS = ${JSON.stringify(skills)};+exports.LANGFUSE_MCP_RESOURCES = ${JSON.stringify(resources)};+`,+ { parser: "babel" },+ );+};const main = async () => {const expectedSkills = await getExpectedSkills();const expectedModule = await renderGeneratedModule(expectedSkills);+ const expectedMcpModule = await renderMcpModule(expectedSkills);if (isCheckMode) {try {if (readFileSync(generatedPath, "utf8") !== expectedModule) {throw new Error("Generated skill catalog differs from upstream");}+ if (readFileSync(mcpGeneratedPath, "utf8") !== expectedMcpModule) {+ throw new Error("Generated MCP skill catalog differs from upstream");+ }} catch (error) {if (error.code === "ENOENT") {throw new Error("Generated skill catalog is missing");@@ -140,17 +243,15 @@ const main = async () => {throw error;}- console.log(- `Generated Langfuse skills are in sync: ${expectedSkills- .map(({ name }) => name)- .join(", ")}`,- );+ console.log("Generated Langfuse skill catalog is in sync.");+ console.log("Generated MCP skill catalog is in sync.");return;}mkdirSync(resolve(packageRoot, "src/generated"), { recursive: true });writeFileSync(generatedPath, expectedModule);- console.log(`Synced ${expectedSkills.length} Langfuse skills.`);+ writeFileSync(mcpGeneratedPath, expectedMcpModule);+ console.log("Synced Langfuse and MCP skill catalogs.");};main().catch((error) => {diff --git a/packages/langfuse-skills/src/generated/mcp-skills.js b/packages/langfuse-skills/src/generated/mcp-skills.jsnew file mode 100644index 0000000000..f92cde7ada--- /dev/null+++ b/packages/langfuse-skills/src/generated/mcp-skills.js@@ -0,0 +1,220 @@+// This file is generated by packages/langfuse-skills/scripts/sync-skills.mjs.+// Do not edit it manually.+// Source: langfuse/skills@298b316eb9a5d302811c4182fb9c1ee81a449257++exports.LANGFUSE_MCP_SKILLS = [+ {+ uri: "skill://langfuse/langfuse/SKILL.md",+ frontmatter: {+ name: "langfuse",+ description:+ "Interact with Langfuse and access its documentation: tracing, monitoring, creating datasets, running experiments, and evaluating AI applications. Use when needing to (1) query or modify Langfuse data, (2) look up Langfuse documentation, concepts, integration guides, a feature or SDK usage, or (3) do any AI engineering task (AI observability, prompt engineering/management, evaluation and evaluator management, experimentation, dataset management, evaluation-driven CI/CD, feedback collection). Invoke it for tasks in this scope even when Langfuse is not configured or explicitly mentioned.",+ "allowed-tools": [+ "WebFetch(domain:langfuse.com)",+ "Bash(curl *langfuse.com/*)",+ "Bash(npx langfuse-cli api __schema *)",+ "Bash(npx langfuse-cli api * --help *)",+ "Bash(npx langfuse-cli api * list *)",+ "Bash(npx langfuse-cli api * get *)",+ "Bash(bunx langfuse-cli api __schema *)",+ "Bash(bunx langfuse-cli api * --help *)",+ "Bash(bunx langfuse-cli api * list *)",+ "Bash(bunx langfuse-cli api * get *)",+ ],+ },+ resources: [+ {+ uri: "skill://langfuse/langfuse/references/ci-cd.md",+ digest:+ "sha256:3982cd4f5ed88c4402a1fccdb8289e8caedab482f122fc0c8ca712e6a727d2c0",+ size: 2950,+ },+ {+ uri: "skill://langfuse/langfuse/references/cli.md",+ digest:+ "sha256:eeb58ce7557978169fb207f53bda866d85bc3d13330019546ac5695c6941ba8c",+ size: 2156,+ },+ {+ uri: "skill://langfuse/langfuse/references/cost-optimization.md",+ digest:+ "sha256:4e046ff992158272ba17c6dc50adea3585440cedd1af7cf43b12f0745eb791c5",+ size: 930,+ },+ {+ uri: "skill://langfuse/langfuse/references/create-dataset.md",+ digest:+ "sha256:b0a0b34620d5054c588ffa4b016c39e92ede1fdbed18c210f2e9476c88bb49ae",+ size: 2454,+ },+ {+ uri: "skill://langfuse/langfuse/references/error-analysis.md",+ digest:+ "sha256:a818baef6fee26f682c50ea2380c12902b1c085ef3d7ac8f06499525045e9024",+ size: 4520,+ },+ {+ uri: "skill://langfuse/langfuse/references/instrumentation.md",+ digest:+ "sha256:668440ae785f67a2d01872eaea780ba2e237e0653894b55c74a9dd93fda647c7",+ size: 10409,+ },+ {+ uri: "skill://langfuse/langfuse/references/judge-calibration.md",+ digest:+ "sha256:95edd5f90aae53218cff3e696484079b14a1dba1fa0779940abcb4782285efda",+ size: 4536,+ },+ {+ uri: "skill://langfuse/langfuse/references/prompt-engineering.md",+ digest:+ "sha256:671c6ff54febec23fe5200408e56410b1e4d42bc7d8bd592cf60d112ae991c5e",+ size: 2802,+ },+ {+ uri: "skill://langfuse/langfuse/references/prompt-migration.md",+ digest:+ "sha256:c194a0df3614bf8ebfabab5a1729538d276fad8d6ba89fac3b2cfed38a3ff5bc",+ size: 5151,+ },+ {+ uri: "skill://langfuse/langfuse/references/setting-up-evals.md",+ digest:+ "sha256:bb17bd66132bd05d43f234c820cd12d19403d53bb657ab441c0039248cd61c3a",+ size: 3800,+ },+ {+ uri: "skill://langfuse/langfuse/references/skill-feedback.md",+ digest:+ "sha256:4a95834cdaacfd4248bdd1a079550981472803a461f824a0b2cc506e055f50c7",+ size: 2157,+ },+ {+ uri: "skill://langfuse/langfuse/references/user-feedback.md",+ digest:+ "sha256:1111706198dad1adeea04fd700688b44b8a16ed9459b77198d6ce6a04556f2d6",+ size: 2159,+ },+ {+ uri: "skill://langfuse/langfuse/references/v4-project-migration.md",+ digest:+ "sha256:912d8639e9a60c5e138c42832bc2374e5622916fcc0cfe8814a0420ab06b587e",+ size: 6422,+ },+ {+ uri: "skill://langfuse/langfuse/SKILL.md",+ digest:+ "sha256:312a76ca32756062f28f95cbf36368e8aeb02212f2e3296e9ac214c029eaaae5",+ size: 8703,+ },+ ],+ },+ {+ uri: "skill://langfuse/migrate-to-langfuse/SKILL.md",+ frontmatter: {+ name: "migrate-to-langfuse",+ description:+ "Migrate to Langfuse from another LLM observability/evals platform (LangSmith, Arize AX, Phoenix, Braintrust, Helicone, Promptfoo, ...). A one-off vendor switch: live tracing cutover plus transfer of historical traces, datasets, prompts, and evaluators. Not for anything Langfuse-to-Langfuse — SDK or version upgrades, project/region moves, self-hosted to Cloud — and not for moving in-code prompts into Langfuse; the main langfuse skill covers those.",+ "allowed-tools": [+ "WebFetch(domain:langfuse.com)",+ "Bash(curl *langfuse.com/*)",+ ],+ },+ resources: [+ {+ uri: "skill://langfuse/migrate-to-langfuse/references/reingest-history.md",+ digest:+ "sha256:500e32fd225d5ed7849d07bb291d906365cece58394eaf882e49fd73773a6470",+ size: 4523,+ },+ {+ uri: "skill://langfuse/migrate-to-langfuse/SKILL.md",+ digest:+ "sha256:9c9203f48f3df0a34f58ebc619ffd9a3787264e08e4129eabb91501b74279696",+ size: 6849,+ },+ ],+ },+];+exports.LANGFUSE_MCP_RESOURCES = [+ {+ uri: "skill://langfuse/langfuse/references/ci-cd.md",+ mimeType: "text/markdown",+ text: "---\nname: langfuse-ci-cd\ndescription: Set up or extend agent regression checks / gating in GitHub Actions CI/CD using `langfuse/experiment-action`.\n---\n\n# Langfuse CI/CD\n\n## Checklist\n\n- [ ] Follow the [CI/CD docs page](https://langfuse.com/docs/evaluation/experiments/experiments-ci-cd) and the [langfuse/experiment-action README](https://github.com/langfuse/experiment-action)\n- [ ] Inspect whether the local repository is a GitHub repo. If not, `langfuse/experiment-action` is not applicable. Follow the guide for other [CI/CD systems](https://langfuse.com/docs/evaluation/experiments/experiments-ci-cd#other-cicd-systems) instead\n- [ ] Ask the user which [evaluators and run evaluators](https://langfuse.com/docs/evaluation/experiments/experiments-via-sdk#evaluators) they want to set up\n- [ ] Ask the user if and when yes which regression thresholds they want to set\n- [ ] Confirm dataset existence and shape of the dataset items before writing code with the Langfuse CLI (see `references/cli.md`)\n - `langfuse-cli api datasets list`\n - `langfuse-cli api dataset-items list --dataset-name <dataset name> --limit 5`\n- [ ] Propose the user to verify the check by actually running the new CI check (e.g. by creating a pull request)\n- [ ] If the evaluator task uses a third-party dependency, add the necessary CI steps to install them\n\n### GitHub specific checklist\n- [ ] Ask the user how they want the [workflow to be triggered](https://docs.github.com/en/actions/reference/workflows-and-actions/events-that-trigger-workflows)\n- [ ] If available, use the `gh` CLI to check secret existence / set secrets for:\n - Langfuse credentials\n - Credentials required by the evaluator task (e.g. OpenAI or Anthropic API keys)\n\n## Common Issues\n\n| Issue | Solution |\n|-------|----------|\n| `gh` is missing or not authenticated | Install the GitHub CLI if needed, then run `gh auth status` and `gh auth login` before using `gh secret` or `gh workflow` commands. |\n| Local Langfuse environment variables are not set | Set `LANGFUSE_PUBLIC_KEY`, `LANGFUSE_SECRET_KEY`, and `LANGFUSE_HOST` locally before using `langfuse-cli`; do not ask the user to paste secret values into chat. |\n| Workflow secrets or action inputs are wrong | Verify `LANGFUSE_PUBLIC_KEY`, `LANGFUSE_SECRET_KEY`, `langfuse_base_url`, and provider secrets exist in the target repo/environment and are passed to the action step. |\n| Forked PR cannot access secrets | GitHub restricts secret access for forked PRs. Document the limitation or choose a trusted trigger such as internal PR, trusted-branch `push`, or `workflow_dispatch`. |\n| No default/base branch exists | Create an initial empty commit on the intended default branch before trying to verify a PR-triggered workflow. |\n| Script fails reading dataset fields | Re-inspect the dataset items with the Langfuse CLI, check `input`, `expected_output`, and metadata, and extract fields from object-shaped outputs explicitly. |\n",+ },+ {+ uri: "skill://langfuse/langfuse/references/cli.md",+ mimeType: "text/markdown",+ text: "---\nname: langfuse-cli\ndescription: Langfuse CLI usage reference — install, resource/action discovery, credentials, and common usage tips for `langfuse-cli`. Use for further tips on using the Langfuse CLI.\n---\n\n# Langfuse CLI Reference\n\nDocumentation: https://langfuse.com/docs/api-and-data-platform/features/cli\n\n## Install\n\n```bash\n# Run directly (recommended)\nnpx langfuse-cli api <resource> <action>\nbunx langfuse-cli api <resource> <action>\n\n# Or install globally\nnpm i -g langfuse-cli\nlangfuse api <resource> <action>\n```\n\n## Discovery\n\n```bash\n# List all resources and auth info\nlangfuse api __schema\n\n# List actions for a resource\nlangfuse api <resource> --help\n\n# Show args/options for a specific action\nlangfuse api <resource> <action> --help\n\n# Preview the curl command without executing\nlangfuse api <resource> <action> --curl\n```\n\n## Credentials\n\nSet environment variables:\n\n```bash\nexport LANGFUSE_PUBLIC_KEY=pk-lf-...\nexport LANGFUSE_SECRET_KEY=sk-lf-...\nexport LANGFUSE_BASE_URL=https://cloud.langfuse.com \n```\n\n## Tips\n\n- Use `--json` for machine-readable JSON output\n- Use `--curl` to preview the HTTP request without executing\n- All list commands support filtering — check `<resource> <action> --help` for available options\n- Prefer `observations` over `legacy-observations-v1s` — `observations` is the modern high-performance endpoint (cursor pagination, selective field groups); `legacy-observations-v1s` is the deprecated v1\n- Prefer `metrics` over `legacy-metrics-v1s` for the same reason\n- Prefer `scores` over `legacy-score-v1s` for list/get operations\n- Always query via `observations`, not `traces`, even when the user phrases the request in terms of traces — the `traces` endpoints are outdated (add `--trace-id` to scope to a known trace). See the [Observations API docs](https://langfuse.com/docs/api-and-data-platform/features/observations-api) for the v1 → v2 mapping.\n- Pagination: legacy v1 endpoints use `--limit` and `--page`; modern endpoints (`observations`, `metrics`, `scores`) use cursor-based pagination — pass `--limit`, then thread `meta.cursor` from the response into the next request's `--cursor`\n",+ },+ {+ uri: "skill://langfuse/langfuse/references/cost-optimization.md",+ mimeType: "text/markdown",+ text: "---\nname: langfuse-cost-optimization\ndescription: Optimize the cost of an LLM application or AI agent while preserving output quality.\n---\n\nRead the [cost optimization Academy page](https://langfuse.com/academy/experiments/cost-optimization) in full.\n\nAlways present the cost optimization opportunities in a table with these columns:\n\n| Opportunity | Explanation and why it works | Estimated savings at analyzed volume | Implementation effort | Risk to output quality |\n|---|---|---:|---|---|\n\nFor every opportunity, explain what would change and why that change should reduce cost. Link to the relevant [Academy guidance](https://langfuse.com/academy/experiments/cost-optimization) when it supports the explanation.\n\nState the current cost and run volume used as the baseline. Estimate every opportunity's savings at that same volume. Label assumptions and use ranges when the available data does not support a precise estimate.\n",+ },+ {+ uri: "skill://langfuse/langfuse/references/create-dataset.md",+ mimeType: "text/markdown",+ text: "---\nname: langfuse-dataset-construction\ndescription: Collaborative Langfuse dataset creation workflow. Use when the user needs to create, design, seed, reshape, review, or upload a Langfuse dataset or dataset version; especially when they need a minimal but complete dataset, e.g. for quality checks or avoiding regression.\n---\n\n# Langfuse Dataset Construction\n\n## Primary Guide\n\nFollow the [Langfuse Academy datasets guide](https://langfuse.com/academy/datasets) as the human-readable source of truth for dataset design; fetch current docs before implementation.\n\nUse CLI schema/help discovery for current Langfuse API mechanics.\n\n## Workflow\n\nGuide the user through dataset creation as an interview, proposal, approval, implementation loop:\n\n1. Read the primary guide and relevant current Langfuse docs.\n2. Clarify the problem the user is facing, interviewing them if context is thin, to decide the goal, release, or evaluation decision the dataset should support.\n3. Inspect available context: the user's goal, the application path or codebase when available, existing datasets, traces, prompts, scores, monitors, user feedback, tickets, expert examples, existing eval and dataset assets.\n4. Interview the user on available sources beyond production traces.\n5. For each step going forward, propose a direction and get the user's approval, specifically:\n - Dataset distribution dimensions (propose and get user input).\n - Item schema: input, expected output, metadata (propose and get user input).\n6. Generate the first minimal draft of items against the approved schema and distribution, then present it for approval.\n7. Heavily prompt the user to review expected outputs, as AI-generated expected outputs cannot be considered ground truth. Send a link with instructions to review.\n\n## Critical Rules\n- Do not create, upsert, reshape, or upload a live Langfuse dataset until the user has approved the dataset goal, source mix, item schema, and first minimal draft, unless the user already gave those details and explicitly asked for immediate mutation.\n- Design the smallest complete dataset version that can serve the goal. Prefer a minimal reviewable v0 over broad coverage. A first draft is usually 5-12 high-signal items unless the user asks for a different size.\n- Keep `input`, `expectedOutput`, and `metadata` responsibilities separate. Put additional information, notes, and comments into `metadata`, not into `input` or `expectedOutput`.\n",+ },+ {+ uri: "skill://langfuse/langfuse/references/error-analysis.md",+ mimeType: "text/markdown",+ text: '---\nname: langfuse-error-analysis\ndescription: Deep-dive error analysis of an LLM pipeline or AI application using Langfuse traces.\n Use this skill whenever the user wants to understand why their AI system is producing\n bad outputs, where their pipeline is failing, or how to categorise and label failures\n into a taxonomy and decide what to prioritise fixing. Also trigger for "review my\n traces", "my outputs look wrong", "help me debug my LLM app", "I want to analyse\n errors", "build a failure taxonomy", or "what\'s going wrong with my pipeline".\n---\n\n# Error Analysis\n\n## Primary Guide\n\n**1. Fetch the guide in this blogpost**\n\nhttps://langfuse.com/guides/cookbook/error-analysis-llm-applications.md\n\nIf fetch is not available query for langfuse.com error analysis guide\n\nRead it in full. It defines the authoritative 5-step process (sample selection → open coding → clustering → labelling → deciding what to fix).\n\n**2. Guide the user through this step by step**\n\nYou as a coding agent and the user go through this together to perform a full error analysis with their data in Langfuse. Set up the scores, representative sample, and annotation queue for the user, and provide direct UI links wherever their action is required. Be proactive and explain what is happening in plain language without reciting internal step numbers or jargon.\n\n## Rules CRITICAL\n- Perform interactions with the user\'s Langfuse instance yourself rather than telling the user to do them.\n- The user performs the open-coding review and supplies human labels; do not do this on their behalf.\n- Where a step needs the user\'s judgment or input, pause and ask before acting.\n- Use charts where possible to display data.\n\n---\n\n## Langfuse Implementation Notes\n\nThe guide describes the process. These notes cover the Langfuse-specific mechanics required to execute it.\n\n### Credentials\n\nMake sure Langfuse credentials are available before starting — a public key (`pk-lf-...`), a secret key (`sk-lf-...`), and the host (e.g. `https://cloud.langfuse.com` for EU, `https://us.cloud.langfuse.com` for US, `https://jp.cloud.langfuse.com` for JP, or a self-hosted URL). If they aren\'t configured, ask the user to set them — do not ask them to paste secret values into chat.\n\nVerify you can actually reach the user\'s project before proceeding. If access fails, stop and ask the user to check their credentials and host.\n\n### Annotation target: OBSERVATION versus TRACE\n\n> **CRITICAL:** In OpenTelemetry-instrumented apps, trace-level `input`/`output` can be null — content often lives in a GENERATION observation. In that case, add the GENERATION observation (not the trace) to the annotation queue, so the content being reviewed is actually visible.\n\n### Annotation queues\n\n> **CRITICAL:** Queues cannot be updated or deleted after creation. Create score configs first, then the queue with all config IDs. To add new configs later, create a new queue.\n\n\n**Always give the user a direct link immediately after creating a queue:**\n\n| Host | URL pattern |\n|------|-------------|\n| EU cloud | `https://cloud.langfuse.com/project/<projectId>/annotation-queues/<queueId>` |\n| US cloud | `https://us.cloud.langfuse.com/project/<projectId>/annotation-queues/<queueId>` |\n| Self-hosted | `<LANGFUSE_BASE_URL>/project/<projectId>/annotation-queues/<queueId>` |\n\nInstruction to give: *"Please open code the first ~50 examples. For each trace, write what you observe in the `open_coding` field (describe behaviour, don\'t diagnose root causes), then set `pass_fail_assessment` to Pass or Fail."*\n\n\n### Prompt fixes\n\nWhen a category warrants a prompt fix, always offer the user two options:\n1. Create it as a versioned prompt in Langfuse (tracked, usable via the prompt API)\n2. Draft the specific text change for them to review and apply\n\n### Setup evaluators\n\nWhen a category warrants an evaluator, build it following `references/setting-up-evals.md`.\n\n\n### Common gotchas\n\n| Mistake | Fix |\n|---------|-----|\n| Annotating the trace instead of the observation | Target the GENERATION observation, not the trace, when content lives there |\n| Creating a score config without checking existing ones | Check existing score configs first; they can\'t be deleted |\n| Queue created before score configs | Create configs → collect IDs → create queue |\n| Requesting too many traces at once | Page size is capped (max 100); paginate to get more |\n| No rate limiting on bulk queue item creation | Space out requests to avoid hitting rate limits (429) |\n',+ },+ {+ uri: "skill://langfuse/langfuse/references/instrumentation.md",+ mimeType: "text/markdown",+ text: "---\nname: langfuse-observability\ndescription: Instrument LLM applications with Langfuse tracing. Use when setting up Langfuse, adding observability to LLM calls, or auditing existing instrumentation.\n---\n\n# Langfuse Observability\n\nInstrument LLM applications with Langfuse tracing, following best practices and tailored to your use case.\n\n## Workflow\n\n### 1. Assess Current State\n\nCheck the project:\n\n- Is Langfuse SDK installed?\n- What LLM frameworks are used? (OpenAI SDK, LangChain, LlamaIndex, Vercel AI SDK, etc.)\n- Is there existing instrumentation?\n\n**No integration yet:** Set up Langfuse using a framework integration if available. Integrations capture more context automatically and require less code than manual instrumentation.\n\n**Integration exists:** Audit against baseline requirements below.\n\n### 2. Verify Baseline Requirements\n\nEvery trace should have these fundamentals:\n\n| Requirement | Check | Why |\n| ------------------------- | ---------------------------------------------------------------------------------------- | ------------------------------------------------------ |\n| Model name | Is the LLM model captured? | Enables model comparison and filtering |\n| Token usage | Are input/output tokens tracked? | Enables automatic cost calculation |\n| Good trace names | Are names descriptive? (`chat-response`, not `trace-1`) | Makes traces findable and filterable |\n| Span hierarchy | Are multi-step operations nested properly? | Shows which step is slow or failing |\n| Correct observation types | Are generations marked as generations, and is each other call given its most specific type (`retriever` for a lookup, `agent` for a subagent, etc.) rather than a generic `tool`/`span`? See the [observation types docs](https://langfuse.com/docs/observability/features/observation-types). | Enables model-specific analytics and drives the Agent Graph |\n| Sensitive data masked | Is PII/confidential data excluded or masked? | Prevents data leakage |\n| Trace input/output | Does the trace capture meaningful input/output? Is input explicitly set to show only relevant data (e.g., user message), not all function args? | Makes traces readable in the UI and avoids leaking sensitive args |\n\nFramework integrations (OpenAI, LangChain, etc.) handle model name, tokens, and observation types automatically. Prefer integrations over manual instrumentation.\n\nDocs: https://langfuse.com/docs/tracing\n\n**Beyond the baseline**, add context relevant to the app. Infer from code where possible; only ask when it's not obvious (e.g. how they judge a good vs. bad response, what they'd filter a dashboard by, which user segments they'd compare). These are not baseline — add only what fits.\n\n| If code shows... | Add | Why |\n| ---------------------------------------------------- | ------------------- | ------------------------------------------------------------------------------- |\n| Conversation history, chat endpoints, message arrays | `session_id` | Groups conversations — [docs](https://langfuse.com/docs/tracing-features/sessions) |\n| User authentication, `user_id` variables | `user_id` | User filtering and cost attribution — [docs](https://langfuse.com/docs/tracing-features/users) |\n| Multiple distinct endpoints/features | `feature` tag | Per-feature analytics — [docs](https://langfuse.com/docs/tracing-features/tags) |\n| Customer/tenant identifiers | `customer_tier` tag | Cost/quality breakdown by segment — [docs](https://langfuse.com/docs/tracing-features/tags) |\n| Feedback collection, ratings | Feedback score | Quality filtering and trends — [docs](https://langfuse.com/docs/scores/overview) |\n| Image/audio/file input or output (e.g. image generation, vision) | Media in observation input/output | Renders media in the UI; base64 data URIs and external URLs are handled automatically, wrap raw bytes/files in `LangfuseMedia`. Follow the [multi-modality docs](https://langfuse.com/docs/observability/features/multi-modality) to implement |\n\n### 3. Run and Self-Audit the Traces (required)\n\nInstrumentation isn't done when the code compiles. This is a loop you own as the agent, it's your responsibility to deliver traces of the highest quality your can produce:\n\n**a.** Execute the instrumented path end-to-end so a trace is actually sent.\n\n**b.** Fetch the trace(s) you just created from Langfuse. Any method works (`langfuse-cli`, REST API, SDK, MCP); the CLI is usually simplest — see [references/cli.md](references/cli.md).\n\n**c.** Audit the trace against the best-practices page. **Always fetch it fresh — never audit from memory, this cannot be skipped** (the guidance changes over time):\n\nhttps://langfuse.com/docs/observability/best-practices\n\nAsk yourself, for each observation: is all data that a user might need in the future, to understand exactly what context the agent had when it made decisions, available in Langfuse?\n\n**d.** Fix every gap you find, then re-run and re-fetch to confirm. Repeat until the trace clears the guidance. Then report what you audited and changed, and link the final trace.\n\n### 4. Explore Traces With the User\n\nWith a clean trace in place, invite the user to explore it in the Langfuse UI:\n\n\"Your traces are now appearing in Langfuse. Take a look at a few of them—see what data is being captured, what's useful, and what's missing.\"\n\nPoint them to the relevant views:\n\n- Traces view: individual requests\n- Sessions view: grouped conversations (if `session_id` added)\n- Dashboard: filtered views using tags\n- Scores: filter by quality metrics\n\nThis helps them understand what they're getting, spot what's missing, and ask better questions about what to add next.\n\n## Always Explain Why\n\nWhen suggesting additions, explain the user benefit:\n\n```\n\"I recommend adding session_id to your traces.\n\nWhy: This groups messages from the same conversation together.\nYou'll be able to see full conversation flows in the Sessions view,\nmaking it much easier to debug multi-turn interactions.\n\nLearn more: https://langfuse.com/docs/tracing-features/sessions\"\n```\n\n## Multi-agent systems (subagent dispatch)\n\nWhen one agent's execution dispatches OTHER agents (coding agents like Claude Code/Codex, research agents, orchestrator/worker architectures), a few extra points on top of the baseline:\n\n- **Type a subagent's own execution as `agent`, not `tool`/`span`.** A bare tool/span for a dispatch hides all of the subagent's internal structure; `agent` lets it show up as its own node in the [Agent Graph](https://langfuse.com/docs/observability/features/agent-graphs).\n- **Don't emit duplicate dispatch + execution nodes.** Emitting both a `tool`-typed \"dispatch\" span and a separate `agent`-typed observation for the same subagent, as siblings, double-represents one event. Emit only the `agent` observation when you have the subagent's actual execution; keep a bare tool span only as a fallback when you have no visibility into what the subagent did.\n- **Nest recursively.** Nest the subagent's `agent` observation under the `agent` or `span` that orchestrates the dispatch, as a sibling of the `generation` that requested it. Within the subagent, put its generations, tool calls, and nested subagent executions under that subagent's `agent` observation; each tool or nested subagent is a sibling of the generation that requested it, not a child of that generation.\n- **Name subagents distinctly.** Frameworks often default every subagent to the same generic role name, making them indistinguishable in the tree and graph (nodes key on name). Derive a specific name from the subagent's actual task/role when the framework doesn't provide one.\n\n## Common Mistakes\n\n| Mistake | Problem | Fix |\n| ---------------------------------------------- | --------------------------------------------------- | --------------------------------------------------------------------------------- |\n| No `flush()` in scripts | Traces never sent | Call `langfuse.flush()` before exit |\n| Flat traces | Can't see which step failed | Use nested spans for distinct steps |\n| Generic trace names | Hard to filter | Use descriptive names: `chat-response`, `doc-summary` |\n| Logging sensitive data | Data leakage risk | Mask PII before tracing |\n| Not explicitly setting input with `@observe` | All function args become trace input (including API keys, configs) | Python: use `langfuse.update_current_span(input=...)`. JS/TS: use `updateActiveObservation({ input: ... })`. Set only the relevant input (e.g., user message) |\n| Manual instrumentation when integration exists | More code, less context | Use framework integration |\n| Langfuse import before env vars loaded | Langfuse initializes with missing/wrong credentials | Import Langfuse AFTER loading environment variables (e.g., after `load_dotenv()`) |\n| Wrong import order with OpenAI | Langfuse can't patch the OpenAI client | Import Langfuse and call its setup BEFORE importing OpenAI client |\n",+ },+ {+ uri: "skill://langfuse/langfuse/references/judge-calibration.md",+ mimeType: "text/markdown",+ text: "---\nname: langfuse-judge-calibration\ndescription: Calibrate a new or existing LLM-as-a-Judge against labeled examples, analyze disagreements, iterate on its prompt with user approval, and deploy the approved version.\n---\n\n# Judge Calibration\n\n## 1. Prepare the calibration\n\nThe calibration data must:\n\n- match the information the judge receives in a real execution, based on a representative production trace, filter rules, and variable mappings;\n- include a good distribution across every possible outcome;\n- include known failure cases when badly judged traces prompted the calibration; and\n- contain expected outputs in exactly the judge's output shape and vocabulary, without passing them to the judge.\n\n1. Fetch the current docs for [LLM-as-a-Judge](https://langfuse.com/docs/evaluation/evaluation-methods/llm-as-a-judge), [datasets](https://langfuse.com/docs/evaluation/experiments/datasets), [experiments via SDK](https://langfuse.com/docs/evaluation/experiments/experiments-via-sdk), [prompt versions and labels](https://langfuse.com/docs/prompt-management/data-model), and [LLM connections](https://langfuse.com/docs/administration/llm-connection). Use the current API schema and SDK rather than assuming fields or methods.\n2. For a new judge, immediately create its evaluator, production filters, mappings, score definition, and model, but leave its rule inactive. If no LLM connection exists, ask the user to create one and explain that the judge needs it; otherwise use the default connection unless the user specified another one.\n3. Use one dataset per judge. If it does not exist, create `judge-calibration/<judge-name>` from labeled examples or known good and bad traces. If an existing calibration dataset is not in a folder intended for future judge-calibration datasets, ask before moving it and leave it where it is if the user declines.\n4. Save the candidate prompt as `judge-calibration/<judge-name>`. If the judge was calibrated before and its prompt already exists, create a new version for the change.\n\n## 2. Run the calibration\n\nRun every calibration directly with the Langfuse Experiments SDK; do not ask the user to trigger a Prompt Experiment in the UI. Each run must:\n\n1. fetch `judge-calibration/<judge-name>` from Langfuse;\n2. fetch the candidate prompt version from `judge-calibration/<judge-name>` in Prompt Management;\n3. execute that prompt against every dataset item using the same model as the judge; and\n4. define a Boolean evaluator named `<judge-name>-judge-output-correct` and run it together with the SDK experiment. It returns `true` only when the experiment output exactly equals the dataset item's expected output.\n\nWait for every task and evaluator execution to finish before reviewing results, checking periodically while work remains pending or delayed.\n\n## 3. Review the results\n\n1. Concisely explain what you did and link to the tested prompt version and that experiment run's results screen. On the first round only, also link to the dataset and, for a new judge, the inactive judge.\n2. Inspect the accuracy and every mismatch. Treat 85% overall accuracy as quite good—about the same as human judgment—only if accuracy is also acceptable in every category; always state that benchmark and explain the mistakes, even when accuracy exceeds it.\n3. Show all mistakes in a table with columns `Dataset input`, `Expected output`, and `Actual output`.\n4. If some labels look like debatable edge cases, group those first as items whose expected labels the user may want to revisit. Otherwise, or when the user is confident in the labels, group mistakes by actual output category.\n\n## 4. Iterate until the judge is approved\n\n1. Wait for the user's response. It may be specific label guidance, a request to fix the failures, or approval of the judge.\n2. If you want to change the judge prompt, follow `references/prompt-engineering.md`. This is important. Show the prompt change to the user before applying it.\n3. After the user approves the proposed change, create a new version of `judge-calibration/<judge-name>` and rerun the SDK experiment from section 2.\n4. Wait for the run to finish, review the results again, and repeat until the user approves the judge.\n\n## 5. Deploy the approved judge\n\nOnly deploy after the user approves the judge.\n\nCopy the approved prompt into the actual evaluator, assign its Prompt Management version the `production` label, and confirm that the model, mappings, output definition, and filters match what was calibrated. For a new judge, activate its rule.\n",+ },+ {+ uri: "skill://langfuse/langfuse/references/prompt-engineering.md",+ mimeType: "text/markdown",+ text: '---\nname: langfuse-prompt-engineering\ndescription: Write or change prompts in Langfuse or code. Use whenever the user asks to create, edit, rewrite, debug, tune, or otherwise modify a prompt, including a small wording or instruction change. Distinct from prompt-migration and judge-calibration.\n---\n\n# Prompt Engineering\n\n## Universal principles\n\n1. **Be specific.** State the required output format, constraints, and, when order matters, numbered steps. Write for someone with minimal context.\n2. **Use labeled sections** for the role, instructions, examples, and context. Use XML tags when they help distinguish rules, data, and variables.\n3. **Give the model a role.** A focused system-prompt sentence can steer tone and behavior.\n4. **Explain why important rules exist** so the model can apply them to adjacent cases.\n\n## Adjusting an existing prompt\n\n1. **Pin down the failure first.** Use concrete misbehaving outputs and name the exact failure mode: wrong format, ignored instruction, wrong tone, hallucination, or another observable error. The task and data show what could fail, not what did fail; do not write a fix before observing the failure.\n2. **Trace the failure to the prompt.** Look for a missing, ambiguous, or conflicting instruction.\n3. **Fix the failure class, not the example.** Generalize the observed mistake into an error class and write the instruction at that level, in your own words. Cover adjacent cases, but do not add definitions, rules, or examples for unobserved problems or encode the failing example itself. If a generalized instruction does not work, report the failure instead of narrowing the rule to that case.\n4. **Preserve existing behavior.** Before changing text, identify what behavior it protects. Avoid contradicting other rules or breaking cases that already work.\n5. **Make the smallest testable edit.** Change one cause per attempt so you can attribute the result. Avoid extra length, emphasis, or all-caps. In Langfuse, save each attempt as a new version or label so you can compare and roll back.\n\n## Model-specific tuning\n\nIdentify the target model if you can — tuning differs by model, so when you know which model the prompt runs on, follow that model\'s own prompting guidance.\n\n| Technique | Standard (GPT) | Reasoning (Claude latest) |\n|-----------|----------------|---------------------------|\n| Instruction density | High, prescriptive | Goal-level; don\'t micro-manage steps |\n| Reasoning | Add explicit chain-of-thought | Native thinking; "think thoroughly" + tune `effort` |\n| Emphasis language | Strong directives fine | Dial back `MUST`/`CRITICAL` |\n| Output format | Prefilling/scaffolds | Prefill removed (4.6+); structured outputs + "no preamble" |\n| Roles | `developer` > `user` > `assistant` | `system` role + `user`/`assistant` |\n',+ },+ {+ uri: "skill://langfuse/langfuse/references/prompt-migration.md",+ mimeType: "text/markdown",+ text: '---\nname: langfuse-prompt-migration\ndescription: Migrate hardcoded prompts to Langfuse for version control and deployment-free iteration. Use when user wants to externalize prompts, move prompts to Langfuse, or set up prompt management.\n---\n\n# Langfuse Prompt Migration\n\nMigrate hardcoded prompts into Langfuse-managed prompts. The API mechanics (`create_prompt`, `get_prompt`, `.compile()`, linking to traces) are in the docs — fetch them at execution time.\n\n## Prerequisites\n\nVerify credentials exist — check presence only, never print the secret key (its value would land in the agent\'s context and transcripts):\n\n```bash\n[ -n "$LANGFUSE_PUBLIC_KEY" ] && echo "public key: set" || echo "public key: missing"\n[ -n "$LANGFUSE_SECRET_KEY" ] && echo "secret key: set" || echo "secret key: missing"\n[ -n "${LANGFUSE_BASE_URL:-${LANGFUSE_HOST:-}}" ] && echo "base url: set" || echo "base url: missing"\n```\n\nUse `LANGFUSE_BASE_URL` for current SDKs. If only `LANGFUSE_HOST` is set, export `LANGFUSE_BASE_URL="$LANGFUSE_HOST"`. If a CLI expects `LANGFUSE_HOST` and only `LANGFUSE_BASE_URL` is set, export `LANGFUSE_HOST="$LANGFUSE_BASE_URL"`.\n\nIf credentials or the base URL are missing, ask the user to set them in their shell or a `.env` file. Do not ask them to paste secret keys into chat.\n\n## 1. Inventory every prompt (before writing any code)\n\nFor each prompt, record:\n\n- **Name**: lowercase, hyphenated (e.g. `chat-assistant`)\n- **Source file**: where the prompt text lives\n- **Code file to refactor**: the file that USES the prompt. For asset files (`.txt`/`.yaml`/`.md`), this is the file that loads the asset, not the asset itself\n- **Type**: `chat` (message array) or `text` (plain string)\n- **Variables**: values interpolated in, converted to `{{var}}`\n- **Content**: the actual text to upload\n\nBefore choosing `text` or `chat`, fetch and follow [Chat vs Text Prompts](https://langfuse.com/docs/prompt-management/data-model#text-vs-chat-prompts). Do not default unrelated prompt flows to one prompt type.\n\nPrompts typically live in OpenAI message arrays, Anthropic system arguments, LangChain prompt templates, Vercel AI system/prompt fields, and raw multi-line strings near LLM calls.\n\n## 2. Convert templating and decide structure\n\n**Variable syntax:** Langfuse substitutes only double-brace `{{var}}`. Convert every single-brace form during upload — `{var}`, `${var}`, f-string `{var}`, `.format(var=...)`, and string concatenation all become `{{var}}`. Uploading `{var}` will silently fail to substitute.\n\n**Complex templates:** Langfuse has no conditionals, loops, or filters. If the code uses them (e.g. Jinja `{% if %}`/`{% for %}`), either pre-compute the value in code and pass a plain `{{variable}}` (recommended), or store the raw template and compile client-side — which loses Playground preview and UI experiments. See https://langfuse.com/docs/prompt-management/features/variables and the [external templating FAQ](https://langfuse.com/faq/all/using-external-templating-libraries).\n\n**What to make a variable vs. keep hardcoded:**\n\n| Make variable | Keep hardcoded |\n|---------------|----------------|\n| User-specific (`{{user_name}}`) | Output format instructions |\n| Dynamic content (`{{context}}`) | Safety guardrails |\n| Per-request (`{{query}}`) | Persona / personality |\n| Environment-specific (`{{company_name}}`) | Static examples |\n\n**Naming:** lowercase-hyphenated, feature-based (`document-summarizer`), hierarchical for related prompts (`support/triage`), prefix subprompts with `_` (`_base-personality`).\n\nBefore extracting subprompts, fetch and follow [Prompt Composability](https://langfuse.com/docs/prompt-management/features/composability). Use composition for the reuse and shared-maintenance cases described there, not merely to decompose one coherent prompt flow.\n\n## 3. Present the plan, then create and refactor\n\nWhen the user explicitly requests a migration and credentials work, treat that request as authorization to create prompts and refactor the call sites. Present the inventory and plan as a progress update, then continue.\n\nAsk only when a materially different design choice would change behavior, required credentials are unavailable, or destructive cleanup needs approval. Do not stop merely to confirm names, prompt types, or optional tracing.\n\nThen:\n\n- Create the prompts (label migrated prompts `production` — they\'re already live) and refactor call sites to fetch each prompt from Langfuse and compile its variables in. The SDK calls differ across Python and JS/TS — fetch the current docs: https://langfuse.com/docs/prompt-management/get-started\n- Fetch by the `production` label\n- If the codebase already has Langfuse tracing (decorators, an instrumented client, or manual spans), link prompts so you can see which version produced each response. See https://langfuse.com/docs/prompt-management/features/link-to-traces\n\n## 4. Verify\n\n- All prompts created with the `production` label; code fetches with `label="production"`\n- Variables and subprompts compile without errors\n- Application behavior is unchanged\n- Generations show the linked prompt in the UI (if tracing enabled)\n',+ },+ {+ uri: "skill://langfuse/langfuse/references/setting-up-evals.md",+ mimeType: "text/markdown",+ text: "---\nname: langfuse-setting-up-evals\ndescription: Set up evaluation by finding gaps across signal capture, monitoring, and evaluator metrics, then addressing the right one first.\n---\n\n# Setting up evals\n\nAct as a PM, analyst, teacher, and at the end, an executor. The bulk of the work consists of probing the user to really understand what they are looking for, and get a clear picture of what would serve them best. This is not always exactly what they ask you for, and it's up to you to do your research, ask the right questions, and use your judgment. Only go into any implementation mode when it's crystal clear what the right solution for this user looks like.\n\n## 1. Determine Online or Offline\n\nDetermine whether the user is looking for online or offline evaluation. If the goal is unclear, ask which decision the user is trying to make. If they need both, establish which one to address first.\n\n## 2. Establish the Metric Set\n\nIf the user supplied metrics, review them against the application, available data, and intended decision. A useful metric names a specific observable behavior, is measurable, and changes a decision. Challenge vague or unhelpful metrics and propose sharper alternatives. Read [Choosing what to evaluate](https://langfuse.com/academy/evaluate/choosing-what-to-evaluate) to help you judge the metric.\n\nIf the user did not supply metrics:\n\n- **Online:** The user likely doesn't know what they want. Execute the complete [User Feedback](user-feedback.md) workflow, backfill derivable signals where possible. Explain to the user why this is the right step to take. People often say they want evals without knowing what they actually need. This is one such case. You are completing their request by executing the user-feedback workflow.\n- **Offline:** inspect existing datasets and experiment results, then ask questions to determine:\n 1. is the user happy with their datasets, or do they want to evaluate something they don't have a dataset for?\n 2. if they want to use an existing dataset, do they already manually review experiment outputs on that dataset, and if yes, what do they look for?\n These questions should help narrow down what's important to the user and what you should focus on. Read [Choosing what to evaluate](https://langfuse.com/academy/evaluate/choosing-what-to-evaluate) to come up with high quality metrics to evaluate.\n\n\nPresent the resulting metric set as one table with: Priority; Status (existing or new); Metric; Source or evidence; Why it matters and what decision it informs; and Measurability. Revise it with the user and do not proceed until they explicitly confirm it.\n\n## 3. Implement and Verify\n\nAfter the metric set is confirmed:\n\n- **Online:** follow [Writing good evaluators](https://langfuse.com/academy/evaluate/writing-evaluators), then implement and verify the online evaluators.\n- **Offline:** confirm that an appropriate dataset exists. If not, execute [Dataset Construction](create-dataset.md). Then implement the evaluators and run the experiment.\n\nDo not default to an LLM-as-a-judge. When methods have significant trade-offs and none is clearly superior, present the options and let the user decide.\n\nFor evaluator functionality, use the unstable API endpoints.\n\n- Before creating an evaluator, fetch the observations matched by its target filter and confirm that they are correct and will not be scored twice.\n- Name the score after what is measured (`refusal`), not the evaluator (`refusal judge`).\n- If the project has a clear existing naming convention, follow it when creating a new evaluator and score. Consider naming best practices, but prioritize the existing project convention.\n- If an LLM-as-a-judge is the best fit, calibrate it on real examples before treating it as ready.\n- Share a link to each evaluator.\n",+ },+ {+ uri: "skill://langfuse/langfuse/references/skill-feedback.md",+ mimeType: "text/markdown",+ text: "---\nname: langfuse-skill-feedback\ndescription: Submit feedback about the Langfuse skill to its maintainers. Use when the skill's instructions are wrong, outdated, missing something, or could be improved.\n---\n\n# Skill Feedback\n\nUse this only for feedback about the skill itself, not Langfuse the product or a user's application data.\n\n1. Draft concise feedback using the following structure:\n - **Describe your idea or feedback** (required)\n A clear description of what went wrong or what could be improved. Include:\n - What the user was trying to do (as `goal`)\n - What the skill did vs what was expected\n - Any specific instructions that were incorrect or missing\n\n - **What would the ideal outcome look like?** (optional)\n What the correct behavior or guidance should be.\n\n If feedback targets existing skill, reference in `target`.\n2. If the user wants a reply, ask them to include an email address in `feedback`; use only an address they explicitly provide.\n3. Show every submitted field exactly as it will be sent and ask for explicit permission. Do not submit without approval.\n\n## Submission options\n\nUse the first available option, or the option the user prefers:\n\n1. **Authenticated Langfuse MCP, CLI, or Public API** — prefer the `submitFeedback` tool on the Langfuse MCP server. If it is unavailable, discover the current feedback operation with the Langfuse CLI schema/help and submit through the authenticated Public API. Do not ask users to paste credentials into chat.\n2. **Langfuse Docs MCP** — use its unauthenticated `submitFeedback` tool when no authenticated Langfuse interface is available.\n3. **GitHub issue or discussion** — if no MCP/CLI/API is available, or the user wants a public, trackable thread, provide a prefilled discussion link for the user to submit: `https://github.com/langfuse/skills/discussions/new?category=ideas-improvements&title=<url-encoded title>&body=<url-encoded body>`. Use `https://github.com/langfuse/skills/issues/new` if they prefer an issue.\n\nReport the Langfuse correlation ID or GitHub URL. If submission fails, give a safe error without exposing credentials or request bodies.\n",+ },+ {+ uri: "skill://langfuse/langfuse/references/user-feedback.md",+ mimeType: "text/markdown",+ text: "---\nname: langfuse-user-feedback\ndescription: Capture user feedback as Langfuse scores. Use when choosing or implementing explicit ratings, behavioral signals, conversation signals, or task outcomes.\n---\n\n# User Feedback\n\n## Workflow\n\n### 1. Determine What Feedback to Capture\n\nIf the user has asked for something specific, go with that. Otherwise, look at the application and **present a few UX options** for how feedback could work, then ask the user which they prefer before implementing.\n\nBefore presenting options, read [Capturing signals](https://langfuse.com/academy/monitoring/capturing-signals) and inspect the relevant application flow and existing scores. Present findings specific to the application rather than repeating examples from the docs.\n\nPresent every set of proposed signals or metrics as a table with:\n\n- Priority: P0, P1, ..\n- Status: whether the score or signal is already implemented or newly proposed\n- Signal name: Name the score after the observed signal, not the quality you hope it represents: draft_edited, not quality. Reuse the same name for the same signal throughout the application.\n- Explanation: what it indicates, its limitations and biases, and why it fits the application\n- Effort: XS-L\n- Implementation: a very short description such as “Langfuse evaluator” or “application code”. When a signal can be captured deterministically in both application code and a Langfuse code evaluator, present both as options.\n\nWhen multiple signals can be captured by one evaluator, group them and reflect this in the Implementation and Effort columns.\n\nWhen proposing an outcome signal, explicitly include the implementation complexity in its effort and explanation; unless the supporting workflow and data already exist or the expected value clearly justifies building them, recommend deferring it.\n\n### 2. Implement the Feedback\n\nRead the [user feedback loop guide](https://langfuse.com/guides/user-feedback-loop) and follow its links to the current SDK documentation before editing code.\n\n### 3. Verify\n\nTrigger each implemented signal and confirm that its score has the intended name, value, data type, and attachment.\n",+ },+ {+ uri: "skill://langfuse/langfuse/references/v4-project-migration.md",+ mimeType: "text/markdown",+ text: "---\nname: langfuse-v4-project-migration\ndescription: Prepare an application and Langfuse project for v4 by upgrading ingestion and API usage, migrating legacy evaluators, and moving exports to the enriched observation schema.\n---\n\n# Langfuse v4 project migration\n\n## Sources of truth\n\nFetch only the pages needed for the surfaces found:\n\n- [V4 upgrade guide](https://langfuse.com/faq/all/upgrade-to-langfuse-v4) and [compatibility](https://langfuse.com/docs/compatibility)\n- [SDK upgrade paths](https://langfuse.com/docs/observability/sdk/upgrade-path)\n- [Custom ingestion migration](https://langfuse.com/integrations/native/opentelemetry/migration-to-v4)\n- [Deprecated API migration](https://langfuse.com/faq/all/deprecated-api-migration)\n- [Evaluator migration](https://langfuse.com/faq/all/llm-as-a-judge-migration)\n- [Evaluation Rules](https://api.reference.langfuse.com/#tag/evaluationrules) and [Evaluators](https://api.reference.langfuse.com/#tag/unstableevaluators) APIs\n- Export migrations: [Blob Storage](https://langfuse.com/docs/api-and-data-platform/features/export-to-blob-storage#upgrade-path), [Mixpanel](https://langfuse.com/integrations/analytics/mixpanel#migrate-export-source), and [PostHog](https://langfuse.com/integrations/analytics/posthog#migrate-export-source)\n\nDiscover the current schemas before use.\n\n## Choose the execution mode\n\n- Confirm the target host and project before project reads or writes. Never request secrets in chat or commit them.\n- Prefer an available project interface. Otherwise use the [CLI](https://langfuse.com/docs/api-and-data-platform/features/cli), starting with `api __schema` and action `--help`.\n- Without project access, enter **code-only mode**: complete verifiable repository work, do not infer evaluator or export state, and mark every project-dependent result blocked.\n- Without codebase access, inspect the project and return an exact code handoff; do not mark repository work ready.\n\n## Migrate the codebase\n\n- Inventory every Langfuse SDK, integration, OTEL exporter, initialization site, lockfile, raw request, generated client, script, notebook, and CI call.\n- Upgrade to the latest stable SDK major required by the current docs and apply every applicable breaking change. Record both declared and resolved versions; update an existing lockfile.\n- Find every source of correlating attributes, including session and user IDs, tags, metadata, version, environment, and trace name; do not search only for removed SDK methods.\n- Put overall input/output on the root observation. Establish the documented propagation scope before observation-producing calls so every applicable child receives the attributes needed for filtering and aggregation, including the session ID on cost-bearing generations.\n- For raw `/api/public/ingestion`, use the current Langfuse SDK in Python or JS/TS. For other languages, use the language's native OpenTelemetry API and follow the custom-ingestion guide.\n- When replacing synchronous ingestion, assess buffering, retries, flushing, shutdown, and error propagation. Do not claim identical delivery semantics without verification.\n- For other deprecated APIs, migrate the path, parameters, pagination, filters, field groups, response parsing, and downstream consumer together using the deprecated-API guide.\n- Check self-hosted compatibility before replacing calls; report code targeting a v3 server as blocked on the server upgrade.\n\n## Migrate evaluators\n\n- Inventory all active evaluators and rules, inspect each referenced definition, and report inactive rules without reactivating them.\n- Check the Evaluators UI on the confirmed target host for active **Legacy** rows ([open in Langfuse Cloud](https://cloud.langfuse.com/project/~/evals)); public interfaces may not expose every legacy target.\n- For each inspected active legacy rule, record its filters, sampling, mappings, representative payloads, one observation or experiment successor, required code changes, and cutover test.\n- Label this a **project-verified contract** only when the rule and representative observations were inspected. Label code-only suggestions **candidate targets**, never project state.\n- Consolidate all required evaluator variables and filter attributes onto the one target observation. Observation evaluators cannot read siblings or children.\n- Create or update the successor disabled, validate it on newly ingested data, obtain approval before enabling it, compare scores and logs, then disable rather than delete the legacy rule.\n\n## Migrate exports\n\n- Inventory Blob Storage, Mixpanel, PostHog, and other sources in **Project Settings > Integrations**.\n- Follow each documented dual-source transition. Enabling dual output is additive; enriched-only stops legacy output and requires explicit downstream-owner confirmation.\n- Validate warehouse schemas, paths, joins, dashboards, transformations, alerts, and historical-data handling. Keep every consumer on one shape during dual mode.\n- Change only the source; preserve credentials, schedules, prefixes, formats, field groups, and secrets. Report the area `manual action` until downstream consumers are confirmed.\n\n## Validate and report\n\n- Test applicable hierarchy, root input/output, propagated attributes, public/release/environment behavior, API pagination and parsing, delivery semantics, and absence of deprecated calls.\n- Before production cutover, send representative traces from the migrated instrumentation to a non-production Langfuse project and inspect the resulting observations there. Mocked tests do not verify backend ingestion or project behavior.\n- On a session path, confirm the root and every applicable child observation carry the intended session ID and that session cost includes the cost-bearing children.\n- Re-read rules and integrations after writes. Preserve disabled legacy rules for rollback; never claim completion without checking the Evaluators UI on the target host for legacy rows.\n- The readiness report must contain exactly these seven rows, each marked `ready`, `changed`, `manual action`, or `blocked`: project access; SDK/instrumentation; trace evaluators; dataset evaluators; direct APIs; exports; verification/rollback.\n- In evaluator rows, separate contracts verified against project data from targets suggested only by code inspection. For every row not marked `ready`, include the blocker, next action, and a direct UI link when applicable.\n",+ },+ {+ uri: "skill://langfuse/langfuse/SKILL.md",+ mimeType: "text/markdown",+ text: '---\nname: langfuse\ndescription: >-\n Interact with Langfuse and access its documentation: tracing, monitoring, creating datasets, running experiments, and evaluating AI applications. Use when needing to (1) query or modify Langfuse data, (2) look up Langfuse documentation, concepts, integration guides, a feature or SDK usage, or (3) do any AI engineering task (AI observability, prompt engineering/management, evaluation and evaluator management, experimentation, dataset management, evaluation-driven CI/CD, feedback collection). Invoke it for tasks in this scope even when Langfuse is not configured or explicitly mentioned.\nallowed-tools:\n - WebFetch(domain:langfuse.com)\n - Bash(curl *langfuse.com/*)\n - Bash(npx langfuse-cli api __schema *)\n - Bash(npx langfuse-cli api * --help *)\n - Bash(npx langfuse-cli api * list *)\n - Bash(npx langfuse-cli api * get *)\n - Bash(bunx langfuse-cli api __schema *)\n - Bash(bunx langfuse-cli api * --help *)\n - Bash(bunx langfuse-cli api * list *)\n - Bash(bunx langfuse-cli api * get *)\n---\n\n# Langfuse\n\nThis skill helps you use Langfuse effectively across all common workflows: instrumenting applications, migrating prompts, debugging traces, and accessing data programmatically.\n\n## Core Principles\n\nFollow these principles for ALL Langfuse work:\n\n1. **Documentation First**: NEVER implement based on memory. Always fetch current docs before writing code (Langfuse updates frequently) See the section below on how to access documentation.\n2. **CLI for Data Access**: Use `langfuse-cli` when querying/modifying Langfuse data. See the section below on how to use the CLI.\n3. **Best Practices by Use Case**: Read the relevant reference below use-case-specific guidelines before asking the user for more details or implementing.\n4. **Use latest Langfuse versions**: Unless the user specified otherwise or there\'s a good reason, always use the latest version of Langfuse SDKs/APIs. Even if you\'re only creating a plan for another agent to execute, be explicit about the exact version to use.\n5. **If you guide the user through UI** and are unsure about a label or location, inspect the user’s screenshots or ask to see the relevant screen. Do not assume UI labels have the exact same names as API, SDK, or CLI fields.\n\n\n## Use case specific references\n\n- instrumenting an existing function/application: references/instrumentation.md\n- creating or getting to a good (evaluation) dataset to measure quality or test for regressions in AI systems: references/create-dataset.md\n- migrating prompts from a codebase into Langfuse: references/prompt-migration.md\n- creating a prompt or changing any part of an existing prompt, including small edits and debugging/tuning: references/prompt-engineering.md\n- setting up evals when the user needs to identify gaps across signal capture, monitoring, and evaluator metrics ("I have traces, how do I set up evals?"): references/setting-up-evals.md\n- capturing user feedback signals (explicit ratings, behavioral events, conversation signals, task outcomes) as scores: references/user-feedback.md\n- further tips on using the Langfuse CLI: references/cli.md\n- preparing a Langfuse project for the v4 platform migration: references/v4-project-migration.md\n- calibrating a new or existing LLM-as-a-Judge against labeled examples, iterating on its prompt, and deploying the approved judge: references/judge-calibration.md\n- systematic error analysis when requested directly or eval setup still lacks concrete failure modes after agent-led trace inspection: references/error-analysis.md\n- setting up CI/CD experiment gates with `langfuse/experiment-action`: references/ci-cd.md\n- optimizing the cost of an LLM application or AI agent: references/cost-optimization.md\n- submitting feedback about this skill: references/skill-feedback.md\n\n\n## 1. Langfuse API via CLI\n\nUse the `langfuse-cli` to interact with the full Langfuse REST API from the command line. Run via npx (no install required):\n\nStart by discovering the schema and available arguments:\n\n```bash\n# Discover all available resources\nnpx langfuse-cli api __schema\n\n# List actions for a resource\nnpx langfuse-cli api <resource> --help\n\n# Show args/options for a specific action\nnpx langfuse-cli api <resource> <action> --help\n```\n\n### Credentials\n\nSet environment variables before making calls:\n\n```bash\nexport LANGFUSE_PUBLIC_KEY=pk-lf-...\nexport LANGFUSE_SECRET_KEY=sk-lf-...\nexport LANGFUSE_BASE_URL=https://cloud.langfuse.com # example for EU cloud. For US cloud it\'s us.cloud.langfuse.com, and can also be a self-hosted URL. The server must always be specified in order to access Langfuse.\n```\nIf `LANGFUSE_BASE_URL` is used instead of `LANGFUSE_HOST`, run `export LANGFUSE_HOST="$LANGFUSE_BASE_URL"`.\nIf not set, ask the user to set them in their shell or a `.env` file. Keys are found in the Langfuse project under Settings -> API Keys; the user should create a project API key pair there. If they do not have a Langfuse account yet, share that they can create one for free at `https://langfuse.com/cloud`. Do not ask them to paste keys into chat for security reasons.\n\n### Detailed CLI Reference\n\nFor common workflows, tips, and full usage patterns, see [references/cli.md](references/cli.md).\n\n## 2. Langfuse Documentation\n\nThree methods to access Langfuse docs, in order of preference. **Always prefer your application\'s native web fetch and search tools** (e.g., `WebFetch`, `WebSearch`, `mcp_fetch`, etc.) over `curl` when available. The URLs and patterns below work with any fetching method — the `curl` examples are just illustrative.\n\nWhen working with self-hosted Langfuse, prefer the [API reference served by the deployment](https://langfuse.com/faq/all/self-hosting-api-reference) so it matches the installed version.\n\n### 2a. Documentation Index (llms.txt)\n\nFetch the full index of all documentation pages:\n\n```bash\ncurl -s https://langfuse.com/llms.txt\n```\n\nReturns a structured list of every doc page with titles and URLs. Use this to discover the right page for a topic, then fetch that page directly.\n\nAlternatively, you can start on `https://langfuse.com/docs` and explore the site to find the page you need.\n\n### 2b. Fetch Individual Pages as Markdown\n\nAny page listed in llms.txt can be fetched as markdown by appending `.md` to its path or by using `Accept: text/markdown` in the request headers. Use this when you know which page contains the information needed. Returns clean markdown with code examples and configuration details.\n\n```bash\ncurl -s "https://langfuse.com/docs/observability/overview.md"\ncurl -s "https://langfuse.com/docs/observability/overview" -H "Accept: text/markdown"\n```\n\n### 2c. Search Documentation\n\nWhen you need to find information across all docs and github issues/discussions without knowing the specific page:\n\n```bash\ncurl -s "https://langfuse.com/api/search-docs?query=<url-encoded-query>"\n```\n\nExample:\n\n```bash\ncurl -s "https://langfuse.com/api/search-docs?query=How+do+I+trace+LangGraph+agents"\n```\n\nReturns a JSON response with:\n\n- `query`: the original query\n- `answer`: a JSON string containing an array of matching documents, each with:\n - `url`: link to the doc page\n - `title`: page title\n - `source.content`: array of relevant text excerpts from the page\n\nSearch is a great fallback if you cannot find the relevant pages or need more context. Especially useful when debugging issues as all GitHub Issues and Discussions are also indexed. Responses can be large — extract only the relevant portions. Note that changelog posts may also surface here: use them only to confirm a feature exists, never to implement from — their examples may be outdated, so always implement from the docs and API/SDK reference.\n\n### Documentation Workflow\n\n1. Start with **llms.txt** to orient — scan for relevant page titles\n2. **Fetch specific pages** when you identify the right one\n3. Fall back to **search** when the topic is unclear and you want more context\n\n## Skill Feedback\n\nWhen the user expresses that something about this skill is not working as expected, gives incorrect guidance, is missing information, or could be improved — offer to submit feedback to the Langfuse skill maintainers. This includes when:\n\n- The skill gave wrong or outdated instructions\n- A workflow didn\'t produce the expected result\n- The user wishes the skill covered something it doesn\'t\n- The user explicitly says something like "this should work differently" or "this is wrong"\n\n**Do NOT trigger this** for issues with Langfuse itself (the product) — only for issues with this skill\'s instructions and behavior.\n\nWhen triggered, follow the process in [references/skill-feedback.md](references/skill-feedback.md).\n',+ },+ {+ uri: "skill://langfuse/migrate-to-langfuse/references/reingest-history.md",+ mimeType: "text/markdown",+ text: "---\nname: migrate-to-langfuse-reingest-history\ndescription: Write and run a script that extracts traces, scores, datasets, and prompts from a source system and reingests them into Langfuse with original timestamps.\n---\n\n# Reingest historical data into Langfuse\n\nBuild a one-off script: extract from the source, transform to the Langfuse observation model, reingest. Extract using the source vendor's **documented** export mechanism (SDK export client or documented API, as named in the matching migration guide on langfuse.com) — never guess REST endpoints or hostnames. If an export call fails with 404/405, stop and re-read the vendor guide instead of trying endpoint variants. Adapt the working implementation in the [project-to-project migration cookbook](https://langfuse.com/guides/cookbook/example_data_migration) — its OTLP reingestion, retry, and ID-mapping code applies to any source.\n\n## Transform target\n\nFetch [migrate custom ingestion to v4](https://langfuse.com/integrations/native/opentelemetry/migration-to-v4) for the current span format and attributes. Non-negotiables:\n\n- POST OTLP JSON to `/api/public/otel/v1/traces`, building the payload as plain JSON — the only path that preserves original `start_time`/`end_time`. Do not create the spans through an OTel or Langfuse SDK tracer: tracers assign new timestamps and random trace/span IDs, and offer no way to set either.\n- Destination IDs must be OTel-shaped: 32-hex-char trace IDs, 16-hex-char span IDs. Derive them deterministically from source IDs (e.g. a sha256 prefix) so every source record maps to one predictable destination ID for reconciliation and resume logic. Deterministic IDs do **not** make re-sends safe: Langfuse v4 does not reliably deduplicate same-ID re-ingests, a re-sent span can create a duplicate observation and inflate counts and cost (see the migration guide above). Idempotency comes from not re-sending: record each successfully sent source ID in a local state file and skip recorded IDs on every re-run. Verify after the sample run that the destination trace IDs match the computed ones. If they do not, the spans went through a tracer.\n- Map source spans to Langfuse observation types (generation, tool, agent, span) and set `gen_ai.usage.*` and model attributes so token usage and cost tracking work. Put user, session, tags, and metadata in the documented `langfuse.*` trace attributes; overall input/output goes on the root observation.\n\n## Migration order\n\nDependencies dictate order: score configs → custom model definitions → prompts (all versions, in version order, then labels) → observations (traces) → scores → datasets (items, then run items linked to migrated traces). Skip whatever the user excluded in the interview.\n\nNot migratable via API (tell the user; do not attempt): LLM-as-a-Judge evaluator configurations, custom dashboards, users/RBAC, project settings. Recreate evaluators in the destination instead, and re-run experiments rather than importing their scores.\n\n## Execution protocol\n\n1. **Cutoff filter** from the interview goes into the source extraction query, not post-hoc filtering. If the live cutover already happened, also bound the window at the cutover time — traces after it are already arriving in the destination, and reingesting them duplicates data.\n2. **Sample run first**: ~10 traces spanning the date range. Print direct links to the destination traces and stop until the user confirms they look correct — hierarchy, timestamps, token usage and cost, scores attached.\n3. **Full run**: paginate the source, batch the ingestion, print progress (`page X — N traces migrated, M failed`), and collect failures with their source IDs into a retry file instead of aborting. Resume from the state file: skip every source ID already recorded as sent, including the sample run — re-sending creates duplicate observations and is billed again.\n4. **Reconcile**: compare source vs destination counts per data type and report the table. Duplicate observations in the destination mean the same ID was sent twice - a migration bug to fix in the state-file skip, not a transient artifact that clears itself. Remind the user that reingested history is billed as new ingestion.\n\nHistorical cost may differ from what the source displayed: the destination recomputes cost from current model prices unless the source's actual cost is carried in the documented cost attributes.\n\nVerify credentials by presence only (`[ -n \"$LANGFUSE_SECRET_KEY\" ]`); never print secret values.\n",+ },+ {+ uri: "skill://langfuse/migrate-to-langfuse/SKILL.md",+ mimeType: "text/markdown",+ text: "---\nname: migrate-to-langfuse\ndescription: >-\n Migrate to Langfuse from another LLM observability/evals platform (LangSmith, Arize AX, Phoenix, Braintrust, Helicone, Promptfoo, ...). A one-off vendor switch: live tracing cutover plus transfer of historical traces, datasets, prompts, and evaluators. Not for anything Langfuse-to-Langfuse — SDK or version upgrades, project/region moves, self-hosted to Cloud — and not for moving in-code prompts into Langfuse; the main langfuse skill covers those.\nallowed-tools:\n - WebFetch(domain:langfuse.com)\n - Bash(curl *langfuse.com/*)\n---\n\n# Migrate to Langfuse\n\nYou are running a one-off vendor switch to Langfuse. Interview first, plan second, execute only after confirmation. Never implement from memory — fetch the current Langfuse docs listed below at execution time.\n\n## 1. Interview (always, before any code)\n\nAsk only what you cannot detect. Inspect the repo first: dependencies and imports usually reveal the source platform (`langsmith`, `arize`, `phoenix`, `braintrust`, `openinference`, old `langfuse` pins).\n\n1. **Source** — which platform and version? Cloud or self-hosted? What access exists: API keys, a database connection, or only the application repo?\n2. **Destination** — Langfuse Cloud (which region) or self-hosted? Verify credentials by presence only (e.g. `[ -n \"$LANGFUSE_SECRET_KEY\" ]`); never ask for secrets in chat, and never access `.env` contents or key values by any means — no shell commands that echo them, no reading `.env` with file tools, no grepping for values. Detect which variable names are set, nothing more.\n3. **Scope** — which of: live tracing cutover, historical traces (including their scores, score configs, and custom model definitions), datasets, prompts, evaluators/LLM judges, experiment code. For history: everything or from a cutoff date? Ask what else exists in the source account (dashboards, alerts, annotation queues, saved views, ...). The repo cannot reveal server-side assets. Anything outside the supported list above is not covered by this skill: name it in the scope table as such, so the user plans to recreate it manually instead of discovering the gap after cutover.\n4. **Mode** — plan only (a written runbook) or execute?\n\nIf the request contradicts the repository (it names a platform that is not present, or asks to upgrade an SDK that is not installed), stop and reconcile with the user first — never reinterpret the request as a different migration and start executing it.\n\nPresent the resulting plan (scope table plus order of operations) and get the user's confirmation before changing any file. However imperative the request sounds, it does not authorize removing or replacing the existing vendor's instrumentation before the plan is confirmed.\n\n## 2. Defaults to apply\n\n- **Live cutover first.** Historical data migration is optional and billed as new ingestion in Langfuse — say so before migrating history. If the source already emits OTel/OpenInference spans, add Langfuse as a second exporter on the existing instrumentation instead of re-instrumenting; the old destination is dropped only when the parallel-run window below closes.\n- **Datasets and prompts copy. Evaluators and judges are recreated.** Experiment code is rewritten and re-run against the migrated dataset; never import old experiment scores.\n- **Test on a sample before the full run.** Migrate ~10 traces first, link the destination traces, and wait for the user to confirm they look correct before migrating everything.\n- **Show progress** (pages/counts) during long runs and make re-runs resumable: track sent source IDs in a state file and skip them, re-sending the same IDs creates duplicate observations (see references/reingest-history.md).\n- **Parallel-run window.** Keep the old exporter until the user confirms the destination data, then remove it in a separate, explicit step. This binds the implementation, not just the plan: after your edits, the old vendor's exporter/instrumentation must still be present and active. Before showing a diff, re-check that no vendor import, register call, exporter endpoint, or exporter env var was deleted or repointed. Implement parallel export as **one instrumentation with two exporters** (both span processors on the same tracer provider) — never instrument the application twice. Duplicated generations and double-counted costs in the destination are the symptom of a second capture path.\n\n## 3. Route by scenario\n\nFetch and follow; do not reproduce their content:\n\n| Scenario | Source of truth |\n| --- | --- |\n| Vendor with a published guide (LangSmith, Arize AX, Phoenix, Braintrust, Helicone, Promptfoo, ...) | `https://langfuse.com/resources/engineering/migrate-from-<vendor>.md` — check `https://langfuse.com/llms.txt` for the current list |\n| Vendor without a published guide, or historical data from any vendor | references/reingest-history.md, plus the vendor's own export API docs |\n\nHistorical-trace reingestion is never improvised: read references/reingest-history.md in full before writing or running any reingestion code, even when a vendor guide exists. The vendor guide covers extraction; the reference governs reingestion — deterministic IDs, raw OTLP payloads, original timestamps, the cutover-time bound, and the warning that imported history is billed as new ingestion.\n\nIf the source turns out to be another Langfuse deployment (an older version, a different project, region, or host), stop — that is not a vendor switch. Point the user to the main langfuse skill (its SDK-upgrade and v4-migration references) and the [project-to-project migration cookbook](https://langfuse.com/guides/cookbook/example_data_migration).\n\nFor the live instrumentation cutover, use the integration guide matching the user's framework or SDK from https://langfuse.com/llms.txt. For deep Langfuse-side work (prompt creation, dataset APIs, experiment code), defer to the main langfuse skill if installed.\n\n## 4. Verify and close\n\n- Sample check confirmed by the user in the destination UI before the full run.\n- Before asking the user to confirm a live-cutover sample, audit it yourself via the Langfuse API or CLI: one trace per request; the root span named after the agent/app (and the trace named after the root); every span type present (generations **and** tool/chain spans); each LLM call captured exactly once; token usage and cost populated. Only then send the user the trace links.\n- Counts match per migrated data type (source vs destination); report a summary table including failures.\n- Live traffic arrives in the destination; the old exporter is removed only after the parallel-run window the user chose.\n- The close-out summary accounts for every scope-table row from the interview, including the ones marked skipped or not covered — nothing is silently dropped.\n",+ },+];diff --git a/packages/langfuse-skills/src/mcp.d.ts b/packages/langfuse-skills/src/mcp.d.tsnew file mode 100644index 0000000000..2f995be5d0--- /dev/null+++ b/packages/langfuse-skills/src/mcp.d.ts@@ -0,0 +1,14 @@+export declare const LANGFUSE_MCP_SKILLS: readonly {+ uri: string;+ frontmatter: {+ name: string;+ description: string;+ [key: string]: unknown;+ };+ resources: { uri: string; digest: string; size: number }[];+}[];++export declare const LANGFUSE_MCP_RESOURCES: readonly ({+ uri: string;+ mimeType: string;+} & ({ text: string; blob?: never } | { blob: string; text?: never }))[];diff --git a/packages/langfuse-skills/src/mcp.js b/packages/langfuse-skills/src/mcp.jsnew file mode 100644index 0000000000..8d946b2add--- /dev/null+++ b/packages/langfuse-skills/src/mcp.js@@ -0,0 +1,6 @@+const {+ LANGFUSE_MCP_SKILLS,+ LANGFUSE_MCP_RESOURCES,+} = require("./generated/mcp-skills.js");++module.exports = { LANGFUSE_MCP_SKILLS, LANGFUSE_MCP_RESOURCES };diff --git a/pnpm-lock.yaml b/pnpm-lock.yamlindex 4d6054f9f7..eea48eb5e5 100644--- a/pnpm-lock.yaml+++ b/pnpm-lock.yaml@@ -299,6 +299,9 @@ importers:prettier:specifier: ^3.8.3version: 3.8.3+ yaml:+ specifier: ^2.9.0+ version: 2.9.0packages/native:devDependencies:@@ -717,6 +720,9 @@ importers:'@radix-ui/react-tooltip':specifier: ^1.2.8version: 1.2.8(@types/[email protected](@types/[email protected]))(@types/[email protected])([email protected]([email protected]))([email protected])+ '@repo/langfuse-skills':+ specifier: workspace:*+ version: link:../packages/langfuse-skills'@scalar/api-reference':specifier: 1.66.1version: 1.66.1([email protected]([email protected]([email protected]))([email protected]))([email protected])([email protected])([email protected])([email protected])diff --git a/web/package.json b/web/package.jsonindex bafcb5d79f..3dea6d1ccd 100644--- a/web/package.json+++ b/web/package.json@@ -96,6 +96,7 @@"@radix-ui/react-tabs": "^1.1.13","@radix-ui/react-toggle": "^1.1.10","@radix-ui/react-tooltip": "^1.2.8",+ "@repo/langfuse-skills": "workspace:*","@scalar/api-reference": "1.66.1","@sentry/nextjs": "^11.8.0","@t3-oss/env-nextjs": "^0.11.1",diff --git a/web/src/features/mcp/server/mcpServer.ts b/web/src/features/mcp/server/mcpServer.tsindex 1f1dbc6540..eaf8fde971 100644--- a/web/src/features/mcp/server/mcpServer.ts+++ b/web/src/features/mcp/server/mcpServer.ts@@ -23,6 +23,7 @@ import { formatErrorForUser } from "../core/error-formatting";import type { ServerContext } from "../types";import type { ToolDefinition } from "../core/define-tool";import { toolRegistry } from "./registry";+import { registerSkillResources } from "./skills/resources";import { contextWithLangfuseProps, logger } from "@langfuse/shared/src/server";import { context as otelContext } from "@opentelemetry/api";@@ -36,6 +37,7 @@ const MCP_SERVER_INSTRUCTIONS = ["Use this server for project-scoped Langfuse data and actions such as prompts, datasets, scores, comments, metrics, observations etc.","Inspect the available tools and their schemas dynamically; do not assume a fixed tool list.","For conceptual Langfuse product guidance, SDK/API documentation, instrumentation help, or prompt-migration guidance, prefer the Langfuse docs MCP server or installed Langfuse agent skills when they are available.",+ "Discover bundled Langfuse skills with skills/list, retrieve metadata with skills/get, and read skill files with resources/read.",];const MCP_FEEDBACK_INSTRUCTION ="To send feedback about Langfuse skills, MCP tools, CLI, docs, or public API, ask the user for permission and show the exact feedback payload; if they want a reply, include only an email address they explicitly provide in the feedback text; exclude secrets, customer/project data, trace payloads, and unrelated context, then call submitFeedback.";@@ -73,11 +75,15 @@ export function createMcpServer(context: ServerContext): Server {{capabilities: {tools: {},+ resources: {},+ extensions: { "io.modelcontextprotocol/skills": {} },},instructions: getMcpServerInstructions(),},);+ registerSkillResources(server);+// ListTools handler - dynamically load from registryserver.setRequestHandler(ListToolsRequestSchema, async () => {const tools = await toolRegistry.getToolDefinitions(context);diff --git a/web/src/features/mcp/server/skills/resources.ts b/web/src/features/mcp/server/skills/resources.tsnew file mode 100644index 0000000000..248ad037cf--- /dev/null+++ b/web/src/features/mcp/server/skills/resources.ts@@ -0,0 +1,76 @@+import { z } from "zod";+import {+ ErrorCode,+ ListResourcesRequestSchema,+ ListResourceTemplatesRequestSchema,+ McpError,+ ReadResourceRequestSchema,+ RequestSchema,+} from "@modelcontextprotocol/sdk/types.js";+import {+ LANGFUSE_MCP_SKILLS,+ LANGFUSE_MCP_RESOURCES,+} from "@repo/langfuse-skills/mcp";+import type { createMcpServer } from "../mcpServer";++const listSchema = RequestSchema.extend({+ method: z.literal("skills/list"),+ params: z.object({ cursor: z.string().optional() }).optional(),+});+const getSchema = RequestSchema.extend({+ method: z.literal("skills/get"),+ params: z.object({ uri: z.string() }),+});++const cache = {+ resultType: "complete",+ ttlMs: 0,+ cacheScope: "private",+} as const;++const notFound = () =>+ new McpError(ErrorCode.InvalidParams, "Unknown skill or resource");++export function registerSkillResources(+ server: ReturnType<typeof createMcpServer>,+) {+ const list = (cursor: string | undefined) => {+ if (cursor !== undefined) {+ throw new McpError(ErrorCode.InvalidParams, "Invalid skills cursor");+ }+ return { ...cache, skills: LANGFUSE_MCP_SKILLS };+ };++ server.setRequestHandler(listSchema, async (request) =>+ list(request.params?.cursor),+ );+ server.setRequestHandler(getSchema, async (request) => {+ const official = LANGFUSE_MCP_SKILLS.find(+ (skill) => skill.uri === request.params.uri,+ );+ if (!official) throw notFound();+ return { ...cache, skill: official };+ });+ server.setRequestHandler(ListResourcesRequestSchema, async (request) => {+ const result = list(request.params?.cursor);+ return {+ ...cache,+ resources: result.skills.map((skill) => ({+ uri: skill.uri,+ name: skill.frontmatter.name,+ description: skill.frontmatter.description,+ mimeType: "text/markdown",+ })),+ };+ });+ server.setRequestHandler(ListResourceTemplatesRequestSchema, async () => ({+ ...cache,+ resourceTemplates: [],+ }));+ server.setRequestHandler(ReadResourceRequestSchema, async (request) => {+ const uri = request.params.uri;+ const resource = LANGFUSE_MCP_RESOURCES.find((file) => file.uri === uri);+ if (!resource) throw notFound();+ return { ...cache, contents: [resource] };+ });+}