Skip to content

πŸͺ’ Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

refactor(assistant): Restructure skill embedding (#16594)

Ben Bachem76234f13669b
diff --git a/.prettierignore b/.prettierignore
index 98f0ab19d3..05f658f671 100644
--- a/.prettierignore
+++ b/.prettierignore
@@ -1,9 +1,5 @@
# Generated code
packages/shared/prisma/generated
-# Verbatim upstream skill markdown; must stay byte-identical for
-# scripts/in-app-agent/sync-raw-skills.mjs --check
-packages/shared/src/in-app-agent/server/skills/generated/raw
-
# Local, untracked screenshot/review archives
.screenshots-review
diff --git a/packages/langfuse-skills/package.json b/packages/langfuse-skills/package.json
new file mode 100644
index 0000000000..d573ff17f5
--- /dev/null
+++ b/packages/langfuse-skills/package.json
@@ -0,0 +1,24 @@
+{
+ "name": "@repo/langfuse-skills",
+ "version": "0.0.0",
+ "private": true,
+ "engines": {
+ "node": "24"
+ },
+ "main": "./src/index.js",
+ "types": "./src/index.d.ts",
+ "exports": {
+ ".": {
+ "types": "./src/index.d.ts",
+ "import": "./src/index.js",
+ "require": "./src/index.js"
+ }
+ },
+ "scripts": {
+ "sync": "node scripts/sync-skills.mjs",
+ "sync:check": "node scripts/sync-skills.mjs --check"
+ },
+ "devDependencies": {
+ "prettier": "^3.8.3"
+ }
+}
diff --git a/packages/langfuse-skills/scripts/sync-skills.mjs b/packages/langfuse-skills/scripts/sync-skills.mjs
new file mode 100644
index 0000000000..a065553fb4
--- /dev/null
+++ b/packages/langfuse-skills/scripts/sync-skills.mjs
@@ -0,0 +1,158 @@
+#!/usr/bin/env node
+
+import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { resolve } from "node:path";
+import process from "node:process";
+import { format as formatWithPrettier } from "prettier";
+
+const packageRoot = resolve(new URL("..", import.meta.url).pathname);
+const generatedPath = resolve(packageRoot, "src/generated/skills.js");
+const sourceApiUrl =
+ "https://api.github.com/repos/langfuse/skills/contents/skills/langfuse/references?ref=main";
+const isCheckMode = process.argv.includes("--check");
+
+const getGitHubHeaders = () => ({
+ Accept: "application/vnd.github+json",
+ Authorization: process.env.GITHUB_TOKEN
+ ? `Bearer ${process.env.GITHUB_TOKEN}`
+ : undefined,
+ "User-Agent": "langfuse-sync-skills",
+});
+
+const fetchResponse = async (url) => {
+ const response = await fetch(url, { headers: getGitHubHeaders() });
+
+ if (!response.ok) {
+ throw new Error(`Failed to fetch ${url}: ${response.status} ${response.statusText}`);
+ }
+
+ return response;
+};
+
+const listRemoteMarkdownFiles = async () => {
+ const response = await fetchResponse(sourceApiUrl);
+ const entries = await response.json();
+
+ if (!Array.isArray(entries)) {
+ throw new Error("Unexpected GitHub API response while listing remote skills");
+ }
+
+ return entries
+ .filter(
+ (entry) =>
+ entry.type === "file" &&
+ typeof entry.name === "string" &&
+ entry.name.endsWith(".md") &&
+ typeof entry.download_url === "string",
+ )
+ .sort((left, right) => left.name.localeCompare(right.name));
+};
+
+const parseSkill = (fileName, markdown) => {
+ const frontmatterMatch = markdown.match(/^---\n([\s\S]*?)\n---\n?/);
+ if (!frontmatterMatch) {
+ throw new Error(`${fileName} is missing frontmatter`);
+ }
+
+ const metadata = {};
+ let currentKey;
+ for (const line of frontmatterMatch[1].split("\n")) {
+ const match = line.match(/^([a-z_]+):\s*(.*)$/);
+ if (match) {
+ currentKey = match[1];
+ metadata[currentKey] = match[2];
+ } else if (currentKey === "description" && line.trim()) {
+ metadata.description += ` ${line.trim()}`;
+ }
+ }
+
+ for (const key of ["name", "description"]) {
+ if (
+ metadata[key]?.startsWith("'") &&
+ metadata[key]?.endsWith("'")
+ ) {
+ metadata[key] = metadata[key].slice(1, -1).replaceAll("''", "'");
+ }
+ }
+ if (
+ !metadata ||
+ typeof metadata !== "object" ||
+ Array.isArray(metadata) ||
+ typeof metadata.name !== "string" ||
+ typeof metadata.description !== "string"
+ ) {
+ throw new Error(`${fileName} has invalid skill metadata`);
+ }
+
+ const instructions = markdown.slice(frontmatterMatch[0].length).trim();
+ if (!instructions) {
+ throw new Error(`${fileName} is missing instructions`);
+ }
+
+ return {
+ name: metadata.name,
+ description: metadata.description,
+ instructions,
+ };
+};
+
+const getExpectedSkills = async () => {
+ const remoteFiles = await listRemoteMarkdownFiles();
+ return Promise.all(
+ remoteFiles.map(async ({ name, download_url: downloadUrl }) => {
+ const markdown = await (await fetchResponse(downloadUrl)).text();
+ return parseSkill(name, markdown);
+ }),
+ );
+};
+
+const renderGeneratedModule = async (skills) =>
+ formatWithPrettier(
+ `// This file is generated by packages/langfuse-skills/scripts/sync-skills.mjs.
+// Do not edit it manually.
+
+exports.LANGFUSE_SKILLS = [
+${skills
+ .map(
+ (skill) =>
+ ` ${JSON.stringify(skill)},`,
+ )
+ .join("\n")}
+];
+`,
+ { parser: "babel" },
+ );
+
+const main = async () => {
+ const expectedSkills = await getExpectedSkills();
+ const expectedModule = await renderGeneratedModule(expectedSkills);
+
+ if (isCheckMode) {
+ try {
+ if (readFileSync(generatedPath, "utf8") !== expectedModule) {
+ throw new Error("Generated skill catalog differs from upstream");
+ }
+ } catch (error) {
+ if (error.code === "ENOENT") {
+ throw new Error("Generated skill catalog is missing");
+ }
+ throw error;
+ }
+
+ console.log(
+ `Generated Langfuse skills are in sync: ${expectedSkills
+ .map(({ name }) => name)
+ .join(", ")}`,
+ );
+ return;
+ }
+
+ mkdirSync(resolve(packageRoot, "src/generated"), { recursive: true });
+ writeFileSync(generatedPath, expectedModule);
+ console.log(`Synced ${expectedSkills.length} Langfuse skills.`);
+};
+
+main().catch((error) => {
+ console.error(error);
+ process.exit(1);
+});
diff --git a/packages/langfuse-skills/src/generated/skills.js b/packages/langfuse-skills/src/generated/skills.js
new file mode 100644
index 0000000000..4317bc9fd6
--- /dev/null
+++ b/packages/langfuse-skills/src/generated/skills.js
@@ -0,0 +1,103 @@
+// This file is generated by packages/langfuse-skills/scripts/sync-skills.mjs.
+// Do not edit it manually.
+
+exports.LANGFUSE_SKILLS = [
+ {
+ name: "langfuse-ci-cd",
+ description:
+ "Set up or extend agent regression checks / gating in GitHub Actions CI/CD using `langfuse/experiment-action`.",
+ instructions:
+ "# Langfuse CI/CD\n\n## Checklist\n\n- [ ] Follow the [CI/CD docs page](https://langfuse.com/docs/evaluation/experiments/experiments-ci-cd) and the [langfuse/experiment-action README](https://github.com/langfuse/experiment-action)\n- [ ] Inspect whether the local repository is a GitHub repo. If not, `langfuse/experiment-action` is not applicable. Follow the guide for other [CI/CD systems](https://langfuse.com/docs/evaluation/experiments/experiments-ci-cd#other-cicd-systems) instead\n- [ ] Ask the user which [evaluators and run evaluators](https://langfuse.com/docs/evaluation/experiments/experiments-via-sdk#evaluators) they want to set up\n- [ ] Ask the user if and when yes which regression thresholds they want to set\n- [ ] Confirm dataset existence and shape of the dataset items before writing code with the Langfuse CLI (see `references/cli.md`)\n - `langfuse-cli api datasets list`\n - `langfuse-cli api dataset-items list --dataset-name <dataset name> --limit 5`\n- [ ] Propose the user to verify the check by actually running the new CI check (e.g. by creating a pull request)\n- [ ] If the evaluator task uses a third-party dependency, add the necessary CI steps to install them\n\n### GitHub specific checklist\n- [ ] Ask the user how they want the [workflow to be triggered](https://docs.github.com/en/actions/reference/workflows-and-actions/events-that-trigger-workflows)\n- [ ] If available, use the `gh` CLI to check secret existence / set secrets for:\n - Langfuse credentials\n - Credentials required by the evaluator task (e.g. OpenAI or Anthropic API keys)\n\n## Common Issues\n\n| Issue | Solution |\n|-------|----------|\n| `gh` is missing or not authenticated | Install the GitHub CLI if needed, then run `gh auth status` and `gh auth login` before using `gh secret` or `gh workflow` commands. |\n| Local Langfuse environment variables are not set | Set `LANGFUSE_PUBLIC_KEY`, `LANGFUSE_SECRET_KEY`, and `LANGFUSE_HOST` locally before using `langfuse-cli`; do not ask the user to paste secret values into chat. |\n| Workflow secrets or action inputs are wrong | Verify `LANGFUSE_PUBLIC_KEY`, `LANGFUSE_SECRET_KEY`, `langfuse_base_url`, and provider secrets exist in the target repo/environment and are passed to the action step. |\n| Forked PR cannot access secrets | GitHub restricts secret access for forked PRs. Document the limitation or choose a trusted trigger such as internal PR, trusted-branch `push`, or `workflow_dispatch`. |\n| No default/base branch exists | Create an initial empty commit on the intended default branch before trying to verify a PR-triggered workflow. |\n| Script fails reading dataset fields | Re-inspect the dataset items with the Langfuse CLI, check `input`, `expected_output`, and metadata, and extract fields from object-shaped outputs explicitly. |",
+ },
+ {
+ name: "langfuse-cli",
+ description:
+ "Langfuse CLI usage reference β€” install, resource/action discovery, credentials, and common usage tips for `langfuse-cli`. Use for further tips on using the Langfuse CLI.",
+ instructions:
+ "# Langfuse CLI Reference\n\nDocumentation: https://langfuse.com/docs/api-and-data-platform/features/cli\n\n## Install\n\n```bash\n# Run directly (recommended)\nnpx langfuse-cli api <resource> <action>\nbunx langfuse-cli api <resource> <action>\n\n# Or install globally\nnpm i -g langfuse-cli\nlangfuse api <resource> <action>\n```\n\n## Discovery\n\n```bash\n# List all resources and auth info\nlangfuse api __schema\n\n# List actions for a resource\nlangfuse api <resource> --help\n\n# Show args/options for a specific action\nlangfuse api <resource> <action> --help\n\n# Preview the curl command without executing\nlangfuse api <resource> <action> --curl\n```\n\n## Credentials\n\nSet environment variables:\n\n```bash\nexport LANGFUSE_PUBLIC_KEY=pk-lf-...\nexport LANGFUSE_SECRET_KEY=sk-lf-...\nexport LANGFUSE_BASE_URL=https://cloud.langfuse.com \n```\n\n## Tips\n\n- Use `--json` for machine-readable JSON output\n- Use `--curl` to preview the HTTP request without executing\n- All list commands support filtering β€” check `<resource> <action> --help` for available options\n- Prefer `observations` over `legacy-observations-v1s` β€” `observations` is the modern high-performance endpoint (cursor pagination, selective field groups); `legacy-observations-v1s` is the deprecated v1\n- Prefer `metrics` over `legacy-metrics-v1s` for the same reason\n- Prefer `scores` over `legacy-score-v1s` for list/get operations\n- Always query via `observations`, not `traces`, even when the user phrases the request in terms of traces β€” the `traces` endpoints are outdated (add `--trace-id` to scope to a known trace). See the [Observations API docs](https://langfuse.com/docs/api-and-data-platform/features/observations-api) for the v1 β†’ v2 mapping.\n- Pagination: legacy v1 endpoints use `--limit` and `--page`; modern endpoints (`observations`, `metrics`, `scores`) use cursor-based pagination β€” pass `--limit`, then thread `meta.cursor` from the response into the next request's `--cursor`",
+ },
+ {
+ name: "langfuse-dataset-construction",
+ description:
+ "Collaborative Langfuse dataset creation workflow. Use when the user needs to create, design, seed, reshape, review, or upload a Langfuse dataset or dataset version; especially when they need a minimal but complete dataset, e.g. for quality checks or avoiding regression.",
+ instructions:
+ "# Langfuse Dataset Construction\n\n## Primary Guide\n\nFollow the [Langfuse Academy datasets guide](https://langfuse.com/academy/datasets) as the human-readable source of truth for dataset design; fetch current docs before implementation.\n\nUse CLI schema/help discovery for current Langfuse API mechanics.\n\n## Workflow\n\nGuide the user through dataset creation as an interview, proposal, approval, implementation loop:\n\n1. Read the primary guide and relevant current Langfuse docs.\n2. Clarify the problem the user is facing, interviewing them if context is thin, to decide the goal, release, or evaluation decision the dataset should support.\n3. Inspect available context: the user's goal, the application path or codebase when available, existing datasets, traces, prompts, scores, monitors, user feedback, tickets, expert examples, existing eval and dataset assets.\n4. Interview the user on available sources beyond production traces.\n5. For each step going forward, propose a direction and get the user's approval, specifically:\n - Dataset distribution dimensions (propose and get user input).\n - Item schema: input, expected output, metadata (propose and get user input).\n6. Generate the first minimal draft of items against the approved schema and distribution, then present it for approval.\n7. Heavily prompt the user to review expected outputs, as AI-generated expected outputs cannot be considered ground truth. Send a link with instructions to review.\n\n## Critical Rules\n- Do not create, upsert, reshape, or upload a live Langfuse dataset until the user has approved the dataset goal, source mix, item schema, and first minimal draft, unless the user already gave those details and explicitly asked for immediate mutation.\n- Design the smallest complete dataset version that can serve the goal. Prefer a minimal reviewable v0 over broad coverage. A first draft is usually 5-12 high-signal items unless the user asks for a different size.\n- Keep `input`, `expectedOutput`, and `metadata` responsibilities separate. Put additional information, notes, and comments into `metadata`, not into `input` or `expectedOutput`.",
+ },
+ {
+ name: "langfuse-error-analysis",
+ description:
+ 'Deep-dive error analysis of an LLM pipeline or AI application using Langfuse traces. Use this skill whenever the user wants to understand why their AI system is producing bad outputs, where their pipeline is failing, or how to categorise and label failures into a taxonomy and decide what to prioritise fixing. Also trigger for "review my traces", "my outputs look wrong", "help me debug my LLM app", "I want to analyse errors", "build a failure taxonomy", or "what\'s going wrong with my pipeline".',
+ instructions:
+ "# Error Analysis\n\n## Primary Guide\n\n**1. Fetch the guide in this blogpost**\n\nhttps://langfuse.com/guides/cookbook/error-analysis-llm-applications.md\n\nIf fetch is not available query for langfuse.com error analysis guide\n\nRead it in full. It defines the authoritative 5-step process (sample selection β†’ open coding β†’ clustering β†’ labelling β†’ deciding what to fix).\n\n**2. Guide the user through this step by step**\n\nYou as a coding agent and the user go through this together to perform a full error analysis with their data in Langfuse. Set up the scores, representative sample, and annotation queue for the user, and provide direct UI links wherever their action is required. Be proactive and explain what is happening in plain language without reciting internal step numbers or jargon.\n\n## Rules CRITICAL\n- Perform interactions with the user's Langfuse instance yourself rather than telling the user to do them.\n- The user performs the open-coding review and supplies human labels; do not do this on their behalf.\n- Where a step needs the user's judgment or input, pause and ask before acting.\n- Use charts where possible to display data.\n\n---\n\n## Langfuse Implementation Notes\n\nThe guide describes the process. These notes cover the Langfuse-specific mechanics required to execute it.\n\n### Credentials\n\nMake sure Langfuse credentials are available before starting β€” a public key (`pk-lf-...`), a secret key (`sk-lf-...`), and the host (e.g. `https://cloud.langfuse.com` for EU, `https://us.cloud.langfuse.com` for US, `https://jp.cloud.langfuse.com` for JP, or a self-hosted URL). If they aren't configured, ask the user to set them β€” do not ask them to paste secret values into chat.\n\nVerify you can actually reach the user's project before proceeding. If access fails, stop and ask the user to check their credentials and host.\n\n### Annotation target: OBSERVATION versus TRACE\n\n> **CRITICAL:** In OpenTelemetry-instrumented apps, trace-level `input`/`output` can be null β€” content often lives in a GENERATION observation. In that case, add the GENERATION observation (not the trace) to the annotation queue, so the content being reviewed is actually visible.\n\n### Annotation queues\n\n> **CRITICAL:** Queues cannot be updated or deleted after creation. Create score configs first, then the queue with all config IDs. To add new configs later, create a new queue.\n\n\n**Always give the user a direct link immediately after creating a queue:**\n\n| Host | URL pattern |\n|------|-------------|\n| EU cloud | `https://cloud.langfuse.com/project/<projectId>/annotation-queues/<queueId>` |\n| US cloud | `https://us.cloud.langfuse.com/project/<projectId>/annotation-queues/<queueId>` |\n| Self-hosted | `<LANGFUSE_BASE_URL>/project/<projectId>/annotation-queues/<queueId>` |\n\nInstruction to give: *\"Please open code the first ~50 examples. For each trace, write what you observe in the `open_coding` field (describe behaviour, don't diagnose root causes), then set `pass_fail_assessment` to Pass or Fail.\"*\n\n\n### Prompt fixes\n\nWhen a category warrants a prompt fix, always offer the user two options:\n1. Create it as a versioned prompt in Langfuse (tracked, usable via the prompt API)\n2. Draft the specific text change for them to review and apply\n\n### Setup evaluators\n\nWhen a category warrants an evaluator, build it following `references/setting-up-evals.md`.\n\n\n### Common gotchas\n\n| Mistake | Fix |\n|---------|-----|\n| Annotating the trace instead of the observation | Target the GENERATION observation, not the trace, when content lives there |\n| Creating a score config without checking existing ones | Check existing score configs first; they can't be deleted |\n| Queue created before score configs | Create configs β†’ collect IDs β†’ create queue |\n| Requesting too many traces at once | Page size is capped (max 100); paginate to get more |\n| No rate limiting on bulk queue item creation | Space out requests to avoid hitting rate limits (429) |",
+ },
+ {
+ name: "langfuse-observability",
+ description:
+ "Instrument LLM applications with Langfuse tracing. Use when setting up Langfuse, adding observability to LLM calls, or auditing existing instrumentation.",
+ instructions:
+ "# Langfuse Observability\n\nInstrument LLM applications with Langfuse tracing, following best practices and tailored to your use case.\n\n## Workflow\n\n### 1. Assess Current State\n\nCheck the project:\n\n- Is Langfuse SDK installed?\n- What LLM frameworks are used? (OpenAI SDK, LangChain, LlamaIndex, Vercel AI SDK, etc.)\n- Is there existing instrumentation?\n\n**No integration yet:** Set up Langfuse using a framework integration if available. Integrations capture more context automatically and require less code than manual instrumentation.\n\n**Integration exists:** Audit against baseline requirements below.\n\n### 2. Verify Baseline Requirements\n\nEvery trace should have these fundamentals:\n\n| Requirement | Check | Why |\n| ------------------------- | ---------------------------------------------------------------------------------------- | ------------------------------------------------------ |\n| Model name | Is the LLM model captured? | Enables model comparison and filtering |\n| Token usage | Are input/output tokens tracked? | Enables automatic cost calculation |\n| Good trace names | Are names descriptive? (`chat-response`, not `trace-1`) | Makes traces findable and filterable |\n| Span hierarchy | Are multi-step operations nested properly? | Shows which step is slow or failing |\n| Correct observation types | Are generations marked as generations, and is each other call given its most specific type (`retriever` for a lookup, `agent` for a subagent, etc.) rather than a generic `tool`/`span`? See the [observation types docs](https://langfuse.com/docs/observability/features/observation-types). | Enables model-specific analytics and drives the Agent Graph |\n| Sensitive data masked | Is PII/confidential data excluded or masked? | Prevents data leakage |\n| Trace input/output | Does the trace capture meaningful input/output? Is input explicitly set to show only relevant data (e.g., user message), not all function args? | Makes traces readable in the UI and avoids leaking sensitive args |\n\nFramework integrations (OpenAI, LangChain, etc.) handle model name, tokens, and observation types automatically. Prefer integrations over manual instrumentation.\n\nDocs: https://langfuse.com/docs/tracing\n\n**Beyond the baseline**, add context relevant to the app. Infer from code where possible; only ask when it's not obvious (e.g. how they judge a good vs. bad response, what they'd filter a dashboard by, which user segments they'd compare). These are not baseline β€” add only what fits.\n\n| If code shows... | Add | Why |\n| ---------------------------------------------------- | ------------------- | ------------------------------------------------------------------------------- |\n| Conversation history, chat endpoints, message arrays | `session_id` | Groups conversations β€” [docs](https://langfuse.com/docs/tracing-features/sessions) |\n| User authentication, `user_id` variables | `user_id` | User filtering and cost attribution β€” [docs](https://langfuse.com/docs/tracing-features/users) |\n| Multiple distinct endpoints/features | `feature` tag | Per-feature analytics β€” [docs](https://langfuse.com/docs/tracing-features/tags) |\n| Customer/tenant identifiers | `customer_tier` tag | Cost/quality breakdown by segment β€” [docs](https://langfuse.com/docs/tracing-features/tags) |\n| Feedback collection, ratings | Feedback score | Quality filtering and trends β€” [docs](https://langfuse.com/docs/scores/overview) |\n\n### 3. Run and Self-Audit the Traces (required)\n\nInstrumentation isn't done when the code compiles. This is a loop you own as the agent, it's your responsibility to deliver traces of the highest quality your can produce:\n\n**a.** Execute the instrumented path end-to-end so a trace is actually sent.\n\n**b.** Fetch the trace(s) you just created from Langfuse. Any method works (`langfuse-cli`, REST API, SDK, MCP); the CLI is usually simplest β€” see [references/cli.md](references/cli.md).\n\n**c.** Audit the trace against the best-practices page. **Always fetch it fresh β€” never audit from memory, this cannot be skipped** (the guidance changes over time):\n\nhttps://langfuse.com/docs/observability/best-practices\n\nAsk yourself, for each observation: is all data that a user might need in the future, to understand exactly what context the agent had when it made decisions, available in Langfuse?\n\n**d.** Fix every gap you find, then re-run and re-fetch to confirm. Repeat until the trace clears the guidance. Then report what you audited and changed, and link the final trace.\n\n### 4. Explore Traces With the User\n\nWith a clean trace in place, invite the user to explore it in the Langfuse UI:\n\n\"Your traces are now appearing in Langfuse. Take a look at a few of themβ€”see what data is being captured, what's useful, and what's missing.\"\n\nPoint them to the relevant views:\n\n- Traces view: individual requests\n- Sessions view: grouped conversations (if `session_id` added)\n- Dashboard: filtered views using tags\n- Scores: filter by quality metrics\n\nThis helps them understand what they're getting, spot what's missing, and ask better questions about what to add next.\n\n## Always Explain Why\n\nWhen suggesting additions, explain the user benefit:\n\n```\n\"I recommend adding session_id to your traces.\n\nWhy: This groups messages from the same conversation together.\nYou'll be able to see full conversation flows in the Sessions view,\nmaking it much easier to debug multi-turn interactions.\n\nLearn more: https://langfuse.com/docs/tracing-features/sessions\"\n```\n\n## Multi-agent systems (subagent dispatch)\n\nWhen one agent's execution dispatches OTHER agents (coding agents like Claude Code/Codex, research agents, orchestrator/worker architectures), a few extra points on top of the baseline:\n\n- **Type a subagent's own execution as `agent`, not `tool`/`span`.** A bare tool/span for a dispatch hides all of the subagent's internal structure; `agent` lets it show up as its own node in the [Agent Graph](https://langfuse.com/docs/observability/features/agent-graphs).\n- **Don't emit duplicate dispatch + execution nodes.** Emitting both a `tool`-typed \"dispatch\" span and a separate `agent`-typed observation for the same subagent, as siblings, double-represents one event. Emit only the `agent` observation when you have the subagent's actual execution; keep a bare tool span only as a fallback when you have no visibility into what the subagent did.\n- **Nest recursively.** Nest the subagent's `agent` observation under the `agent` or `span` that orchestrates the dispatch, as a sibling of the `generation` that requested it. Within the subagent, put its generations, tool calls, and nested subagent executions under that subagent's `agent` observation; each tool or nested subagent is a sibling of the generation that requested it, not a child of that generation.\n- **Name subagents distinctly.** Frameworks often default every subagent to the same generic role name, making them indistinguishable in the tree and graph (nodes key on name). Derive a specific name from the subagent's actual task/role when the framework doesn't provide one.\n\n## Common Mistakes\n\n| Mistake | Problem | Fix |\n| ---------------------------------------------- | --------------------------------------------------- | --------------------------------------------------------------------------------- |\n| No `flush()` in scripts | Traces never sent | Call `langfuse.flush()` before exit |\n| Flat traces | Can't see which step failed | Use nested spans for distinct steps |\n| Generic trace names | Hard to filter | Use descriptive names: `chat-response`, `doc-summary` |\n| Logging sensitive data | Data leakage risk | Mask PII before tracing |\n| Not explicitly setting input with `@observe` | All function args become trace input (including API keys, configs) | Python: use `langfuse.update_current_span(input=...)`. JS/TS: use `updateActiveObservation({ input: ... })`. Set only the relevant input (e.g., user message) |\n| Manual instrumentation when integration exists | More code, less context | Use framework integration |\n| Langfuse import before env vars loaded | Langfuse initializes with missing/wrong credentials | Import Langfuse AFTER loading environment variables (e.g., after `load_dotenv()`) |\n| Wrong import order with OpenAI | Langfuse can't patch the OpenAI client | Import Langfuse and call its setup BEFORE importing OpenAI client |",
+ },
+ {
+ name: "langfuse-judge-calibration",
+ description:
+ "Calibrate and validate LLM-as-a-Judge evaluators against dataset ground truth. Runs the judge prompt as a Langfuse dataset experiment, compares judge outputs with dataset item expected outputs, and reports simple accuracy or advanced confusion-matrix metrics. Use this guide whenever a user asks if their LLM judge is actually useful, aligned with human judgment, or safe to trust for monitoring decisions.",
+ instructions:
+ '# Judge Calibration (LLM-as-a-Judge)\n\n## Goal\n\nValidate judge outputs against human labels using the smallest reliable workflow\nfor the user\'s goal.\n\nDefault to a **Langfuse dataset experiment** when the user has a Langfuse\ndataset or wants results in the Langfuse Experiments UI.\n\nDefault to **simple calibration** unless the user asks for deeper metrics,\nsplit-based validation, thresholding, or production automation.\n\n## 1) Choose the calibration mode\n\n### Simple calibration\n\nUse this when the user wants a quick answer like "does this judge basically\nmatch human labels?" or explicitly asks for accuracy only.\n\n- No train/dev/test split is required.\n- Compute `exact_match` for each valid row.\n- Report valid sample size, invalid-label count, accuracy, and a short\n recommendation.\n- Do not include Precision/Recall/F1, TPR/TNR, denominator notes, or top failure\n direction unless the user asks for advanced metrics.\n\n### Advanced calibration\n\nUse this when the user asks for confusion matrix metrics, thresholds,\nproduction monitoring, high-stakes automation, or train/test-style validation.\n\n- If split labels exist, keep train/dev/test separate.\n- If no split exists, compute metrics on the provided rows and state that this is\n not a held-out final quality claim.\n- Compute TP/FP/FN/TN and derived metrics.\n- Use `references/error-analysis.md` for qualitative diagnosis of disagreements.\n\n## 2) Primary workflow\n\n1. Confirm the dataset name, ground-truth label location in `expectedOutput`,\n judge prompt name/version, judge model, and label vocabulary.\n2. Choose simple or advanced mode. If ambiguous, use simple mode.\n3. Run the judge prompt against each dataset item input as a Langfuse experiment.\n4. Compare the judge output to `item.expected_output` in evaluator functions.\n5. Return the matching report format from section 7.\n\n## 3) Langfuse experiment workflow\n\nUse the SDK experiment runner as the default implementation. A Langfuse-hosted\ndataset automatically creates a dataset run that can be inspected and compared\nin the Langfuse UI.\n\nBefore implementing, you **must** retrieve the current experiment SDK\ndocumentation from the Langfuse docs β€” do not rely on memory, the SDK changes\nfrequently. Fetch these pages (see SKILL.md section 2 for retrieval methods):\n\n- [Experiments via SDK](https://langfuse.com/docs/evaluation/experiments/experiments-via-sdk) β€” primary reference for `dataset.run_experiment`, task/evaluator signatures, and `Evaluation` return shape\n- [Datasets](https://langfuse.com/docs/evaluation/experiments/datasets) β€” dataset item structure (`input`, `expectedOutput`) and how to load a hosted dataset\n- [Experiments data model](https://langfuse.com/docs/evaluation/experiments/data-model) β€” how runs, items, and scores relate in the UI\n\nHigh-level shape of a simple-mode calibration experiment:\n\n```\nload dataset and judge prompt from Langfuse\ndefine POSITIVE / NEGATIVE label set\n\ntask(item):\n compile judge prompt with item.input\n call judge model\n return normalized label\n # never read item.expected_output here β€” that would leak the answer\n\nitem_evaluator(output, expected_output):\n if either label is outside the allowed set:\n return invalid (excluded from accuracy denominator)\n return exact_match score (0 or 1)\n\nrun_evaluator(item_results):\n accuracy = matches / valid_rows\n return aggregate score + invalid_label count\n\ndataset.run_experiment(task, [item_evaluator], [run_evaluator],\n metadata={calibration_mode, judge_prompt, labels})\n```\n\nFor advanced calibration, the item evaluator emits one score per confusion-matrix\ncell (`judge-is-tp`, `judge-is-fp`, `judge-is-fn`, `judge-is-tn`), and the run\nevaluator aggregates them into Precision / Recall / F1 / TPR / TNR. See section\n6 for the metric definitions and zero-denominator guardrails.\n\nRules:\n- Use a Langfuse-hosted dataset when the user wants a real Langfuse experiment.\n Local SDK datasets create traces and scores, but not Langfuse dataset runs.\n- The dataset item `input` must contain everything needed to run the judge\n prompt. The dataset item `expectedOutput` must contain the ground-truth label.\n- Never pass `expectedOutput` into the judge prompt or task. That would leak the\n answer and invalidate calibration.\n- Return `Evaluation(...)` objects from item and run evaluators for stable SDK\n formatting and score ingestion.\n- Store prompt name/version, judge model, label vocabulary, dataset version, and\n calibration mode in run metadata.\n- Use a unique run name so the experiment appears as a separate dataset run.\n\n## 4) Label validation\n\nNever silently treat unknown labels as negative.\n\n- For binary judges, define the positive and negative labels before computing\n confusion-matrix metrics.\n- Normalize only deterministic differences such as surrounding whitespace or\n casing, and mention that normalization was applied.\n- Rows where `expected` or `actual` is outside the allowed label set are invalid.\n Exclude them from metric denominators and report the invalid count.\n- For multi-class judges, use simple `exact_match` / accuracy unless the user\n defines one positive class for binary metrics.\n\nExample binary labels:\n- Positive: `ESCALATE`\n- Negative: `RESOLVE`\n\n## 5) Simple metrics\n\nFor each valid row:\n\n- `exact_match = 1 if actual == expected else 0`\n\nAggregate:\n\n- `accuracy = sum(exact_match) / valid_rows`\n\nIf `valid_rows == 0`, report that accuracy is undefined and ask for valid\nexpected/actual labels.\n\n## 6) Advanced metrics\n\n### Dataset and split discipline\n\nUse split discipline only when the user asks for advanced validation or final\nquality claims.\n\n- **Train**: optional few-shot examples for the judge prompt\n- **Dev**: iterative prompt/model refinement\n- **Test**: single final calibration pass\n\nDo not tune on the same rows used to claim final quality. Use balanced classes\nin dev/test when possible so both error directions are measurable.\n\n### Per-row classification mapping\n\nFor each row with `expected` and `actual` labels:\n\n- **TP**: expected = positive and actual = positive\n- **FP**: expected = negative and actual = positive\n- **FN**: expected = positive and actual = negative\n- **TN**: expected = negative and actual = negative\n\nAlso compute:\n- `exact_match = 1 if actual == expected else 0`\n\n### Aggregate metrics\n\nFrom aggregate counts:\n- `accuracy = (TP + TN) / valid_rows`\n- `precision = TP / (TP + FP)`\n- `recall = TP / (TP + FN)`\n- `f1 = 2 * precision * recall / (precision + recall)`\n- `TPR = TP / (TP + FN)`\n- `TNR = TN / (TN + FP)`\n\nGuardrails:\n- if `TP + FP == 0`, precision is undefined (report null + note)\n- if `TP + FN == 0`, recall and TPR are undefined (report null + note)\n- if `TN + FP == 0`, TNR is undefined (report null + note)\n- if `precision + recall == 0`, set `f1 = 0`\n\n### Advanced quality gates\n\nBefore trusting the judge on production traffic:\n\n1. **Split integrity**: no leakage from held-out rows into prompt examples.\n2. **Confusion matrix sanity**: `TP + FP + FN + TN == valid_rows`.\n3. **Metric recomputation check**: recompute aggregate stats from row-level\n flags and compare.\n4. **TPR/TNR review**: inspect both directions for class-direction bias.\n5. **Acceptance criteria**: before the held-out test, agree on thresholds for\n the relevant metrics based on the costs of false positives and false\n negatives and whether the judge informs monitoring or automation. Do not\n infer "ship" from a universal cutoff.\n\n## 7) Report format\n\n### Simple report\n\nReturn only:\n- dataset name and dataset run URL when available\n- valid rows / total rows\n- invalid-label count\n- accuracy\n- one-sentence recommendation\n\n### Advanced report\n\nAdd:\n- confusion matrix: TP, FP, FN, TN\n- accuracy, precision, recall, F1, TPR, TNR\n- denominator notes for undefined metrics\n- top failure direction: false positives or false negatives\n- recommendation: ship, iterate, collect more labels, or do not automate\n\n## 8) Langfuse implementation notes\n\nPrefer SDK experiment evaluators for score creation. They attach item-level\nscores to the experiment traces and run-level scores to the dataset run.\n\nUse manual REST score creation only as a fallback when not using the SDK\nexperiment runner, or for local smoke tests. See\n[Scores via SDK](https://langfuse.com/docs/evaluation/evaluation-methods/scores-via-sdk)\nand the [Scores API reference](https://langfuse.com/docs/api) for the current\npayload shape. Before using the CLI for score creation, inspect its current\nschema and action help instead of assuming resource names, arguments, or\ncapabilities from a known package version.\n\nScore names to emit:\n\nSimple mode:\n- `judge-exact-match`\n- `judge-accuracy`\n\nAdvanced mode:\n- `judge-exact-match`\n- `judge-is-tp`\n- `judge-is-fp`\n- `judge-is-fn`\n- `judge-is-tn`\n\nRecommended metadata:\n- `expected_label`, `actual_label`\n- calibration mode: `simple` or `advanced`\n- positive/negative labels when binary metrics are used\n- evaluator prompt name+version\n- dataset/split version when used\n- run identifier\n\n## 9) Classification logic (pseudo-code)\n\nThe per-row classification each evaluator must perform:\n\n```\nnormalize(label) = strip whitespace, uppercase\nALLOWED = {POSITIVE, NEGATIVE}\n\nclassify(expected, actual):\n expected, actual = normalize(expected), normalize(actual)\n\n if expected βˆ‰ ALLOWED or actual βˆ‰ ALLOWED:\n mark row invalid β†’ exclude from denominators\n\n exact_match = (expected == actual)\n is_tp = (expected == POSITIVE and actual == POSITIVE)\n is_fp = (expected == NEGATIVE and actual == POSITIVE)\n is_fn = (expected == POSITIVE and actual == NEGATIVE)\n is_tn = (expected == NEGATIVE and actual == NEGATIVE)\n```\n\n## 10) Common failure modes\n\n- label vocabulary not constrained (judge outputs free text instead of strict\n labels)\n- positive/negative label inversion between annotators and evaluator code\n- leaking `expectedOutput` into the judge task instead of only the evaluator\n- using local SDK data when the user expects a Langfuse dataset run in the UI\n- reporting only accuracy when classes are imbalanced and error direction matters\n- calculating F1 without explicit zero-denominator handling\n- using advanced validation claims without a held-out split\n\n## 11) What to do after calibration\n\n- If simple accuracy is enough: report it and stop.\n- If metrics are weak and advanced validation is needed: iterate prompt and\n few-shots on dev data only.\n- If metrics pass: freeze the baseline and monitor drift over time.\n- For qualitative diagnosis of disagreements, switch to\n `references/error-analysis.md`.',
+ },
+ {
+ name: "langfuse-prompt-engineering",
+ description:
+ "Write or change prompts in Langfuse or code. Use whenever the user asks to create, edit, rewrite, debug, tune, or otherwise modify a prompt, including a small wording or instruction change. Distinct from prompt-migration and judge-calibration.",
+ instructions:
+ '# Prompt Engineering\n\n## Universal principles\n\n1. **Be specific.** State the required output format, constraints, and, when order matters, numbered steps. Write for someone with minimal context.\n2. **Use labeled sections** for the role, instructions, examples, and context. Use XML tags when they help distinguish rules, data, and variables.\n3. **Give the model a role.** A focused system-prompt sentence can steer tone and behavior.\n4. **Explain why important rules exist** so the model can apply them to adjacent cases.\n\n## Adjusting an existing prompt\n\n1. **Pin down the failure first.** Use concrete misbehaving outputs and name the exact failure mode: wrong format, ignored instruction, wrong tone, hallucination, or another observable error. The task and data show what could fail, not what did fail; do not write a fix before observing the failure.\n2. **Trace the failure to the prompt.** Look for a missing, ambiguous, or conflicting instruction.\n3. **Fix the failure class, not the example.** Generalize the observed mistake into an error class and write the instruction at that level, in your own words. Cover adjacent cases, but do not add definitions, rules, or examples for unobserved problems or encode the failing example itself. If a generalized instruction does not work, report the failure instead of narrowing the rule to that case.\n4. **Preserve existing behavior.** Before changing text, identify what behavior it protects. Avoid contradicting other rules or breaking cases that already work.\n5. **Make the smallest testable edit.** Change one cause per attempt so you can attribute the result. Avoid extra length, emphasis, or all-caps. In Langfuse, save each attempt as a new version or label so you can compare and roll back.\n\n## Model-specific tuning\n\nIdentify the target model if you can β€” tuning differs by model, so when you know which model the prompt runs on, follow that model\'s own prompting guidance.\n\n| Technique | Standard (GPT) | Reasoning (Claude latest) |\n|-----------|----------------|---------------------------|\n| Instruction density | High, prescriptive | Goal-level; don\'t micro-manage steps |\n| Reasoning | Add explicit chain-of-thought | Native thinking; "think thoroughly" + tune `effort` |\n| Emphasis language | Strong directives fine | Dial back `MUST`/`CRITICAL` |\n| Output format | Prefilling/scaffolds | Prefill removed (4.6+); structured outputs + "no preamble" |\n| Roles | `developer` > `user` > `assistant` | `system` role + `user`/`assistant` |',
+ },
+ {
+ name: "langfuse-prompt-migration",
+ description:
+ "Migrate hardcoded prompts to Langfuse for version control and deployment-free iteration. Use when user wants to externalize prompts, move prompts to Langfuse, or set up prompt management.",
+ instructions:
+ '# Langfuse Prompt Migration\n\nMigrate hardcoded prompts into Langfuse-managed prompts. The API mechanics (`create_prompt`, `get_prompt`, `.compile()`, linking to traces) are in the docs β€” fetch them at execution time.\n\n## Prerequisites\n\nVerify credentials exist β€” check presence only, never print the secret key (its value would land in the agent\'s context and transcripts):\n\n```bash\n[ -n "$LANGFUSE_PUBLIC_KEY" ] && echo "public key: set" || echo "public key: missing"\n[ -n "$LANGFUSE_SECRET_KEY" ] && echo "secret key: set" || echo "secret key: missing"\n[ -n "${LANGFUSE_BASE_URL:-${LANGFUSE_HOST:-}}" ] && echo "base url: set" || echo "base url: missing"\n```\n\nUse `LANGFUSE_BASE_URL` for current SDKs. If only `LANGFUSE_HOST` is set, export `LANGFUSE_BASE_URL="$LANGFUSE_HOST"`. If a CLI expects `LANGFUSE_HOST` and only `LANGFUSE_BASE_URL` is set, export `LANGFUSE_HOST="$LANGFUSE_BASE_URL"`.\n\nIf credentials or the base URL are missing, ask the user to set them in their shell or a `.env` file. Do not ask them to paste secret keys into chat.\n\n## 1. Inventory every prompt (before writing any code)\n\nFor each prompt, record:\n\n- **Name**: lowercase, hyphenated (e.g. `chat-assistant`)\n- **Source file**: where the prompt text lives\n- **Code file to refactor**: the file that USES the prompt. For asset files (`.txt`/`.yaml`/`.md`), this is the file that loads the asset, not the asset itself\n- **Type**: `chat` (message array) or `text` (plain string)\n- **Variables**: values interpolated in, converted to `{{var}}`\n- **Content**: the actual text to upload\n\nBefore choosing `text` or `chat`, fetch and follow [Chat vs Text Prompts](https://langfuse.com/docs/prompt-management/data-model#text-vs-chat-prompts). Do not default unrelated prompt flows to one prompt type.\n\nPrompts typically live in OpenAI message arrays, Anthropic system arguments, LangChain prompt templates, Vercel AI system/prompt fields, and raw multi-line strings near LLM calls.\n\n## 2. Convert templating and decide structure\n\n**Variable syntax:** Langfuse substitutes only double-brace `{{var}}`. Convert every single-brace form during upload β€” `{var}`, `${var}`, f-string `{var}`, `.format(var=...)`, and string concatenation all become `{{var}}`. Uploading `{var}` will silently fail to substitute.\n\n**Complex templates:** Langfuse has no conditionals, loops, or filters. If the code uses them (e.g. Jinja `{% if %}`/`{% for %}`), either pre-compute the value in code and pass a plain `{{variable}}` (recommended), or store the raw template and compile client-side β€” which loses Playground preview and UI experiments. See https://langfuse.com/docs/prompt-management/features/variables and the [external templating FAQ](https://langfuse.com/faq/all/using-external-templating-libraries).\n\n**What to make a variable vs. keep hardcoded:**\n\n| Make variable | Keep hardcoded |\n|---------------|----------------|\n| User-specific (`{{user_name}}`) | Output format instructions |\n| Dynamic content (`{{context}}`) | Safety guardrails |\n| Per-request (`{{query}}`) | Persona / personality |\n| Environment-specific (`{{company_name}}`) | Static examples |\n\n**Naming:** lowercase-hyphenated, feature-based (`document-summarizer`), hierarchical for related prompts (`support/triage`), prefix subprompts with `_` (`_base-personality`).\n\nBefore extracting subprompts, fetch and follow [Prompt Composability](https://langfuse.com/docs/prompt-management/features/composability). Use composition for the reuse and shared-maintenance cases described there, not merely to decompose one coherent prompt flow.\n\n## 3. Present the plan, then create and refactor\n\nWhen the user explicitly requests a migration and credentials work, treat that request as authorization to create prompts and refactor the call sites. Present the inventory and plan as a progress update, then continue.\n\nAsk only when a materially different design choice would change behavior, required credentials are unavailable, or destructive cleanup needs approval. Do not stop merely to confirm names, prompt types, or optional tracing.\n\nThen:\n\n- Create the prompts (label migrated prompts `production` β€” they\'re already live) and refactor call sites to fetch each prompt from Langfuse and compile its variables in. The SDK calls differ across Python and JS/TS β€” fetch the current docs: https://langfuse.com/docs/prompt-management/get-started\n- Fetch by the `production` label\n- If the codebase already has Langfuse tracing (decorators, an instrumented client, or manual spans), link prompts so you can see which version produced each response. See https://langfuse.com/docs/prompt-management/features/link-to-traces\n\n## 4. Verify\n\n- All prompts created with the `production` label; code fetches with `label="production"`\n- Variables and subprompts compile without errors\n- Application behavior is unchanged\n- Generations show the linked prompt in the UI (if tracing enabled)',
+ },
+ {
+ name: "langfuse-sdk-upgrade",
+ description:
+ "Upgrade Langfuse SDKs and application instrumentation while preserving trace attributes across observations. Use for Python or JS/TS SDK migrations, including the application side of a v4 platform migration.",
+ instructions:
+ "# Langfuse SDK upgrade\n\n## Sources of truth\n\nDetermine every installed Langfuse SDK major version, then fetch each applicable leaf guide in full before editing:\n\n- [SDK upgrade paths](https://langfuse.com/docs/observability/sdk/upgrade-path)\n- [Python v3 to v4](https://langfuse.com/docs/observability/sdk/upgrade-path/python-v3-to-v4)\n- [JS/TS v4 to v5](https://langfuse.com/docs/observability/sdk/upgrade-path/js-v4-to-v5)\n- [Instrumentation and attribute propagation](https://langfuse.com/docs/observability/sdk/instrumentation#add-attributes)\n- [Sessions and session-level metrics](https://langfuse.com/docs/observability/features/sessions)\n- [Direct OpenTelemetry ingestion](https://langfuse.com/integrations/native/opentelemetry)\n\nFollow every intermediate guide when the installed version is more than one major behind. Use the current docs for implementation details; do not copy examples from this file.\n\n## Workflow\n\n1. Inventory every SDK, integration package, direct OpenTelemetry exporter, initialization site, instrumentation wrapper, lockfile, worker, script, and test that can emit Langfuse data.\n2. Find every source of correlating attributes, including `session_id`/`sessionId`, `user_id`/`userId`, tags, metadata, version, environment, and trace name. Search for the values and surrounding application concepts, not only removed SDK method names.\n3. Apply every relevant item from the exact version-specific guides. Preserve each correlating attribute by establishing its documented propagation scope before any observation-producing call that must inherit it.\n\n\n## Completion report\n\nReport the versions before and after, changed instrumentation paths, attribute sources and propagation scopes, validation performed, inspected trace or session, and any remaining blocked verification.",
+ },
+ {
+ name: "langfuse-setting-up-evals",
+ description:
+ "Set up evaluation by finding gaps across signal capture, monitoring, and evaluator metrics, then addressing the right one first.",
+ instructions:
+ "# Setting up evals\n\n- **Act as executor and teacher.** Do the work and, above all, explain why the approach fits the user's goal and data.\n- **Teach decision-making, not configuration.** Explain the reasoning, tradeoffs, and limitationsβ€”not sampling rates, targets, score types, or other setup details. The response must leave the user able to repeat or change the approach and make future decisions without you.\n\nAlways read [Choosing what to evaluate](https://langfuse.com/academy/evaluate/choosing-what-to-evaluate) and [Evaluation](https://langfuse.com/academy/evaluate). Use their guidance to interpret their project, but do not return it as a generic setup plan.\n\nIf the user has a precise metric, skip to **Build the evaluator**. Otherwise, do not assume that choosing evaluator metrics is the next step.\n\n## Find the measurement gap\n\nBefore advising:\n\n- Inspect enough representative traces to present concrete findings of your own before asking about metrics.\n- Inventory existing datasets, evaluators, traces, etc. as context; do not assume they should be reused or represent a current priority. Treat their importance as unconfirmed until the user agrees.\n- Identify what is already surfaced through trace errors, logged data, existing scores, or other monitoring; what is available but not monitored; what important user or product signals are not captured; and what genuinely requires an evaluator.\n- Do not ask the user for information you can inspect.\n- If project access is unavailable, say so instead of substituting a generic plan.\n\nDo not add an evaluator that duplicates an existing error or logged signal. For example, do not evaluate valid JSON when invalid JSON already produces a trace error.\n\nTreat trace errors and logged failures as existing coverage. Unless the user wants operational monitoring, do not let them displace the search for subtler qualitative signals that are not yet captured. Do not propose alerts, paging, or release gates before asking how the measurement should be used.\n\nDo not propose evaluating a known failure with an owner and a planned one-time fix unless the user wants to track whether it recurs.\n\nIf formal error analysis is the right next step, explain why and ask whether the user wants to do it. If they agree, run it following `references/error-analysis.md`, and do not start metric selection until the analysis is complete.\n\n## Define the metric set\n\nExisting project materials inform the questions you'll ask; it does not determine what the user values.\n\nWhen forming tentative recommendations:\n\n- Start with direct evidence of user or product outcomes, recurring failures visible in traces or user reactions, and important signals the application is not yet capturing.\n- Prefer quick wins: signals that are application-specific, actionable, reasonably reliable to start measuring. A good example signal is user (dis)satisfaction.\n- Recommend metrics only for problems evidenced in current data. A behavior being common or theoretically risky is not a reason to run an evaluator.\n- For live evaluation, recommend only metrics that can be scored from the available live data without ground truth; never suggest a metric that requires knowing the correct answer, expected outcome, or ideal resolution.\n- Do not propose generic starting metrics such as `helpfulness`, `quality`, `relevance`, `hallucination`, `groundedness`, `task completion`, `task success`, or `reliability`.\n- Before presenting any metric, verify that it can actually be measured using the available data and Langfuse's supported evaluator inputs. If not, identify what must be captured instead.\n- Explain why each recommendation deserves attention before the alternatives, then ask the user whether that priority matches their goals.\n\nWhen specific failures or hard requirements are known and metric selection is the next task, ask focused, dependent questions about which ones matter and what decisions their measurement should support. Give a recommendation grounded in the project, leave the decision to the user, and wait for their answer.\n\nDo not treat any metric as selected, choose evaluator types, ask implementation questions, or implement anything until the user has answered and explicitly confirmed the metric set.\n\n## Build the evaluator\n\nFor evaluator functionality, use the unstable API endpoints.\n\nFollow [Writing good evaluators](https://langfuse.com/academy/evaluate/writing-evaluators) to choose the evaluator type; do not always default to an LLM-as-a-judge.\n\n- Explain why the chosen evaluation method measures the intended behavior, what evidence it relies on, what it cannot tell the user, and when another method would be better.\n- When methods have significant trade-offs and none is clearly superior, explain the options and let the user decide before implementation.\n- Only if an LLM-as-a-judge is the best fit, calibrate it on real examples before treating it as ready(`references/judge-calibration.md`). You can do this by running an experiment on a dataset where the prompt being tested is the LLM-as-a-judge prompt.\n- Share a link to the evaluator.",
+ },
+ {
+ name: "langfuse-skill-feedback",
+ description:
+ "Submit feedback about the Langfuse skill to its maintainers. Use when the skill's instructions are wrong, outdated, missing something, or could be improved.",
+ instructions:
+ "# Skill Feedback\n\nUse this only for feedback about the skill itself, not Langfuse the product or a user's application data.\n\n1. Draft concise feedback using the following structure:\n - **Describe your idea or feedback** (required)\n A clear description of what went wrong or what could be improved. Include:\n - What the user was trying to do (as `goal`)\n - What the skill did vs what was expected\n - Any specific instructions that were incorrect or missing\n\n - **What would the ideal outcome look like?** (optional)\n What the correct behavior or guidance should be.\n\n If feedback targets existing skill, reference in `target`.\n2. If the user wants a reply, ask them to include an email address in `feedback`; use only an address they explicitly provide.\n3. Show every submitted field exactly as it will be sent and ask for explicit permission. Do not submit without approval.\n\n## Submission options\n\nUse the first available option, or the option the user prefers:\n\n1. **Authenticated Langfuse MCP, CLI, or Public API** β€” prefer the `submitFeedback` tool on the Langfuse MCP server. If it is unavailable, discover the current feedback operation with the Langfuse CLI schema/help and submit through the authenticated Public API. Do not ask users to paste credentials into chat.\n2. **Langfuse Docs MCP** β€” use its unauthenticated `submitFeedback` tool when no authenticated Langfuse interface is available.\n3. **GitHub issue or discussion** β€” if no MCP/CLI/API is available, or the user wants a public, trackable thread, provide a prefilled discussion link for the user to submit: `https://github.com/langfuse/skills/discussions/new?category=ideas-improvements&title=<url-encoded title>&body=<url-encoded body>`. Use `https://github.com/langfuse/skills/issues/new` if they prefer an issue.\n\nReport the Langfuse correlation ID or GitHub URL. If submission fails, give a safe error without exposing credentials or request bodies.",
+ },
+ {
+ name: "langfuse-evaluator-upgrade",
+ description:
+ "Upgrade legacy trace-level or dataset-item LLM-as-a-Judge evaluators to observation-level or experiment evaluators. Use when a project needs the coding handoff for evaluator migration during the v4 transition.",
+ instructions:
+ "# Evaluator upgrade\n\nUse the [evaluator migration guide](https://langfuse.com/faq/all/llm-as-a-judge-migration),\n[v4 overview](https://langfuse.com/docs/v4), and current [Evaluation Rules](https://api.reference.langfuse.com/#tag/unstableevaluationrules)\nand [Evaluators](https://api.reference.langfuse.com/#tag/unstableevaluators) API schemas as\nthe sources of truth. Read the current unstable schema before using it: only the unstable\nEvaluation Rules API returns legacy `trace` and `dataset` targets; new rules use observation\nor experiment targets.\n\n## Inventory and scope\n\n- Page through all rules and fetch each referenced evaluator. Record target, status, filters, mappings/JSONPaths, sampling, delay, time scope, evaluator, and score name.\n- If a bulk page fails, retry with `limit=1` to isolate unreadable entries; report any page that still fails as a blocker and do not claim a complete inventory.\n- Separate active, inactive, and blocked rules. Migrate only active rules that depend on new/live data. Historical-only rules (`EXISTING` without `NEW`) may be ignored; deactivation or deletion is optional and requires approval.\n- Migrate only legacy trace and dataset targets here. Do not touch existing observation, experiment, or event rules. If the unstable API cannot mutate a legacy rule, return the exact UI action.\n- Show the complete inventory and a consolidated retain/delete decision before changing project configuration.\n\n## Trace to observation\n\nIn v4, a trace groups observations and has no separate trace input/output target. For each\nretained trace rule, select one stable observation and verify its own fields:\n\n1. Prefer an existing root or parent observation that already contains the required result.\n2. If the rule used overall trace input/output, confirm that the previous SDK upgrade has already put those values on the root observation.\n3. If it mixed trace context with one observation, propagate only the required context or filter attributes to that observation.\n4. If required values span multiple observations, add a dedicated evaluation observation only when no existing observation can own them. Do not serialize the whole trace into metadata.\n\nUse a stable observation name/type and root status where relevant. The selector should ideally match at\nmost one observation per trace when the legacy rule produced one score per trace. If it matches multiple observation, convey to the user that the rule will produce one score per observation. Keep names\nand types stable across executions.\n\nPreserve the evaluator-template: prompt, model, output, score name, sampling, delay. Time scope, filters, mappings, and JSONPaths need to be compatible with the new format. A legacy trace\nname filter becomes `traceName`; it does not replace the observation selector. Every variable\nand filter must exist on the selected observation, with trace attributes propagated according\nto the applicable SDK or OpenTelemetry guide.\n\nInspect the resolved SDK/OTel path and a representative new trace; do not infer readiness\nfrom a dependency declaration alone. Complete application instrumentation first when needed.\nKeep deprecated trace-I/O calls only while an active legacy rule requires them, and remove\nthem after the successor is verified and the legacy rule is disabled.\n\n## Dataset item to experiment\n\nRead the legacy `dataset` rule and referenced evaluator, then create an `experiment` rule with\nthe same evaluator, score name, sampling, and mappings. Use the experiment target's current\ndataset filter and dataset ID from the unstable schema.\nAs of the current API, this is `datasetId` with IDs from the v2 dataset endpoint. Translate\ndataset-item expected output and metadata to the corresponding experiment-item fields, and\npreserve input mappings and JSONPaths against the experiment-item shape.\n\nFor dataset runs, the root observation carries the run's input and output, so mapping legacy\ntrace input/output to observation input/output is safe and needs no evaluator-specific\ninstrumentation change.\n\nKeep the legacy rule active while the project still uses low-level dataset-run APIs. Disable\nit after the project emits experiment context; leaving both active can create duplicate\nscores. Review any mapping that relies on a named observation or assumes a one-to-one input.\n\n## Cut over and report\n\n- Create the successor disabled when supported, validate it on new data, then enable it with approval. A brief overlap can create duplicate scores.\n- Disable the legacy rule only after the successor is verified. Do not backfill historical data or delete historical scores.\n- Verify the selected observation exists once when required, contains every mapped value and filter, and excludes sibling observations. Do not claim live verification without project access or a runnable path.\n\nReturn a concise per-rule table with the legacy rule, target, selector, mappings/filters,\nrequired code changes, validation result, and remaining project actions or blockers.",
+ },
+ {
+ name: "langfuse-user-feedback",
+ description:
+ "Wires up user feedback (thumbs up/down, ratings, comments) from an application's frontend to Langfuse scores. Use when user wants to capture end-user feedback, add ratings to traces, or connect user complaints to Langfuse.",
+ instructions:
+ "# User Feedback\n\nTracing must already be set up β€” feedback is stored as scores on traces.\n\nDocs: https://langfuse.com/docs/observability/features/user-feedback\n\n## Workflow\n\n### 1. Determine What Feedback to Capture\n\nIf the user has asked for something specific, go with that. Otherwise, look at the application and **present a few UX options** for how feedback could work, then ask the user which they prefer before implementing.\n\nCommon UX patterns to suggest:\n\n| UX Pattern | Best for | How it works |\n|------------|----------|--------------|\n| Thumbs up/down | Chat apps, Q&A | Simple binary buttons next to each response |\n| Star rating (1–5) | Content generation, summaries | Star row or dropdown after each output |\n| \"Was this helpful?\" banner | Search, documentation assistants | Single yes/no prompt at the bottom of a response |\n| Regenerate / copy tracking | Any app with these actions | Implicit β€” log when users retry (negative signal) or copy output (positive signal) |\n| Free-text comment | Complex outputs, internal tools | Optional text field alongside a rating |\n| Report button | Any user-facing app | Flag icon to report bad/harmful responses |\n\nThis table is not exhaustive β€” if the application suggests a different feedback pattern that fits better, propose that instead. Present 2–3 options that match the application's use case and ask the user which approach they'd like. This decision shapes everything downstream (score names, data types, frontend components), so it's important to align early.\n\nFeedback can be **explicit** (user rates via thumbs, stars, etc.) or **implicit** (derived from behavior like copying output, retrying, or escalating to support). Both are stored as scores. Explicit feedback requires the trace ID to reach the frontend; implicit feedback is logged server-side where the event already happens.\n\n### 2. Choose Score Names\n\nName reflects the signal source, not what you hope it measures (e.g., `user-thumbs` not `response-quality` β€” a thumbs down doesn't tell you *what* was wrong). Avoid generic names like `feedback` or `score`.\n\nRules:\n- Lowercase with hyphens\n- One consistent name per feedback type across the entire app\n- If capturing multiple signals, each gets its own distinct name\n\n### 3. Implement Score Creation\n\nFetch and follow the current [user feedback guide](https://langfuse.com/docs/observability/features/user-feedback) and [score ingestion docs](https://langfuse.com/docs/evaluation/evaluation-methods/scores-via-sdk) before editing code. They contain the current browser and server SDK APIs plus framework-specific patterns for returning trace IDs to the frontend.\n\n**For implicit feedback (server-side):** Create the score where the behavior is already handled in application code.\n\n**For explicit feedback (frontend):** Make the relevant trace ID available to the frontend and use the current Langfuse browser SDK with a public key only. Never expose a Langfuse secret key in browser code.\n\n### 4. Verify\n\nTrigger a feedback action and check the trace's Scores tab in Langfuse. Confirm the score name, value, and data type are correct.\n\nPoint users to what they can do with feedback data: filter traces by low scores, use score analytics for trends, build annotation queues for team review.\n\n## Common Mistakes\n\n| Mistake | Problem | Fix |\n|---------|---------|-----|\n| Secret key in frontend code | Security risk | Use the current browser SDK with a public key only |\n| Missing `dataType` on boolean scores | Value `1` inferred as `NUMERIC` | Always pass `dataType: \"BOOLEAN\"` explicitly |\n| Inconsistent score names across the app | Can't aggregate or filter reliably | Pick one name per feedback type, use it everywhere |",
+ },
+ {
+ name: "langfuse-v4-project-migration",
+ description:
+ "Prepare an application and Langfuse project for v4 by upgrading ingestion and API usage, migrating legacy evaluators, and moving exports to the enriched observation schema.",
+ instructions:
+ "# Langfuse v4 project migration\n\n## Sources of truth\n\nFetch only the pages needed for the surfaces found:\n\n- [V4 overview](https://langfuse.com/docs/v4) and [compatibility](https://langfuse.com/docs/compatibility)\n- [SDK upgrade paths](https://langfuse.com/docs/observability/sdk/upgrade-path)\n- [Custom ingestion migration](https://langfuse.com/integrations/native/opentelemetry/migration-to-v4)\n- [Deprecated API migration](https://langfuse.com/faq/all/deprecated-api-migration)\n- [Evaluator migration](https://langfuse.com/faq/all/llm-as-a-judge-migration)\n- [Evaluation Rules](https://api.reference.langfuse.com/#tag/unstableevaluationrules) and [Evaluators](https://api.reference.langfuse.com/#tag/unstableevaluators) APIs\n- Export migrations: [Blob Storage](https://langfuse.com/docs/api-and-data-platform/features/export-to-blob-storage#upgrade-path), [Mixpanel](https://langfuse.com/integrations/analytics/mixpanel#migrate-export-source), and [PostHog](https://langfuse.com/integrations/analytics/posthog#migrate-export-source)\n\nDiscover unstable schemas before use.\n\n## Choose the execution mode\n\n- Confirm the target host and project before project reads or writes. Never request secrets in chat or commit them.\n- Prefer an available project interface. Otherwise use the [CLI](https://langfuse.com/docs/api-and-data-platform/features/cli), starting with `api __schema` and action `--help`.\n- Without project access, enter **code-only mode**: complete verifiable repository work, do not infer evaluator or export state, and mark every project-dependent result blocked.\n- Without codebase access, inspect the project and return an exact code handoff; do not mark repository work ready.\n\n## Migrate the codebase\n\n- Inventory every Langfuse SDK, integration, OTEL exporter, initialization site, lockfile, raw request, generated client, script, notebook, and CI call.\n- Upgrade to the latest stable SDK major required by the current docs and apply every applicable breaking change. Record both declared and resolved versions; update an existing lockfile.\n- Find every source of correlating attributes, including session and user IDs, tags, metadata, version, environment, and trace name; do not search only for removed SDK methods.\n- Put overall input/output on the root observation. Establish the documented propagation scope before observation-producing calls so every applicable child receives the attributes needed for filtering and aggregation, including the session ID on cost-bearing generations.\n- For raw `/api/public/ingestion`, use the current Langfuse SDK in Python or JS/TS. For other languages, use the language's native OpenTelemetry API and follow the custom-ingestion guide.\n- When replacing synchronous ingestion, assess buffering, retries, flushing, shutdown, and error propagation. Do not claim identical delivery semantics without verification.\n- For other deprecated APIs, migrate the path, parameters, pagination, filters, field groups, response parsing, and downstream consumer together using the deprecated-API guide.\n- Check self-hosted compatibility before replacing calls; report code targeting a v3 server as blocked on the server upgrade.\n\n## Migrate evaluators\n\n- Inventory all active evaluators and rules, inspect each referenced definition, and report inactive rules without reactivating them.\n- Check the Evaluators UI on the confirmed target host for active **Legacy** rows ([open in Langfuse Cloud](https://cloud.langfuse.com/project/~/evals)); public interfaces may not expose every legacy target.\n- For each inspected active legacy rule, record its filters, sampling, mappings, representative payloads, one observation or experiment successor, required code changes, and cutover test.\n- Label this a **project-verified contract** only when the rule and representative observations were inspected. Label code-only suggestions **candidate targets**, never project state.\n- Consolidate all required evaluator variables and filter attributes onto the one target observation. Observation evaluators cannot read siblings or children.\n- Create or update the successor disabled, validate it on newly ingested data, obtain approval before enabling it, compare scores and logs, then disable rather than delete the legacy rule.\n\n### Deprecated trace input/output escape hatch\n\n- Default to removing deprecated trace input/output and migrating the evaluator to root-observation input/output.\n- Never add or retain `set_current_trace_io()`, `span.set_trace_io()`, `setActiveTraceIO()`, `span.setTraceIO()`, or equivalent compatibility merely because a legacy rule exists or project access is unavailable.\n- Offer compatibility only when automatic migration is currently impossible or the user explicitly insists on preserving the legacy evaluator unchanged after being warned that trace input/output is deprecated.\n- Even then, require explicit confirmation before changing code, mark the evaluator migration `manual action` or `blocked`, and document the condition for removing the escape hatch. Do not present it as the recommended migration.\n\n## Migrate exports\n\n- Inventory Blob Storage, Mixpanel, PostHog, and other sources in **Project Settings > Integrations**.\n- Follow each documented dual-source transition. Enabling dual output is additive; enriched-only stops legacy output and requires explicit downstream-owner confirmation.\n- Validate warehouse schemas, paths, joins, dashboards, transformations, alerts, and historical-data handling. Keep every consumer on one shape during dual mode.\n- Change only the source; preserve credentials, schedules, prefixes, formats, field groups, and secrets. Report the area `manual action` until downstream consumers are confirmed.\n\n## Validate and report\n\n- Test applicable hierarchy, root input/output, propagated attributes, public/release/environment behavior, API pagination and parsing, delivery semantics, and absence of deprecated calls.\n- Before production cutover, send representative traces from the migrated instrumentation to a non-production Langfuse project and inspect the resulting observations there. Mocked tests do not verify backend ingestion or project behavior.\n- On a session path, confirm the root and every applicable child observation carry the intended session ID and that session cost includes the cost-bearing children.\n- Re-read rules and integrations after writes. Preserve disabled legacy rules for rollback; never claim completion without checking the Evaluators UI on the target host for legacy rows.\n- The readiness report must contain exactly these seven rows, each marked `ready`, `changed`, `manual action`, or `blocked`: project access; SDK/instrumentation; trace evaluators; dataset evaluators; direct APIs; exports; verification/rollback.\n- In evaluator rows, separate contracts verified against project data from targets suggested only by code inspection. For every row not marked `ready`, include the blocker, next action, and a direct UI link when applicable.",
+ },
+];
diff --git a/packages/langfuse-skills/src/index.d.ts b/packages/langfuse-skills/src/index.d.ts
new file mode 100644
index 0000000000..1b6d4db421
--- /dev/null
+++ b/packages/langfuse-skills/src/index.d.ts
@@ -0,0 +1,7 @@
+export type LangfuseSkill = {
+ name: string;
+ description: string;
+ instructions: string;
+};
+
+export declare const LANGFUSE_SKILLS: readonly LangfuseSkill[];
diff --git a/packages/langfuse-skills/src/index.js b/packages/langfuse-skills/src/index.js
new file mode 100644
index 0000000000..7a0ae107ea
--- /dev/null
+++ b/packages/langfuse-skills/src/index.js
@@ -0,0 +1,3 @@
+const { LANGFUSE_SKILLS } = require("./generated/skills.js");
+
+module.exports = { LANGFUSE_SKILLS };
diff --git a/packages/shared/src/in-app-agent/server/systemPrompt.ts b/packages/shared/src/in-app-agent/server/systemPrompt.ts
index 328c47ff27..4f829315a9 100644
--- a/packages/shared/src/in-app-agent/server/systemPrompt.ts
+++ b/packages/shared/src/in-app-agent/server/systemPrompt.ts
@@ -31,7 +31,7 @@ IMPORTANT: You should NOT answer with unnecessary preamble or postamble (such as
<tools>
Prefer Langfuse MCP tools over sandbox tools whenever an MCP tool can perform the task, especially the metrics tools for aggregations.
Use the docs tools to find relevant general information about Langfuse or best practices.
-For questions about best practices or specific workflows (e.g. setting up evals), use the available Langfuse skills when they apply.
+For questions about best practices or specific workflows (e.g. setting up evals), use the available Langfuse skills when they apply. Select only the skills relevant to the user's request.
If the skill ever tells you to reference the docs, always use the langfuse-docs-mcp server to access those docs.
</tools>
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 83e38ffbf1..579771d605 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -243,6 +243,12 @@ importers:
specifier: ^4.1.10
+ packages/langfuse-skills:
+ devDependencies:
+ prettier:
+ specifier: ^3.8.3
+ version: 3.8.3
+
packages/shared:
dependencies:
'@ag-ui/client':
@@ -1084,6 +1090,9 @@ importers:
'@prisma/instrumentation':
specifier: ^6.19.3
version: 6.19.3(@opentelemetry/[email protected])([email protected])
+ '@repo/langfuse-skills':
+ specifier: workspace:*
+ version: link:../packages/langfuse-skills
ai-sdk-amazon-bedrock-v4:
specifier: npm:@ai-sdk/[email protected]
@@ -1150,9 +1159,6 @@ importers:
uuid:
specifier: ^14.0.0
version: 14.0.0
- yaml:
- specifier: ^2.9.0
- version: 2.9.0
zod:
specifier: 4.3.6
version: 4.3.6
diff --git a/scripts/in-app-agent/README.md b/scripts/in-app-agent/README.md
index 05cba033f7..8ecf5f12b2 100644
--- a/scripts/in-app-agent/README.md
+++ b/scripts/in-app-agent/README.md
@@ -2,31 +2,19 @@
## Skill Sync
-The generated in-app agent skills live at:
+The generic Langfuse skill catalog and its sync script live in the
+`@repo/langfuse-skills` package:
-`worker/src/features/in-app-agent/runtime/skills/generated/raw`
-
-The generated TypeScript index lives at:
-
-`worker/src/features/in-app-agent/runtime/skills/generated/skill-markdown.ts`
-
-The generated module inlines each markdown file as a string literal so it
-compiles under plain `tsc` in `worker`; the `raw/*.md` copies exist
-for reviewable diffs only.
-
-Use `sync-raw-skills.mjs` to download all markdown files from
-`https://github.com/langfuse/skills/tree/main/skills/langfuse/references` whose
-frontmatter either has no access requirements or only
-`LANGFUSE_PROJECT_INTERFACE` in `metadata.required_access`:
+`packages/langfuse-skills/src/generated/skills.ts`
```sh
-node scripts/in-app-agent/sync-raw-skills.mjs
+pnpm --filter @repo/langfuse-skills run sync
```
To verify the checked-in generated skills match upstream without changing files:
```sh
-node scripts/in-app-agent/sync-raw-skills.mjs --check
+pnpm --filter @repo/langfuse-skills run sync:check
```
## Prompt Sync
diff --git a/scripts/in-app-agent/sync-raw-skills.mjs b/scripts/in-app-agent/sync-raw-skills.mjs
deleted file mode 100644
index c0a791c888..0000000000
--- a/scripts/in-app-agent/sync-raw-skills.mjs
+++ /dev/null
@@ -1,269 +0,0 @@
-#!/usr/bin/env node
-
-import {
- mkdirSync,
- readdirSync,
- readFileSync,
- rmSync,
- writeFileSync,
-} from "node:fs";
-import { basename, resolve } from "node:path";
-import process from "node:process";
-import { format as formatWithPrettier } from "prettier";
-
-const repoRoot = resolve(new URL("../..", import.meta.url).pathname);
-const generatedDir = resolve(
- repoRoot,
- "worker/src/features/in-app-agent/runtime/skills/generated",
-);
-const rawTargetDir = resolve(generatedDir, "raw");
-const generatedModuleName = "skill-markdown.ts";
-const sourceApiUrl =
- "https://api.github.com/repos/langfuse/skills/contents/skills/langfuse/references?ref=main";
-
-const requiredAccess = "LANGFUSE_PROJECT_INTERFACE";
-
-const isCheckMode = process.argv.includes("--check");
-
-const getGitHubHeaders = () => ({
- Accept: "application/vnd.github+json",
- Authorization: process.env.GITHUB_TOKEN
- ? `Bearer ${process.env.GITHUB_TOKEN}`
- : undefined,
- "User-Agent": "langfuse-sync-in-app-agent-raw-skills",
-});
-
-const fetchText = async (url) => {
- const response = await fetch(url, {
- headers: getGitHubHeaders(),
- });
-
- if (!response.ok) {
- throw new Error(
- `Failed to fetch ${url}: ${response.status} ${response.statusText}${
- !process.env.GITHUB_TOKEN &&
- (response.status === 404 || response.status === 429)
- ? ". Set GITHUB_TOKEN if the repository is private or CI is hitting GitHub API rate limits."
- : ""
- }`,
- );
- }
-
- return response.text();
-};
-
-const extractFrontmatter = (content) => {
- if (!content.startsWith("---\n")) return null;
-
- const endIndex = content.indexOf("\n---", 4);
- if (endIndex === -1) return null;
-
- return content.slice(4, endIndex);
-};
-
-const hasSupportedAccessRequirements = (content) => {
- const frontmatter = extractFrontmatter(content);
- if (!frontmatter) return false;
-
- const lines = frontmatter.split("\n");
- const metadataIndex = lines.findIndex((line) => line === "metadata:");
- if (metadataIndex === -1) return true;
-
- const requiredAccessIndex = lines.findIndex(
- (line, index) => index > metadataIndex && line === " required_access:",
- );
- if (requiredAccessIndex === -1) return true;
-
- const accessValues = [];
-
- for (let i = requiredAccessIndex + 1; i < lines.length; i += 1) {
- const line = lines[i];
-
- if (line.startsWith(" ") && !line.startsWith(" - ")) {
- break;
- }
-
- if (!line.startsWith(" - ")) {
- continue;
- }
-
- accessValues.push(line.slice(" - ".length).trim());
- }
-
- return accessValues.length === 1 && accessValues[0] === requiredAccess;
-};
-
-const listRemoteMarkdownFiles = async () => {
- const response = await fetch(sourceApiUrl, {
- headers: getGitHubHeaders(),
- });
-
- if (!response.ok) {
- throw new Error(
- `Failed to list remote skills: ${response.status} ${response.statusText}${
- !process.env.GITHUB_TOKEN &&
- (response.status === 404 || response.status === 429)
- ? ". Set GITHUB_TOKEN if the repository is private or CI is hitting GitHub API rate limits."
- : ""
- }`,
- );
- }
-
- const entries = await response.json();
-
- if (!Array.isArray(entries)) {
- throw new Error(
- "Unexpected GitHub API response while listing remote skills",
- );
- }
-
- return entries
- .filter(
- (entry) =>
- entry.type === "file" &&
- typeof entry.name === "string" &&
- entry.name.endsWith(".md") &&
- typeof entry.download_url === "string",
- )
- .sort((left, right) => left.name.localeCompare(right.name));
-};
-
-const getExpectedFiles = async () => {
- const remoteFiles = await listRemoteMarkdownFiles();
- const expectedFiles = [];
-
- for (const remoteFile of remoteFiles) {
- const content = await fetchText(remoteFile.download_url);
-
- if (hasSupportedAccessRequirements(content)) {
- expectedFiles.push({ name: remoteFile.name, content });
- }
- }
-
- return expectedFiles;
-};
-
-const getCurrentManagedFileNames = () =>
- readdirSync(rawTargetDir, { withFileTypes: true })
- .filter(
- (entry) =>
- entry.isFile() &&
- entry.name.endsWith(".md") &&
- entry.name !== "README.md",
- )
- .map((entry) => entry.name)
- .sort((left, right) => left.localeCompare(right));
-
-const renderGeneratedModule = async (files) => {
- // Markdown is inlined as string literals (instead of raw `.md` imports) so
- // the module compiles under plain tsc in worker without bundler
- // support. The raw/*.md files are still written for reviewable diffs.
- const entries = files
- .map(
- ({ name, content }) =>
- ` { fileName: ${JSON.stringify(name)}, markdown: ${JSON.stringify(content)} },`,
- )
- .join("\n");
-
- return formatWithPrettier(
- `// This file is generated by scripts/in-app-agent/sync-raw-skills.mjs.
-// Do not edit it manually.
-
-export const LANGFUSE_IN_APP_AGENT_SKILL_MARKDOWN = [
-${entries}
-] as const;
-`,
- { parser: "typescript" },
- );
-};
-
-const main = async () => {
- const expectedFiles = await getExpectedFiles();
- const expectedFileNames = expectedFiles.map((file) => file.name);
- const generatedModuleContent = await renderGeneratedModule(expectedFiles);
-
- mkdirSync(rawTargetDir, { recursive: true });
-
- if (isCheckMode) {
- const mismatches = [];
- const expectedFileNameSet = new Set(expectedFileNames);
-
- for (const fileName of getCurrentManagedFileNames()) {
- if (!expectedFileNameSet.has(fileName)) {
- mismatches.push(`${fileName} exists locally but is not expected`);
- }
- }
-
- for (const { name, content } of expectedFiles) {
- const localPath = resolve(rawTargetDir, name);
-
- try {
- if (readFileSync(localPath, "utf8") !== content) {
- mismatches.push(`${name} differs from upstream`);
- }
- } catch (error) {
- if (error.code === "ENOENT") {
- mismatches.push(`${name} is missing locally`);
- continue;
- }
-
- throw error;
- }
- }
-
- try {
- if (
- readFileSync(resolve(generatedDir, generatedModuleName), "utf8") !==
- generatedModuleContent
- ) {
- mismatches.push(`${generatedModuleName} differs from generated output`);
- }
- } catch (error) {
- if (error.code === "ENOENT") {
- mismatches.push(`${generatedModuleName} is missing locally`);
- } else {
- throw error;
- }
- }
-
- if (mismatches.length > 0) {
- console.error("Generated in-app agent skills are out of sync:");
- for (const mismatch of mismatches) {
- console.error(`- ${mismatch}`);
- }
- process.exit(1);
- }
-
- console.log(
- `Generated in-app agent skills are in sync: ${expectedFileNames.join(", ")}`,
- );
- return;
- }
-
- const expectedFileNameSet = new Set(expectedFileNames);
-
- for (const fileName of getCurrentManagedFileNames()) {
- if (!expectedFileNameSet.has(fileName)) {
- rmSync(resolve(rawTargetDir, fileName));
- }
- }
-
- for (const { name, content } of expectedFiles) {
- writeFileSync(resolve(rawTargetDir, basename(name)), content);
- }
-
- writeFileSync(
- resolve(generatedDir, generatedModuleName),
- generatedModuleContent,
- );
-
- console.log(`Synced ${expectedFiles.length} generated in-app agent skills:`);
- for (const fileName of expectedFileNames) {
- console.log(`- ${fileName}`);
- }
-};
-
-main().catch((error) => {
- console.error(error);
- process.exit(1);
-});
diff --git a/worker/package.json b/worker/package.json
index 56fe26a8dc..86bf987564 100644
--- a/worker/package.json
+++ b/worker/package.json
@@ -34,6 +34,7 @@
"@anthropic-ai/tokenizer": "^0.0.4",
"@ag-ui/core": "^0.0.52",
"@ag-ui/mastra": "1.1.1",
+ "@repo/langfuse-skills": "workspace:*",
"@appsignal/opentelemetry-instrumentation-bullmq": "^0.8.0",
"@langfuse/shared": "workspace:*",
"@aws-sdk/client-lambda-microvms": "catalog:",
@@ -75,7 +76,6 @@
"stripe": "^17.4.0",
"tiktoken": "^1.0.20",
"uuid": "^14.0.0",
- "yaml": "^2.9.0",
"zod": "^4.3.6"
},
"devDependencies": {
diff --git a/worker/src/features/in-app-agent/runtime/agent.ts b/worker/src/features/in-app-agent/runtime/agent.ts
index 03656a0d4f..3ab74da850 100644
--- a/worker/src/features/in-app-agent/runtime/agent.ts
+++ b/worker/src/features/in-app-agent/runtime/agent.ts
@@ -148,12 +148,14 @@ function formatSandboxContext(sandbox?: InAppAgentSandbox): string {
return "";
}
- return `
-<sandbox_filesystem>
-When working in the sandbox filesystem, assume this layout:
+ return `<sandbox_filesystem>
+The sandbox provides read, write, edit, and bash tools for the current task.
+The sandbox has no egress network connection, so the Langfuse CLI, Langfuse SDKs, and other application-specific CLIs or SDKs cannot act on the user's project or environment from there.
+Use the sandbox to inspect and edit files supplied for this task, write ad-hoc scripts, and efficiently process or prepare data locally.
+You may also process or transform data fetched through MCP tools in the sandbox;
+When working in the sandbox, assume this layout:
- "/workspace" is the current working directory for normal file operations and shell commands.
-</sandbox_filesystem>
-`;
+</sandbox_filesystem>`;
}
/** Run-scoped, not part of the managed prompt: it describes one turn's environment. */
diff --git a/worker/src/features/in-app-agent/runtime/skills.ts b/worker/src/features/in-app-agent/runtime/skills.ts
index 4d32d494ef..d611025ad6 100644
--- a/worker/src/features/in-app-agent/runtime/skills.ts
+++ b/worker/src/features/in-app-agent/runtime/skills.ts
@@ -1,67 +1,7 @@
import { createSkill } from "@mastra/core/skills";
-import { parse } from "yaml";
-import { LANGFUSE_IN_APP_AGENT_SKILL_MARKDOWN } from "./skills/generated/skill-markdown";
+import { LANGFUSE_SKILLS } from "@repo/langfuse-skills";
-type LangfuseInAppAgentSkillDefinition = {
- name: string;
- description: string;
- instructions: string;
-};
-
-function parseSkillMarkdown(
- markdown: unknown,
-): LangfuseInAppAgentSkillDefinition {
- if (typeof markdown !== "string") {
- throw new Error(
- "In-app agent skill markdown import did not resolve to a string.",
- );
- }
-
- const frontmatterMatch = markdown.match(/^---\n([\s\S]*?)\n---\n?/);
- if (!frontmatterMatch) {
- throw new Error("In-app agent skill is missing frontmatter.");
- }
-
- const metadata = parseFrontmatter(frontmatterMatch[1] ?? "");
- const instructions = markdown.slice(frontmatterMatch[0].length).trim();
-
- if (!metadata.name) {
- throw new Error("In-app agent skill frontmatter is missing name.");
- }
-
- if (!metadata.description) {
- throw new Error("In-app agent skill frontmatter is missing description.");
- }
-
- if (!instructions) {
- throw new Error("In-app agent skill is missing instructions.");
- }
-
- return {
- name: metadata.name,
- description: metadata.description,
- instructions,
- };
-}
-
-function parseFrontmatter(frontmatter: string): Record<string, string> {
- const parsed: unknown = parse(frontmatter);
-
- if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
- throw new Error("In-app agent skill frontmatter must be a YAML object.");
- }
-
- const metadata = parsed as Record<string, unknown>;
-
- return Object.fromEntries(
- Object.entries(metadata).flatMap(([key, value]) =>
- typeof value === "string" ? [[key, value]] : [],
- ),
- );
-}
-
-export const LANGFUSE_IN_APP_AGENT_SKILLS =
- LANGFUSE_IN_APP_AGENT_SKILL_MARKDOWN.map(({ markdown }) =>
- createSkill(parseSkillMarkdown(markdown)),
- );
+export const LANGFUSE_IN_APP_AGENT_SKILLS = LANGFUSE_SKILLS.map((skill) =>
+ createSkill(skill),
+);
diff --git a/worker/src/features/in-app-agent/runtime/skills/generated/raw/README.md b/worker/src/features/in-app-agent/runtime/skills/generated/raw/README.md
deleted file mode 100644
index c14e069232..0000000000
--- a/worker/src/features/in-app-agent/runtime/skills/generated/raw/README.md
+++ /dev/null
@@ -1,2 +0,0 @@
-These skills are pulled from https://github.com/langfuse/skills/tree/main, never update them manually.
-Update them by running `scripts/in-app-agent/sync-raw-skills.mjs`, following the README.md in that folder.
diff --git a/worker/src/features/in-app-agent/runtime/skills/generated/raw/cli.md b/worker/src/features/in-app-agent/runtime/skills/generated/raw/cli.md
deleted file mode 100644
index 9aa7f6e3e4..0000000000
--- a/worker/src/features/in-app-agent/runtime/skills/generated/raw/cli.md
+++ /dev/null
@@ -1,60 +0,0 @@
----
-name: langfuse-cli
-description: Langfuse CLI usage reference β€” install, resource/action discovery, credentials, and common usage tips for `langfuse-cli`. Use for further tips on using the Langfuse CLI.
-metadata:
- required_access:
- - LANGFUSE_PROJECT_INTERFACE
----
-
-# Langfuse CLI Reference
-
-Documentation: https://langfuse.com/docs/api-and-data-platform/features/cli
-
-## Install
-
-```bash
-# Run directly (recommended)
-npx langfuse-cli api <resource> <action>
-bunx langfuse-cli api <resource> <action>
-
-# Or install globally
-npm i -g langfuse-cli
-langfuse api <resource> <action>
-```
-
-## Discovery
-
-```bash
-# List all resources and auth info
-langfuse api __schema
-
-# List actions for a resource
-langfuse api <resource> --help
-
-# Show args/options for a specific action
-langfuse api <resource> <action> --help
-
-# Preview the curl command without executing
-langfuse api <resource> <action> --curl
-```
-
-## Credentials
-
-Set environment variables:
-
-```bash
-export LANGFUSE_PUBLIC_KEY=pk-lf-...
-export LANGFUSE_SECRET_KEY=sk-lf-...
-export LANGFUSE_BASE_URL=https://cloud.langfuse.com
-```
-
-## Tips
-
-- Use `--json` for machine-readable JSON output
-- Use `--curl` to preview the HTTP request without executing
-- All list commands support filtering β€” check `<resource> <action> --help` for available options
-- Prefer `observations` over `legacy-observations-v1s` β€” `observations` is the modern high-performance endpoint (cursor pagination, selective field groups); `legacy-observations-v1s` is the deprecated v1
-- Prefer `metrics` over `legacy-metrics-v1s` for the same reason
-- Prefer `scores` over `legacy-score-v1s` for list/get operations
-- For broad trace queries, `traces list` can time out on Langfuse Cloud β€” use `observations list` (with `--trace-id` if you're traversing from a known trace) instead. See the [Observations API docs](https://langfuse.com/docs/api-and-data-platform/features/observations-api) for the v1 β†’ v2 mapping.
-- Pagination: legacy v1 endpoints use `--limit` and `--page`; modern endpoints (`observations`, `metrics`, `scores`) use cursor-based pagination β€” pass `--limit`, then thread `meta.cursor` from the response into the next request's `--cursor`
diff --git a/worker/src/features/in-app-agent/runtime/skills/generated/raw/error-analysis.md b/worker/src/features/in-app-agent/runtime/skills/generated/raw/error-analysis.md
deleted file mode 100644
index 89bc6f5b22..0000000000
--- a/worker/src/features/in-app-agent/runtime/skills/generated/raw/error-analysis.md
+++ /dev/null
@@ -1,88 +0,0 @@
----
-name: langfuse-error-analysis
-description: Deep-dive error analysis of an LLM pipeline or AI application using Langfuse traces.
- Use this skill whenever the user wants to understand why their AI system is producing
- bad outputs, where their pipeline is failing, how to categorise or label failures,
- what to prioritise fixing, or how to set up evaluators. Also trigger for "review my
- traces", "my outputs look wrong", "help me debug my LLM app", "I want to analyse
- errors", "build a failure taxonomy", "what's going wrong with my pipeline", or any
- request to systematically inspect, annotate, or score Langfuse traces. If the user
- is trying to understand or improve the quality of an AI system's outputs, use this skill.
-metadata:
- required_access:
- - LANGFUSE_PROJECT_INTERFACE
----
-
-# Error Analysis
-
-## Primary Guide
-
-**1. Fetch the guide in this blogpost**
-
-https://langfuse.com/guides/cookbook/error-analysis-llm-applications.md
-
-If fetch is not available query for langfuse.com error analysis guide
-
-Read it in full. It defines the authoritative 5-step process (sample selection β†’ open coding β†’ clustering β†’ labelling β†’ deciding what to fix).
-
-**2. Guide the user through this step by step**
-
-You as a coding agent and the user go through this together to perform a full error analysis with their data in langfuse. Do as much of the work as you can directly for the user (look up traces, create annotation queues, ...). Provide them with direct links to UI wherever their action is required. Be proactive and narrate what is going on for the user.
-
-## Rules CRITICAL
-Perform interactions with the user's Langfuse instance yourself rather than telling the user to do them β€” don't say "now do this in Langfuse" when you can do it directly
-But don't barrel through on assumptions: where a step needs the user's judgment or input (e.g. what to fix, how to label, which evaluator), pause and ask before acting
-Use charts where possible to display data
-
----
-
-## Langfuse Implementation Notes
-
-The guide describes the process. These notes cover the Langfuse-specific mechanics required to execute it.
-
-### Credentials
-
-Make sure Langfuse credentials are available before starting β€” a public key (`pk-lf-...`), a secret key (`sk-lf-...`), and the host (e.g. `https://cloud.langfuse.com` for EU, `https://us.cloud.langfuse.com` for US, `https://jp.cloud.langfuse.com` for JP, or a self-hosted URL). If they aren't configured, ask the user to set them β€” do not ask them to paste secret values into chat.
-
-Verify you can actually reach the user's project before proceeding. If access fails, stop and ask the user to check their credentials and host.
-
-### Annotation target: OBSERVATION versus TRACE
-
-> **CRITICAL:** In OpenTelemetry-instrumented apps, trace-level `input`/`output` can be null β€” content often lives in a GENERATION observation. In that case, add the GENERATION observation (not the trace) to the annotation queue, so the content being reviewed is actually visible.
-
-### Annotation queues
-
-> **CRITICAL:** Queues cannot be updated or deleted after creation. Create score configs first, then the queue with all config IDs. To add new configs later, create a new queue.
-
-
-**Always give the user a direct link immediately after creating a queue:**
-
-| Host | URL pattern |
-|------|-------------|
-| EU cloud | `https://cloud.langfuse.com/project/<projectId>/annotation-queues/<queueId>` |
-| US cloud | `https://us.cloud.langfuse.com/project/<projectId>/annotation-queues/<queueId>` |
-| Self-hosted | `<LANGFUSE_BASE_URL>/project/<projectId>/annotation-queues/<queueId>` |
-
-Instruction to give: *"Please open code the first ~50 examples. For each trace, write what you observe in the `open_coding` field (describe behaviour, don't diagnose root causes), then set `pass_fail_assessment` to Pass or Fail."*
-
-
-### Prompt fixes
-
-When a category warrants a prompt fix, always offer the user two options:
-1. Create it as a versioned prompt in Langfuse (tracked, usable via the prompt API)
-2. Draft the specific text change for them to review and apply
-
-### Setup evaluators
-
-When a category warrants an evaluator setup, propose the type of evaluator and offer to set it up for the user
-
-
-### Common gotchas
-
-| Mistake | Fix |
-|---------|-----|
-| Annotating the trace instead of the observation | Target the GENERATION observation, not the trace, when content lives there |
-| Creating a score config without checking existing ones | Check existing score configs first; they can't be deleted |
-| Queue created before score configs | Create configs β†’ collect IDs β†’ create queue |
-| Requesting too many traces at once | Page size is capped (max 100); paginate to get more |
-| No rate limiting on bulk queue item creation | Space out requests to avoid hitting rate limits (429) |
diff --git a/worker/src/features/in-app-agent/runtime/skills/generated/raw/prompt-engineering.md b/worker/src/features/in-app-agent/runtime/skills/generated/raw/prompt-engineering.md
deleted file mode 100644
index 88de94f51e..0000000000
--- a/worker/src/features/in-app-agent/runtime/skills/generated/raw/prompt-engineering.md
+++ /dev/null
@@ -1,35 +0,0 @@
----
-name: langfuse-prompt-engineering
-description: Write or change prompts in Langfuse or code. Use whenever the user asks to create, edit, rewrite, debug, tune, or otherwise modify a prompt, including a small wording or instruction change. Distinct from prompt-migration and judge-calibration.
-metadata:
- required_access: []
----
-
-# Prompt Engineering
-
-## Universal principles
-
-1. **Be specific.** State the required output format, constraints, and, when order matters, numbered steps. Write for someone with minimal context.
-2. **Use labeled sections** for the role, instructions, examples, and context. Use XML tags when they help distinguish rules, data, and variables.
-3. **Give the model a role.** A focused system-prompt sentence can steer tone and behavior.
-4. **Explain why important rules exist** so the model can apply them to adjacent cases.
-
-## Adjusting an existing prompt
-
-1. **Pin down the failure first.** Use concrete misbehaving outputs and name the exact failure mode: wrong format, ignored instruction, wrong tone, hallucination, or another observable error. The task and data show what could fail, not what did fail; do not write a fix before observing the failure.
-2. **Trace the failure to the prompt.** Look for a missing, ambiguous, or conflicting instruction.
-3. **Fix the failure class, not the example.** Generalize the observed mistake into an error class and write the instruction at that level, in your own words. Cover adjacent cases, but do not add definitions, rules, or examples for unobserved problems or encode the failing example itself. If a generalized instruction does not work, report the failure instead of narrowing the rule to that case.
-4. **Preserve existing behavior.** Before changing text, identify what behavior it protects. Avoid contradicting other rules or breaking cases that already work.
-5. **Make the smallest testable edit.** Change one cause per attempt so you can attribute the result. Avoid extra length, emphasis, or all-caps. In Langfuse, save each attempt as a new version or label so you can compare and roll back.
-
-## Model-specific tuning
-
-Identify the target model if you can β€” tuning differs by model, so when you know which model the prompt runs on, follow that model's own prompting guidance.
-
-| Technique | Standard (GPT) | Reasoning (Claude latest) |
-|-----------|----------------|---------------------------|
-| Instruction density | High, prescriptive | Goal-level; don't micro-manage steps |
-| Reasoning | Add explicit chain-of-thought | Native thinking; "think thoroughly" + tune `effort` |
-| Emphasis language | Strong directives fine | Dial back `MUST`/`CRITICAL` |
-| Output format | Prefilling/scaffolds | Prefill removed (4.6+); structured outputs + "no preamble" |
-| Roles | `developer` > `user` > `assistant` | `system` role + `user`/`assistant` |
diff --git a/worker/src/features/in-app-agent/runtime/skills/generated/raw/v4-project-migration.md b/worker/src/features/in-app-agent/runtime/skills/generated/raw/v4-project-migration.md
deleted file mode 100644
index 6801a63d1c..0000000000
--- a/worker/src/features/in-app-agent/runtime/skills/generated/raw/v4-project-migration.md
+++ /dev/null
@@ -1,134 +0,0 @@
----
-name: langfuse-v4-project-migration
-description: Prepare a Langfuse project for the v4 platform by migrating deprecated API calls to their replacement endpoints, inventorying active evaluation rules, migrating legacy trace and dataset evaluators to observation and experiment targets, and moving exports to the enriched observation schema.
-metadata:
- required_access:
- - LANGFUSE_PROJECT_INTERFACE
----
-
-# Langfuse v4 project migration
-
-Use this as the canonical v4 platform migration workflow. A coding agent should execute the SDK and code changes using the SDK and trace-level evaluator upgrade workflows; the Langfuse in-app agent can execute the project steps and produce the same code handoff.
-
-## Sources of truth
-
-Fetch the applicable pages before taking action:
-
-- [Langfuse v4 overview](https://langfuse.com/docs/v4)
-- [Langfuse CLI](https://langfuse.com/docs/api-and-data-platform/features/cli)
-- [SDK upgrade paths](https://langfuse.com/docs/observability/sdk/upgrade-path)
-- [Deprecated API migration](https://langfuse.com/docs/deprecated-api-migration)
-- [Deprecated APIs reference](https://langfuse.com/docs/api-and-data-platform/features/deprecated-apis)
-- [Evaluator migration guide](https://langfuse.com/faq/all/llm-as-a-judge-migration)
-- [Observation evaluator context](https://langfuse.com/docs/evaluation/evaluation-methods/llm-as-a-judge#observation-evaluator-context)
-- [Evaluators API](https://api.reference.langfuse.com/#tag/unstableevaluators)
-- [Evaluation Rules API](https://api.reference.langfuse.com/#tag/unstableevaluationrules)
-- [Blob storage export migration](https://langfuse.com/docs/api-and-data-platform/features/export-to-blob-storage#export-source-fast-preview)
-- [Export field reference](https://langfuse.com/docs/api-and-data-platform/features/blob-storage-export-fields#enriched-vs-legacy-differences)
-- [Mixpanel export migration](https://langfuse.com/integrations/analytics/mixpanel#migrate-export-source)
-- [PostHog export migration](https://langfuse.com/integrations/analytics/posthog#migrate-export-source)
-
-Discover the current API or tool schema before writes; the evaluator endpoints are unstable.
-
-## 1. Set up project access
-
-- Prefer an available Langfuse project interface. Otherwise recommend the [Langfuse CLI](https://langfuse.com/docs/api-and-data-platform/features/cli) and run it directly with `npx langfuse-cli` or `bunx langfuse-cli`; a global installation is optional.
-- Ask the user to configure `LANGFUSE_PUBLIC_KEY`, `LANGFUSE_SECRET_KEY`, and `LANGFUSE_BASE_URL` in their environment. Never ask them to paste secrets into the conversation or commit credentials.
-- Verify access and discover the current resources with `npx langfuse-cli api __schema`. Inspect each resource and action with `--help` before use, and request machine-readable output with `--json`.
-- Confirm the project and host before any read or write. If credentials or a project interface are unavailable, continue with the code migration and report project-side checks as blocked.
-
-## 2. Upgrade SDKs and instrumentation
-
-- For a coding agent with repository access, execute the repository-wide SDK upgrade in the steps below β€” using the version-specific breaking-changes guide under [SDK upgrade paths](https://langfuse.com/docs/observability/sdk/upgrade-path) as the docs reference β€” before declaring the platform migration ready.
-- Inventory every Langfuse SDK, integration package, direct OpenTelemetry exporter, initialization site, and lockfile across the repository. Upgrade each ingestion path to the latest stable release in the major required by the current v4 migration docs, unless the repository has a documented compatibility constraint.
-- Apply every applicable SDK migration step and update removed tracing APIs using the current docs. A dependency-only update is incomplete. Deprecated API routes reached outside the SDK client are migrated in the next step.
-- Use the evaluator migration contract below to consolidate all required evaluation input, output, metadata, tool calls, and propagated filter attributes onto the single target observation.
-- Verify the resolved versions and representative ingestion behavior. If the agent cannot access or edit the codebase, return an exact SDK and instrumentation handoff and keep this area blocked rather than marking it ready.
-
-## 3. Migrate direct API calls
-
-- Fetch the [deprecated API migration guide](https://langfuse.com/docs/deprecated-api-migration) before rewriting any call. It maps every deprecated endpoint to its replacement with parameter mappings, semantic differences, and before/after examples, and is also served as plain markdown at `https://langfuse.com/docs/deprecated-api-migration.md` with stable section anchors for programmatic use.
-- This step covers API usage outside the SDK client methods migrated in the previous step: raw HTTP calls in application code, `curl` in scripts and CI, generated API clients, SDK API namespaces such as `langfuse.api.*` and `langfuse.api.legacy.*`, notebooks, dashboards, and data pipelines. Inventory them by searching for `/api/public` paths and the deprecated endpoint names, not only for client libraries.
-- Map each inventoried call to its replacement using the guide's quick reference and per-endpoint sections; do not rewrite calls from memory. Check the guide before touching a call β€” some routes on the same resources are not deprecated and must stay unchanged.
-- Migrate parameters, pagination, filters, field groups, and response parsing together with the path, following the guide's parameter mappings and semantic differences per endpoint. The replacements are not drop-in; a path-only rewrite is incomplete.
-- Where the guide changes the response shape rather than only the route, rework the downstream consumer to the new model instead of reshaping the response back into the deprecated structure.
-- Confirm the deployment the code targets serves each replacement endpoint before cutover, using the [compatibility matrix](https://langfuse.com/docs/compatibility). Code calling a self-hosted v3 deployment must keep the deprecated routes; report its cutover as blocked on the server upgrade.
-- Verify each rewritten call against real project data, comparing with the deprecated response where it is still available. Return calls that cannot be migrated or verified as part of the deprecated API code handoff instead of marking this area ready.
-
-## 4. Inventory active evaluation rules
-
-- Confirm the project, host, and whether it is Cloud or self-hosted.
-- Page through all available evaluators and evaluation rules. Inspect each referenced evaluator definition, not only its name.
-- Migrate rules whose effective `status` is `active`. Report paused/inactive rules separately; do not reactivate or migrate them unless requested.
-- Use unstable evaluation-rule `list` and `get` as the source of truth for legacy trace rules. If a bulk page fails, retry with `limit=1` and report unreadable entries as blockers.
-- Open the [Evaluators UI](https://cloud.langfuse.com/project/~/evals) and check for active rows marked **Legacy** whenever the interface cannot list those targets. In the in-app agent, redirect the user there. Treat this confirmation as required before declaring the project ready.
-
-## 5. Build an evaluator migration contract
-
-For every active legacy rule, record:
-
-| Field | Required decision |
-| ------------- | -------------------------------------------------------------------------------------------- |
-| Existing rule | Name, evaluator, active status, filters, sampling, and variable mappings |
-| New target | Trace to one observation; dataset/dataset run to experiment |
-| Selector | Stable observation name/type plus any propagated trace-attribute filters |
-| Mapping | Every evaluator variable mapped exactly once to a supported source and optional JSONPath |
-| Required data | Exact input, output, metadata, or tool-call fields that must exist on the target observation |
-| Code handoff | Missing observation fields or attributes that the coding agent must add |
-| Cutover | How the new rule will be verified before the legacy rule is disabled |
-
-- Inspect representative observations instead of guessing the target or data shape.
-- Use observation sources for live rules. Experiment rules may additionally use expected output and experiment-item metadata; confirm the current schema before writing.
-- Observation evaluators see only the matched observation. If the evaluator needs an end-to-end request, response, or summary assembled from multiple steps, target a root observation and require the application to write that context onto it.
-- Rebuild filters and mappings deliberately. Do not assume the UI upgrade wizard semantically preserves a legacy rule.
-- When a trace-level evaluator requires code changes, make the handoff self-contained: include the full legacy filters and mappings, representative observation names/types and payload shapes, the preferred target observation, and each missing field or propagated attribute. The coding agent follows `references/trace-evaluator-upgrade.md`; do not ask it to rediscover project configuration from code.
-
-## 6. Create and cut over successor rules
-
-- Reuse the existing evaluator definition when it remains valid. Create a new evaluator version only when the prompt, output definition, or variable contract must change.
-- Update an existing successor rule instead of creating a duplicate with the same name.
-- Create the observation or experiment successor disabled first when the interface supports it. Validate its target, filters, mappings, sampling, and evaluator-variable coverage against real project data.
-- Enable the successor only after the user approves the write. Verify it on newly ingested data; public evaluation rules are live-ingestion rules and do not perform historical backfills.
-- Compare resulting scores and execution logs. Keep the legacy rule available for rollback until the successor is proven.
-- Disable the active legacy rule in the UI after verification. Do not delete legacy rules or historical scores by default.
-- Re-list rules and re-check the UI. Completion requires no unintended active legacy trace/dataset rule and an active verified successor for every migrated rule.
-
-If the available project interface cannot read or update a legacy rule, provide the exact [Evaluators UI](https://cloud.langfuse.com/project/~/evals) action and retain it as an explicit blocker rather than claiming completion.
-
-## 7. Migrate exports
-
-Switching an integration to enriched-only output is a breaking change for downstream consumers; enabling the dual source (legacy plus enriched observations) is additive and safe. Fetch the documented upgrade paths and follow them instead of restating export mechanics from memory:
-
-- [Blob Storage upgrade path](https://langfuse.com/docs/api-and-data-platform/features/export-to-blob-storage#upgrade-path) and the [enriched vs. legacy field differences](https://langfuse.com/docs/api-and-data-platform/features/blob-storage-export-fields#enriched-vs-legacy-differences)
-- [Mixpanel export migration](https://langfuse.com/integrations/analytics/mixpanel#migrate-export-source) and the [legacy vs. enriched event changes](https://langfuse.com/integrations/analytics/mixpanel#legacy-vs-enriched)
-- [PostHog export migration](https://langfuse.com/integrations/analytics/posthog#migrate-export-source) and the [legacy vs. enriched event changes](https://langfuse.com/integrations/analytics/posthog#legacy-vs-enriched)
-
-Key takeaways:
-
-- Inventory configured Blob Storage, Mixpanel, PostHog, and other export integrations in **Project Settings > Integrations**.
-- Check eligibility first: many integrations can no longer select legacy sources (Langfuse Cloud date cutoffs; self-hosted deployments that completed the v4 migration β€” see the docs for the exact rules). An integration that already exports enriched observations only is complete by default.
-- Follow the documented dual-export transition per integration. Switching from legacy to legacy plus enriched observations is additive and needs no downstream sign-off β€” apply it directly. The switch from the dual source to enriched observations only stops the legacy output and requires the user's explicit confirmation that downstream owners are prepared. Never switch a legacy integration directly to enriched-only.
-- A source change applies to future exports only; already-exported history keeps the legacy shape and is not re-exported.
-- Change only the export source. Leave credentials, schedules, prefixes, file formats, field groups, and integration secrets untouched. Apply the change in the integration settings UI β€” for Blob Storage the documented public API (project-scoped keys) also works; Mixpanel and PostHog have no API migration path.
-
-Effects outside Langfuse β€” spell these out per configured integration before the enriched-only cutover:
-
-- **Blob Storage:** the exported tables, file paths, and column sets change (scores are unaffected). Warehouse loaders, table schemas, trace-observation joins, dashboards, and any storage-event automation keyed on the legacy file paths must be updated β€” the integration setting is the smallest part of the migration. After the switch the legacy directories receive no further files, so pipelines still watching them go silent without an error signal.
-- **Mixpanel and PostHog:** the source determines which events and properties are sent, so dashboards, transformations, funnels, and alerts built on the legacy events in those systems must be revalidated.
-- **Dual mode** exports the same data in both the legacy and the enriched shape by design. Downstream counts, costs, and metrics are inflated only where a consumer ingests both β€” for example a warehouse model loading both observation tables, or analytics queries matching both event sets. Keep each consumer on exactly one shape during the transition.
-- Because history is not re-exported, downstream consumers must either keep handling the legacy schema for historical data or the user must re-export history as documented.
-- Treat the downstream consumer update as part of the migration. Report this area as `manual action`, not `ready`, until the user confirms consumers handle the new schema.
-
-## 8. Report readiness
-
-Return one row per area with `ready`, `changed`, `manual action`, or `blocked`:
-
-- CLI or project-interface setup
-- SDK versions and instrumentation migration, including any code handoff
-- active trace-to-observation evaluator migrations
-- active dataset-to-experiment evaluator migrations
-- direct API call migration, including any deprecated API code handoff
-- Blob Storage and analytics export migrations
-- verification and rollback status
-
-Include direct UI links and the evaluator migration contract so an off-platform coding agent and the in-app agent can continue from the same facts.
diff --git a/worker/src/features/in-app-agent/runtime/skills/generated/skill-markdown.ts b/worker/src/features/in-app-agent/runtime/skills/generated/skill-markdown.ts
deleted file mode 100644
index f043a46d95..0000000000
--- a/worker/src/features/in-app-agent/runtime/skills/generated/skill-markdown.ts
+++ /dev/null
@@ -1,25 +0,0 @@
-// This file is generated by scripts/in-app-agent/sync-raw-skills.mjs.
-// Do not edit it manually.
-
-export const LANGFUSE_IN_APP_AGENT_SKILL_MARKDOWN = [
- {
- fileName: "cli.md",
- markdown:
- "---\nname: langfuse-cli\ndescription: Langfuse CLI usage reference β€” install, resource/action discovery, credentials, and common usage tips for `langfuse-cli`. Use for further tips on using the Langfuse CLI.\nmetadata:\n required_access:\n - LANGFUSE_PROJECT_INTERFACE\n---\n\n# Langfuse CLI Reference\n\nDocumentation: https://langfuse.com/docs/api-and-data-platform/features/cli\n\n## Install\n\n```bash\n# Run directly (recommended)\nnpx langfuse-cli api <resource> <action>\nbunx langfuse-cli api <resource> <action>\n\n# Or install globally\nnpm i -g langfuse-cli\nlangfuse api <resource> <action>\n```\n\n## Discovery\n\n```bash\n# List all resources and auth info\nlangfuse api __schema\n\n# List actions for a resource\nlangfuse api <resource> --help\n\n# Show args/options for a specific action\nlangfuse api <resource> <action> --help\n\n# Preview the curl command without executing\nlangfuse api <resource> <action> --curl\n```\n\n## Credentials\n\nSet environment variables:\n\n```bash\nexport LANGFUSE_PUBLIC_KEY=pk-lf-...\nexport LANGFUSE_SECRET_KEY=sk-lf-...\nexport LANGFUSE_BASE_URL=https://cloud.langfuse.com \n```\n\n## Tips\n\n- Use `--json` for machine-readable JSON output\n- Use `--curl` to preview the HTTP request without executing\n- All list commands support filtering β€” check `<resource> <action> --help` for available options\n- Prefer `observations` over `legacy-observations-v1s` β€” `observations` is the modern high-performance endpoint (cursor pagination, selective field groups); `legacy-observations-v1s` is the deprecated v1\n- Prefer `metrics` over `legacy-metrics-v1s` for the same reason\n- Prefer `scores` over `legacy-score-v1s` for list/get operations\n- For broad trace queries, `traces list` can time out on Langfuse Cloud β€” use `observations list` (with `--trace-id` if you're traversing from a known trace) instead. See the [Observations API docs](https://langfuse.com/docs/api-and-data-platform/features/observations-api) for the v1 β†’ v2 mapping.\n- Pagination: legacy v1 endpoints use `--limit` and `--page`; modern endpoints (`observations`, `metrics`, `scores`) use cursor-based pagination β€” pass `--limit`, then thread `meta.cursor` from the response into the next request's `--cursor`\n",
- },
- {
- fileName: "error-analysis.md",
- markdown:
- '---\nname: langfuse-error-analysis\ndescription: Deep-dive error analysis of an LLM pipeline or AI application using Langfuse traces.\n Use this skill whenever the user wants to understand why their AI system is producing\n bad outputs, where their pipeline is failing, how to categorise or label failures,\n what to prioritise fixing, or how to set up evaluators. Also trigger for "review my\n traces", "my outputs look wrong", "help me debug my LLM app", "I want to analyse\n errors", "build a failure taxonomy", "what\'s going wrong with my pipeline", or any\n request to systematically inspect, annotate, or score Langfuse traces. If the user\n is trying to understand or improve the quality of an AI system\'s outputs, use this skill.\nmetadata:\n required_access:\n - LANGFUSE_PROJECT_INTERFACE\n---\n\n# Error Analysis\n\n## Primary Guide\n\n**1. Fetch the guide in this blogpost**\n\nhttps://langfuse.com/guides/cookbook/error-analysis-llm-applications.md\n\nIf fetch is not available query for langfuse.com error analysis guide\n\nRead it in full. It defines the authoritative 5-step process (sample selection β†’ open coding β†’ clustering β†’ labelling β†’ deciding what to fix).\n\n**2. Guide the user through this step by step**\n\nYou as a coding agent and the user go through this together to perform a full error analysis with their data in langfuse. Do as much of the work as you can directly for the user (look up traces, create annotation queues, ...). Provide them with direct links to UI wherever their action is required. Be proactive and narrate what is going on for the user. \n\n## Rules CRITICAL\nPerform interactions with the user\'s Langfuse instance yourself rather than telling the user to do them β€” don\'t say "now do this in Langfuse" when you can do it directly\nBut don\'t barrel through on assumptions: where a step needs the user\'s judgment or input (e.g. what to fix, how to label, which evaluator), pause and ask before acting\nUse charts where possible to display data\n\n---\n\n## Langfuse Implementation Notes\n\nThe guide describes the process. These notes cover the Langfuse-specific mechanics required to execute it.\n\n### Credentials\n\nMake sure Langfuse credentials are available before starting β€” a public key (`pk-lf-...`), a secret key (`sk-lf-...`), and the host (e.g. `https://cloud.langfuse.com` for EU, `https://us.cloud.langfuse.com` for US, `https://jp.cloud.langfuse.com` for JP, or a self-hosted URL). If they aren\'t configured, ask the user to set them β€” do not ask them to paste secret values into chat.\n\nVerify you can actually reach the user\'s project before proceeding. If access fails, stop and ask the user to check their credentials and host.\n\n### Annotation target: OBSERVATION versus TRACE\n\n> **CRITICAL:** In OpenTelemetry-instrumented apps, trace-level `input`/`output` can be null β€” content often lives in a GENERATION observation. In that case, add the GENERATION observation (not the trace) to the annotation queue, so the content being reviewed is actually visible.\n\n### Annotation queues\n\n> **CRITICAL:** Queues cannot be updated or deleted after creation. Create score configs first, then the queue with all config IDs. To add new configs later, create a new queue.\n\n\n**Always give the user a direct link immediately after creating a queue:**\n\n| Host | URL pattern |\n|------|-------------|\n| EU cloud | `https://cloud.langfuse.com/project/<projectId>/annotation-queues/<queueId>` |\n| US cloud | `https://us.cloud.langfuse.com/project/<projectId>/annotation-queues/<queueId>` |\n| Self-hosted | `<LANGFUSE_BASE_URL>/project/<projectId>/annotation-queues/<queueId>` |\n\nInstruction to give: *"Please open code the first ~50 examples. For each trace, write what you observe in the `open_coding` field (describe behaviour, don\'t diagnose root causes), then set `pass_fail_assessment` to Pass or Fail."*\n\n\n### Prompt fixes\n\nWhen a category warrants a prompt fix, always offer the user two options:\n1. Create it as a versioned prompt in Langfuse (tracked, usable via the prompt API)\n2. Draft the specific text change for them to review and apply\n\n### Setup evaluators\n\nWhen a category warrants an evaluator setup, propose the type of evaluator and offer to set it up for the user\n\n\n### Common gotchas\n\n| Mistake | Fix |\n|---------|-----|\n| Annotating the trace instead of the observation | Target the GENERATION observation, not the trace, when content lives there |\n| Creating a score config without checking existing ones | Check existing score configs first; they can\'t be deleted |\n| Queue created before score configs | Create configs β†’ collect IDs β†’ create queue |\n| Requesting too many traces at once | Page size is capped (max 100); paginate to get more |\n| No rate limiting on bulk queue item creation | Space out requests to avoid hitting rate limits (429) |\n',
- },
- {
- fileName: "prompt-engineering.md",
- markdown:
- '---\nname: langfuse-prompt-engineering\ndescription: Write or change prompts in Langfuse or code. Use whenever the user asks to create, edit, rewrite, debug, tune, or otherwise modify a prompt, including a small wording or instruction change. Distinct from prompt-migration and judge-calibration.\nmetadata:\n required_access: []\n---\n\n# Prompt Engineering\n\n## Universal principles\n\n1. **Be specific.** State the required output format, constraints, and, when order matters, numbered steps. Write for someone with minimal context.\n2. **Use labeled sections** for the role, instructions, examples, and context. Use XML tags when they help distinguish rules, data, and variables.\n3. **Give the model a role.** A focused system-prompt sentence can steer tone and behavior.\n4. **Explain why important rules exist** so the model can apply them to adjacent cases.\n\n## Adjusting an existing prompt\n\n1. **Pin down the failure first.** Use concrete misbehaving outputs and name the exact failure mode: wrong format, ignored instruction, wrong tone, hallucination, or another observable error. The task and data show what could fail, not what did fail; do not write a fix before observing the failure.\n2. **Trace the failure to the prompt.** Look for a missing, ambiguous, or conflicting instruction.\n3. **Fix the failure class, not the example.** Generalize the observed mistake into an error class and write the instruction at that level, in your own words. Cover adjacent cases, but do not add definitions, rules, or examples for unobserved problems or encode the failing example itself. If a generalized instruction does not work, report the failure instead of narrowing the rule to that case.\n4. **Preserve existing behavior.** Before changing text, identify what behavior it protects. Avoid contradicting other rules or breaking cases that already work.\n5. **Make the smallest testable edit.** Change one cause per attempt so you can attribute the result. Avoid extra length, emphasis, or all-caps. In Langfuse, save each attempt as a new version or label so you can compare and roll back.\n\n## Model-specific tuning\n\nIdentify the target model if you can β€” tuning differs by model, so when you know which model the prompt runs on, follow that model\'s own prompting guidance.\n\n| Technique | Standard (GPT) | Reasoning (Claude latest) |\n|-----------|----------------|---------------------------|\n| Instruction density | High, prescriptive | Goal-level; don\'t micro-manage steps |\n| Reasoning | Add explicit chain-of-thought | Native thinking; "think thoroughly" + tune `effort` |\n| Emphasis language | Strong directives fine | Dial back `MUST`/`CRITICAL` |\n| Output format | Prefilling/scaffolds | Prefill removed (4.6+); structured outputs + "no preamble" |\n| Roles | `developer` > `user` > `assistant` | `system` role + `user`/`assistant` |\n',
- },
- {
- fileName: "v4-project-migration.md",
- markdown:
- "---\nname: langfuse-v4-project-migration\ndescription: Prepare a Langfuse project for the v4 platform by migrating deprecated API calls to their replacement endpoints, inventorying active evaluation rules, migrating legacy trace and dataset evaluators to observation and experiment targets, and moving exports to the enriched observation schema.\nmetadata:\n required_access:\n - LANGFUSE_PROJECT_INTERFACE\n---\n\n# Langfuse v4 project migration\n\nUse this as the canonical v4 platform migration workflow. A coding agent should execute the SDK and code changes using the SDK and trace-level evaluator upgrade workflows; the Langfuse in-app agent can execute the project steps and produce the same code handoff.\n\n## Sources of truth\n\nFetch the applicable pages before taking action:\n\n- [Langfuse v4 overview](https://langfuse.com/docs/v4)\n- [Langfuse CLI](https://langfuse.com/docs/api-and-data-platform/features/cli)\n- [SDK upgrade paths](https://langfuse.com/docs/observability/sdk/upgrade-path)\n- [Deprecated API migration](https://langfuse.com/docs/deprecated-api-migration)\n- [Deprecated APIs reference](https://langfuse.com/docs/api-and-data-platform/features/deprecated-apis)\n- [Evaluator migration guide](https://langfuse.com/faq/all/llm-as-a-judge-migration)\n- [Observation evaluator context](https://langfuse.com/docs/evaluation/evaluation-methods/llm-as-a-judge#observation-evaluator-context)\n- [Evaluators API](https://api.reference.langfuse.com/#tag/unstableevaluators)\n- [Evaluation Rules API](https://api.reference.langfuse.com/#tag/unstableevaluationrules)\n- [Blob storage export migration](https://langfuse.com/docs/api-and-data-platform/features/export-to-blob-storage#export-source-fast-preview)\n- [Export field reference](https://langfuse.com/docs/api-and-data-platform/features/blob-storage-export-fields#enriched-vs-legacy-differences)\n- [Mixpanel export migration](https://langfuse.com/integrations/analytics/mixpanel#migrate-export-source)\n- [PostHog export migration](https://langfuse.com/integrations/analytics/posthog#migrate-export-source)\n\nDiscover the current API or tool schema before writes; the evaluator endpoints are unstable.\n\n## 1. Set up project access\n\n- Prefer an available Langfuse project interface. Otherwise recommend the [Langfuse CLI](https://langfuse.com/docs/api-and-data-platform/features/cli) and run it directly with `npx langfuse-cli` or `bunx langfuse-cli`; a global installation is optional.\n- Ask the user to configure `LANGFUSE_PUBLIC_KEY`, `LANGFUSE_SECRET_KEY`, and `LANGFUSE_BASE_URL` in their environment. Never ask them to paste secrets into the conversation or commit credentials.\n- Verify access and discover the current resources with `npx langfuse-cli api __schema`. Inspect each resource and action with `--help` before use, and request machine-readable output with `--json`.\n- Confirm the project and host before any read or write. If credentials or a project interface are unavailable, continue with the code migration and report project-side checks as blocked.\n\n## 2. Upgrade SDKs and instrumentation\n\n- For a coding agent with repository access, execute the repository-wide SDK upgrade in the steps below β€” using the version-specific breaking-changes guide under [SDK upgrade paths](https://langfuse.com/docs/observability/sdk/upgrade-path) as the docs reference β€” before declaring the platform migration ready.\n- Inventory every Langfuse SDK, integration package, direct OpenTelemetry exporter, initialization site, and lockfile across the repository. Upgrade each ingestion path to the latest stable release in the major required by the current v4 migration docs, unless the repository has a documented compatibility constraint.\n- Apply every applicable SDK migration step and update removed tracing APIs using the current docs. A dependency-only update is incomplete. Deprecated API routes reached outside the SDK client are migrated in the next step.\n- Use the evaluator migration contract below to consolidate all required evaluation input, output, metadata, tool calls, and propagated filter attributes onto the single target observation.\n- Verify the resolved versions and representative ingestion behavior. If the agent cannot access or edit the codebase, return an exact SDK and instrumentation handoff and keep this area blocked rather than marking it ready.\n\n## 3. Migrate direct API calls\n\n- Fetch the [deprecated API migration guide](https://langfuse.com/docs/deprecated-api-migration) before rewriting any call. It maps every deprecated endpoint to its replacement with parameter mappings, semantic differences, and before/after examples, and is also served as plain markdown at `https://langfuse.com/docs/deprecated-api-migration.md` with stable section anchors for programmatic use.\n- This step covers API usage outside the SDK client methods migrated in the previous step: raw HTTP calls in application code, `curl` in scripts and CI, generated API clients, SDK API namespaces such as `langfuse.api.*` and `langfuse.api.legacy.*`, notebooks, dashboards, and data pipelines. Inventory them by searching for `/api/public` paths and the deprecated endpoint names, not only for client libraries.\n- Map each inventoried call to its replacement using the guide's quick reference and per-endpoint sections; do not rewrite calls from memory. Check the guide before touching a call β€” some routes on the same resources are not deprecated and must stay unchanged.\n- Migrate parameters, pagination, filters, field groups, and response parsing together with the path, following the guide's parameter mappings and semantic differences per endpoint. The replacements are not drop-in; a path-only rewrite is incomplete.\n- Where the guide changes the response shape rather than only the route, rework the downstream consumer to the new model instead of reshaping the response back into the deprecated structure.\n- Confirm the deployment the code targets serves each replacement endpoint before cutover, using the [compatibility matrix](https://langfuse.com/docs/compatibility). Code calling a self-hosted v3 deployment must keep the deprecated routes; report its cutover as blocked on the server upgrade.\n- Verify each rewritten call against real project data, comparing with the deprecated response where it is still available. Return calls that cannot be migrated or verified as part of the deprecated API code handoff instead of marking this area ready.\n\n## 4. Inventory active evaluation rules\n\n- Confirm the project, host, and whether it is Cloud or self-hosted.\n- Page through all available evaluators and evaluation rules. Inspect each referenced evaluator definition, not only its name.\n- Migrate rules whose effective `status` is `active`. Report paused/inactive rules separately; do not reactivate or migrate them unless requested.\n- Use unstable evaluation-rule `list` and `get` as the source of truth for legacy trace rules. If a bulk page fails, retry with `limit=1` and report unreadable entries as blockers.\n- Open the [Evaluators UI](https://cloud.langfuse.com/project/~/evals) and check for active rows marked **Legacy** whenever the interface cannot list those targets. In the in-app agent, redirect the user there. Treat this confirmation as required before declaring the project ready.\n\n## 5. Build an evaluator migration contract\n\nFor every active legacy rule, record:\n\n| Field | Required decision |\n| ------------- | -------------------------------------------------------------------------------------------- |\n| Existing rule | Name, evaluator, active status, filters, sampling, and variable mappings |\n| New target | Trace to one observation; dataset/dataset run to experiment |\n| Selector | Stable observation name/type plus any propagated trace-attribute filters |\n| Mapping | Every evaluator variable mapped exactly once to a supported source and optional JSONPath |\n| Required data | Exact input, output, metadata, or tool-call fields that must exist on the target observation |\n| Code handoff | Missing observation fields or attributes that the coding agent must add |\n| Cutover | How the new rule will be verified before the legacy rule is disabled |\n\n- Inspect representative observations instead of guessing the target or data shape.\n- Use observation sources for live rules. Experiment rules may additionally use expected output and experiment-item metadata; confirm the current schema before writing.\n- Observation evaluators see only the matched observation. If the evaluator needs an end-to-end request, response, or summary assembled from multiple steps, target a root observation and require the application to write that context onto it.\n- Rebuild filters and mappings deliberately. Do not assume the UI upgrade wizard semantically preserves a legacy rule.\n- When a trace-level evaluator requires code changes, make the handoff self-contained: include the full legacy filters and mappings, representative observation names/types and payload shapes, the preferred target observation, and each missing field or propagated attribute. The coding agent follows `references/trace-evaluator-upgrade.md`; do not ask it to rediscover project configuration from code.\n\n## 6. Create and cut over successor rules\n\n- Reuse the existing evaluator definition when it remains valid. Create a new evaluator version only when the prompt, output definition, or variable contract must change.\n- Update an existing successor rule instead of creating a duplicate with the same name.\n- Create the observation or experiment successor disabled first when the interface supports it. Validate its target, filters, mappings, sampling, and evaluator-variable coverage against real project data.\n- Enable the successor only after the user approves the write. Verify it on newly ingested data; public evaluation rules are live-ingestion rules and do not perform historical backfills.\n- Compare resulting scores and execution logs. Keep the legacy rule available for rollback until the successor is proven.\n- Disable the active legacy rule in the UI after verification. Do not delete legacy rules or historical scores by default.\n- Re-list rules and re-check the UI. Completion requires no unintended active legacy trace/dataset rule and an active verified successor for every migrated rule.\n\nIf the available project interface cannot read or update a legacy rule, provide the exact [Evaluators UI](https://cloud.langfuse.com/project/~/evals) action and retain it as an explicit blocker rather than claiming completion.\n\n## 7. Migrate exports\n\nSwitching an integration to enriched-only output is a breaking change for downstream consumers; enabling the dual source (legacy plus enriched observations) is additive and safe. Fetch the documented upgrade paths and follow them instead of restating export mechanics from memory:\n\n- [Blob Storage upgrade path](https://langfuse.com/docs/api-and-data-platform/features/export-to-blob-storage#upgrade-path) and the [enriched vs. legacy field differences](https://langfuse.com/docs/api-and-data-platform/features/blob-storage-export-fields#enriched-vs-legacy-differences)\n- [Mixpanel export migration](https://langfuse.com/integrations/analytics/mixpanel#migrate-export-source) and the [legacy vs. enriched event changes](https://langfuse.com/integrations/analytics/mixpanel#legacy-vs-enriched)\n- [PostHog export migration](https://langfuse.com/integrations/analytics/posthog#migrate-export-source) and the [legacy vs. enriched event changes](https://langfuse.com/integrations/analytics/posthog#legacy-vs-enriched)\n\nKey takeaways:\n\n- Inventory configured Blob Storage, Mixpanel, PostHog, and other export integrations in **Project Settings > Integrations**.\n- Check eligibility first: many integrations can no longer select legacy sources (Langfuse Cloud date cutoffs; self-hosted deployments that completed the v4 migration β€” see the docs for the exact rules). An integration that already exports enriched observations only is complete by default.\n- Follow the documented dual-export transition per integration. Switching from legacy to legacy plus enriched observations is additive and needs no downstream sign-off β€” apply it directly. The switch from the dual source to enriched observations only stops the legacy output and requires the user's explicit confirmation that downstream owners are prepared. Never switch a legacy integration directly to enriched-only.\n- A source change applies to future exports only; already-exported history keeps the legacy shape and is not re-exported.\n- Change only the export source. Leave credentials, schedules, prefixes, file formats, field groups, and integration secrets untouched. Apply the change in the integration settings UI β€” for Blob Storage the documented public API (project-scoped keys) also works; Mixpanel and PostHog have no API migration path.\n\nEffects outside Langfuse β€” spell these out per configured integration before the enriched-only cutover:\n\n- **Blob Storage:** the exported tables, file paths, and column sets change (scores are unaffected). Warehouse loaders, table schemas, trace-observation joins, dashboards, and any storage-event automation keyed on the legacy file paths must be updated β€” the integration setting is the smallest part of the migration. After the switch the legacy directories receive no further files, so pipelines still watching them go silent without an error signal.\n- **Mixpanel and PostHog:** the source determines which events and properties are sent, so dashboards, transformations, funnels, and alerts built on the legacy events in those systems must be revalidated.\n- **Dual mode** exports the same data in both the legacy and the enriched shape by design. Downstream counts, costs, and metrics are inflated only where a consumer ingests both β€” for example a warehouse model loading both observation tables, or analytics queries matching both event sets. Keep each consumer on exactly one shape during the transition.\n- Because history is not re-exported, downstream consumers must either keep handling the legacy schema for historical data or the user must re-export history as documented.\n- Treat the downstream consumer update as part of the migration. Report this area as `manual action`, not `ready`, until the user confirms consumers handle the new schema.\n\n## 8. Report readiness\n\nReturn one row per area with `ready`, `changed`, `manual action`, or `blocked`:\n\n- CLI or project-interface setup\n- SDK versions and instrumentation migration, including any code handoff\n- active trace-to-observation evaluator migrations\n- active dataset-to-experiment evaluator migrations\n- direct API call migration, including any deprecated API code handoff\n- Blob Storage and analytics export migrations\n- verification and rollback status\n\nInclude direct UI links and the evaluator migration contract so an off-platform coding agent and the in-app agent can continue from the same facts.\n",
- },
-] as const;