🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(automations): require fresh tokens for dispatch URL changes (#16479)
Hassieb Pakzadbbc391cf5770
diff --git a/web/src/__tests__/server/automations-trpc.servertest.ts b/web/src/__tests__/server/automations-trpc.servertest.tsindex 65db99bf52..d73fdc8cb6 100644--- a/web/src/__tests__/server/automations-trpc.servertest.ts+++ b/web/src/__tests__/server/automations-trpc.servertest.ts@@ -2660,7 +2660,7 @@ describe("automations trpc", () => {});describe("automations.updateAutomation with GITHUB_DISPATCH", () => {- it("should update GitHub dispatch automation URL without requiring token", async () => {+ it("should reject a GitHub dispatch URL update without a new token", async () => {const { project, caller } = await prepare();// Create initial automation@@ -2699,42 +2699,34 @@ describe("automations trpc", () => {},});- // Update URL and event type without providing new token- const response = await caller.automations.updateAutomation({- projectId: project.id,- automationId: automation.id,- name: "Updated GitHub Dispatch",- eventSource: "prompt",- eventAction: ["created", "updated"],- filter: [],- status: JobConfigState.ACTIVE,- actionType: "GITHUB_DISPATCH",- actionConfig: {- type: "GITHUB_DISPATCH",- url: "https://api.github.com/repos/owner/new-repo/dispatches",- eventType: "new-event",- githubToken: "", // Empty token means keep existing- },- });-- expect(response.action.id).toBe(action.id);- expect(response.action.type).toBe("GITHUB_DISPATCH");-- const config = response.action.config as any;- expect(config.url).toBe(- "https://api.github.com/repos/owner/new-repo/dispatches",+ await expect(+ caller.automations.updateAutomation({+ projectId: project.id,+ automationId: automation.id,+ name: "Updated GitHub Dispatch",+ eventSource: "prompt",+ eventAction: ["created", "updated"],+ filter: [],+ status: JobConfigState.ACTIVE,+ actionType: "GITHUB_DISPATCH",+ actionConfig: {+ type: "GITHUB_DISPATCH",+ url: "https://api.github.com/repos/owner/new-repo/dispatches",+ eventType: "new-event",+ },+ }),+ ).rejects.toThrow(+ "GitHub Personal Access Token is required when changing the dispatch URL",);- expect(config.eventType).toBe("new-event");- expect(config.displayGitHubToken).toBe("ghp_...ken"); // Preserved- // Verify secrets not exposed in response- expect(config).not.toHaveProperty("githubToken");-- // Verify the automation name was updated- const updatedAutomation = await prisma.automation.findFirst({- where: { id: automation.id },+ const unchangedAction = await prisma.action.findUniqueOrThrow({+ where: { id: action.id },});- expect(updatedAutomation?.name).toBe("Updated GitHub Dispatch");+ const unchangedConfig = unchangedAction.config as any;+ expect(unchangedConfig.url).toBe(+ "https://api.github.com/repos/owner/repo/dispatches",+ );+ expect(decrypt(unchangedConfig.githubToken)).toBe("ghp_old_token");});it("should update GitHub dispatch automation with new token", async () => {@@ -2776,7 +2768,7 @@ describe("automations trpc", () => {},});- // Update with new token+ // Update the URL and rotate the token together.const response = await caller.automations.updateAutomation({projectId: project.id,automationId: automation.id,@@ -2788,7 +2780,7 @@ describe("automations trpc", () => {actionType: "GITHUB_DISPATCH",actionConfig: {type: "GITHUB_DISPATCH",- url: "https://api.github.com/repos/owner/repo/dispatches",+ url: "https://api.github.com/repos/owner/new-repo/dispatches",githubToken: "ghp_new_token_456",},});@@ -2810,9 +2802,12 @@ describe("automations trpc", () => {const dbConfig = updatedAction?.config as any;expect(dbConfig.githubToken).not.toBe("ghp_new_token_456"); // Encryptedexpect(decrypt(dbConfig.githubToken)).toBe("ghp_new_token_456"); // Can decrypt+ expect(dbConfig.url).toBe(+ "https://api.github.com/repos/owner/new-repo/dispatches",+ );});- it("should preserve encrypted token when updating without providing new token", async () => {+ it("should preserve encrypted token when updating an unchanged URL", async () => {const { project, caller } = await prepare();const originalToken = "ghp_original_token_xyz";@@ -2837,7 +2832,7 @@ describe("automations trpc", () => {type: "GITHUB_DISPATCH",config: {type: "GITHUB_DISPATCH",- url: "https://api.github.com/repos/owner/repo/dispatches",+ url: "https://github.com/api/v3/repos/owner/repo/dispatches",eventType: "original-event-type",githubToken: encryptedToken,displayGitHubToken: "ghp_...xyz",@@ -2854,7 +2849,9 @@ describe("automations trpc", () => {},});- // Update URL without providing token+ // Update the event type using an equivalent URL representation without+ // providing a token. Hostnames are case-insensitive and :443 is the+ // default port for HTTPS, so this remains the same destination.await caller.automations.updateAutomation({projectId: project.id,automationId: automation.id,@@ -2866,7 +2863,7 @@ describe("automations trpc", () => {actionType: "GITHUB_DISPATCH",actionConfig: {type: "GITHUB_DISPATCH",- url: "https://api.github.com/repos/new-owner/new-repo/dispatches",+ url: "https://GITHUB.COM:443/api/v3/repos/owner/repo/dispatches",eventType: "new-event-type",// githubToken intentionally omitted},@@ -2887,7 +2884,7 @@ describe("automations trpc", () => {// Verify URL and eventType were updatedexpect(dbConfig.url).toBe(- "https://api.github.com/repos/new-owner/new-repo/dispatches",+ "https://GITHUB.COM:443/api/v3/repos/owner/repo/dispatches",);expect(dbConfig.eventType).toBe("new-event-type");});diff --git a/web/src/features/automations/components/actions/GitHubDispatchActionForm.tsx b/web/src/features/automations/components/actions/GitHubDispatchActionForm.tsxindex c08937eba2..18f7507b2f 100644--- a/web/src/features/automations/components/actions/GitHubDispatchActionForm.tsx+++ b/web/src/features/automations/components/actions/GitHubDispatchActionForm.tsx@@ -11,6 +11,7 @@ import { type UseFormReturn } from "react-hook-form";import { type ActionDomain } from "@langfuse/shared";import { ExternalLink } from "lucide-react";import Link from "next/link";+import { areGitHubDispatchUrlsEquivalent } from "../../githubDispatchUrl";interface GitHubDispatchActionFormProps {form: UseFormReturn<any>;@@ -23,6 +24,11 @@ export const GitHubDispatchActionForm: React.FC<GitHubDispatchActionFormProps> = ({ form, disabled }) => {const displayGitHubToken = form.watch("githubDispatch.displayGitHubToken");+ const currentUrl = form.watch("githubDispatch.url");+ const originalUrl = form.watch("githubDispatch.originalUrl");+ const isUrlChanged =+ originalUrl !== undefined &&+ !areGitHubDispatchUrlsEquivalent(currentUrl, originalUrl);return (<div className="space-y-4">@@ -93,7 +99,7 @@ export const GitHubDispatchActionForm: React.FC<<FormItem><FormLabel className="flex items-center">GitHub Personal Access Token- {!displayGitHubToken && (+ {(!displayGitHubToken || isUrlChanged) && (<span className="text-destructive ml-1">*</span>)}</FormLabel>@@ -108,9 +114,11 @@ export const GitHubDispatchActionForm: React.FC<<FormDescription>GitHub PAT with <code className="text-xs">repo</code> scope forrepository dispatch.- {displayGitHubToken- ? " Leave empty to keep existing token."- : ""}{" "}+ {isUrlChanged+ ? " Enter a new token when changing the dispatch URL."+ : displayGitHubToken+ ? " Leave empty to keep existing token."+ : ""}{" "}<Linkhref="https://github.com/settings/tokens/new?scopes=repo&description=Langfuse%20Automation"target="_blank"diff --git a/web/src/features/automations/components/actions/GitHubDispatchActionHandler.clienttest.ts b/web/src/features/automations/components/actions/GitHubDispatchActionHandler.clienttest.tsnew file mode 100644index 0000000000..e19f87ea21--- /dev/null+++ b/web/src/features/automations/components/actions/GitHubDispatchActionHandler.clienttest.ts@@ -0,0 +1,43 @@+// @vitest-environment node++import { describe, expect, it } from "vitest";+import { GitHubDispatchActionHandler } from "./GitHubDispatchActionHandler";++const handler = new GitHubDispatchActionHandler();++const formData = (overrides: Record<string, string> = {}) => ({+ githubDispatch: {+ url: overrides.url ?? "https://api.github.com/repos/owner/repo/dispatches",+ originalUrl: overrides.originalUrl,+ eventType: "repository-updated",+ githubToken: overrides.githubToken ?? "",+ displayGitHubToken: overrides.displayGitHubToken ?? "ghp_...ken",+ },+});++describe("GitHubDispatchActionHandler URL and token validation", () => {+ it("requires a new token when the dispatch URL changes", () => {+ const result = handler.validateFormData(+ formData({+ originalUrl: "https://api.github.com/repos/owner/repo/dispatches",+ url: "https://api.github.com/repos/owner/other-repo/dispatches",+ }),+ );++ expect(result).toEqual({+ isValid: false,+ errors: ["GitHub token is required when changing the dispatch URL"],+ });+ });++ it("allows an update without a token for an equivalent URL", () => {+ const result = handler.validateFormData(+ formData({+ originalUrl: "https://github.com/api/v3/repos/owner/repo/dispatches",+ url: "https://GITHUB.COM:443/api/v3/repos/owner/repo/dispatches",+ }),+ );++ expect(result).toEqual({ isValid: true });+ });+});diff --git a/web/src/features/automations/components/actions/GitHubDispatchActionHandler.tsx b/web/src/features/automations/components/actions/GitHubDispatchActionHandler.tsxindex 0771d63992..e9200a5942 100644--- a/web/src/features/automations/components/actions/GitHubDispatchActionHandler.tsx+++ b/web/src/features/automations/components/actions/GitHubDispatchActionHandler.tsx@@ -8,6 +8,7 @@ import {type ActionDomain,} from "@langfuse/shared";import { z } from "zod";+import { areGitHubDispatchUrlsEquivalent } from "../../githubDispatchUrl";// Define the form schema for GitHub dispatch actionsexport const GitHubDispatchActionFormSchema = z.object({@@ -16,6 +17,7 @@ export const GitHubDispatchActionFormSchema = z.object({eventType: z.string().min(1, "Event type is required").max(100),githubToken: z.string(),displayGitHubToken: z.string().optional(), // Display value for existing token+ originalUrl: z.string().optional(),}),});@@ -50,6 +52,12 @@ export class GitHubDispatchActionHandler implements BaseActionHandler<GitHubDisp"displayGitHubToken" in automation.action.config? automation.action.config.displayGitHubToken: undefined,+ originalUrl:+ automation?.action?.type === "GITHUB_DISPATCH" &&+ automation?.action?.config &&+ "url" in automation.action.config+ ? automation.action.config.url+ : undefined,},};}@@ -70,9 +78,18 @@ export class GitHubDispatchActionHandler implements BaseActionHandler<GitHubDisperrors.push("Event type must be 100 characters or less");}- // Token is required only if there's no existing token (displayGitHubToken)- if (- !formData.githubDispatch?.githubToken &&+ const existingUrl = formData.githubDispatch?.originalUrl;+ const isUrlChanged =+ existingUrl !== undefined &&+ !areGitHubDispatchUrlsEquivalent(+ formData.githubDispatch.url,+ existingUrl,+ );++ if (isUrlChanged && !formData.githubDispatch?.githubToken.trim()) {+ errors.push("GitHub token is required when changing the dispatch URL");+ } else if (+ !formData.githubDispatch?.githubToken.trim() &&!formData.githubDispatch?.displayGitHubToken) {errors.push("GitHub token is required");diff --git a/web/src/features/automations/components/automationForm.tsx b/web/src/features/automations/components/automationForm.tsxindex 38fc30db4f..ade3f18eca 100644--- a/web/src/features/automations/components/automationForm.tsx+++ b/web/src/features/automations/components/automationForm.tsx@@ -68,6 +68,7 @@ const githubDispatchSchema = z.object({eventType: z.string().min(1, "Event type is required").max(100),githubToken: z.string(),displayGitHubToken: z.string().optional(),+ originalUrl: z.string().optional(),});/** promptEventActionDefaults is the default eventAction set for a fresh prompt-source automation. */@@ -499,6 +500,7 @@ export const AutomationForm = ({githubToken: githubDefaults.githubDispatch.githubToken || "",displayGitHubToken:githubDefaults.githubDispatch.displayGitHubToken || undefined,+ originalUrl: githubDefaults.githubDispatch.originalUrl,},};}diff --git a/web/src/features/automations/githubDispatchUrl.ts b/web/src/features/automations/githubDispatchUrl.tsnew file mode 100644index 0000000000..a5c6ea58d0--- /dev/null+++ b/web/src/features/automations/githubDispatchUrl.ts@@ -0,0 +1,10 @@+export function areGitHubDispatchUrlsEquivalent(+ firstUrl: string,+ secondUrl: string,+): boolean {+ try {+ return new URL(firstUrl).href === new URL(secondUrl).href;+ } catch {+ return firstUrl === secondUrl;+ }+}diff --git a/web/src/features/automations/server/githubDispatchHelpers.ts b/web/src/features/automations/server/githubDispatchHelpers.tsindex 0dac565228..601aa012a1 100644--- a/web/src/features/automations/server/githubDispatchHelpers.ts+++ b/web/src/features/automations/server/githubDispatchHelpers.ts@@ -11,6 +11,7 @@ import {validateWebhookURL,} from "@langfuse/shared/src/server";import { TRPCError } from "@trpc/server";+import { areGitHubDispatchUrlsEquivalent } from "../githubDispatchUrl";interface GitHubDispatchConfigOptions {actionConfig: ActionCreate;@@ -82,6 +83,23 @@ export async function processGitHubDispatchActionConfig({});}+ const isUrlChanged =+ existingActionConfig !== undefined &&+ !areGitHubDispatchUrlsEquivalent(urlToUse, existingActionConfig.url);+ const newGithubToken =+ typeof gitHubDispatchConfig.githubToken === "string" &&+ gitHubDispatchConfig.githubToken.trim() !== ""+ ? gitHubDispatchConfig.githubToken+ : undefined;++ if (isUrlChanged && !newGithubToken) {+ throw new TRPCError({+ code: "BAD_REQUEST",+ message:+ "GitHub Personal Access Token is required when changing the dispatch URL",+ });+ }+// Determine event type to uselet eventTypeToUse: string;if (@@ -102,11 +120,8 @@ export async function processGitHubDispatchActionConfig({let displayToken: string;let returnToken: string | undefined;- if (- gitHubDispatchConfig.githubToken &&- gitHubDispatchConfig.githubToken.trim() !== ""- ) {- tokenToUse = gitHubDispatchConfig.githubToken;+ if (newGithubToken) {+ tokenToUse = newGithubToken;displayToken = maskGitHubToken(tokenToUse);returnToken = tokenToUse;} else if (existingActionConfig?.githubToken) {