🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(datasets): bind secret headers to remote URL (#16307)
Hassieb Pakzad58cc50069c7e
diff --git a/web/src/__tests__/server/datasets-remote-experiment.servertest.ts b/web/src/__tests__/server/datasets-remote-experiment.servertest.tsindex 14d317ca0a..7214c35399 100644--- a/web/src/__tests__/server/datasets-remote-experiment.servertest.ts+++ b/web/src/__tests__/server/datasets-remote-experiment.servertest.ts@@ -8,7 +8,7 @@ import { randomUUID } from "crypto";const orgIds: string[] = [];-const prepare = async () => {+const prepare = async (projectRole: "ADMIN" | "MEMBER" = "ADMIN") => {const setup = await createOrgProjectAndApiKey();orgIds.push(setup.orgId);@@ -31,7 +31,7 @@ const prepare = async () => {projects: [{id: setup.projectId,- role: "ADMIN",+ role: projectRole,name: "Test Project",deletedAt: null,retentionDays: null,@@ -50,7 +50,7 @@ const prepare = async () => {observationEvals: false,experimentsV4Enabled: false,},- admin: true,+ admin: false,},environment: {} as any,};@@ -157,6 +157,7 @@ describe("remote experiment auth headers and signing secret", () => {prisma.dataset.findUniqueOrThrow({where: { id_projectId: { id: datasetId, projectId } },select: {+ remoteExperimentUrl: true,remoteExperimentSecretKey: true,remoteExperimentDisplaySecretKey: true,remoteExperimentRequestHeaders: true,@@ -270,11 +271,11 @@ describe("remote experiment auth headers and signing secret", () => {// The plaintext secret must not appear anywhere in the safe readexpect(JSON.stringify(config)).not.toContain("token-123");- // URL-only update (no requestHeaders) preserves the headers+ // Same-URL update (no requestHeaders) preserves the headersawait caller.datasets.upsertRemoteExperiment({projectId,datasetId,- url: "https://example.com/hook-updated",+ url: "https://example.com/hook",defaultPayload: "{}",enabled: true,});@@ -282,6 +283,106 @@ describe("remote experiment auth headers and signing secret", () => {expect(preserved.remoteExperimentRequestHeaders).toEqual(storedHeaders);});+ it.each([+ {+ name: "omitted headers",+ requestHeaders: undefined,+ },+ {+ name: "an empty masked value",+ requestHeaders: {+ authorization: { secret: true, value: "" },+ },+ },+ ])(+ "rejects reusing secret headers across a URL change with $name",+ async ({ requestHeaders }) => {+ const { caller, projectId } = await prepare("MEMBER");+ const datasetId = await createDataset(projectId);++ await caller.datasets.upsertRemoteExperiment({+ projectId,+ datasetId,+ url: "https://example.com/hook",+ defaultPayload: "{}",+ enabled: true,+ requestHeaders: {+ authorization: { secret: true, value: "Bearer token-123" },+ },+ });++ await expect(+ caller.datasets.upsertRemoteExperiment({+ projectId,+ datasetId,+ url: "https://example.com/collect",+ defaultPayload: "{}",+ enabled: true,+ requestHeaders,+ }),+ ).rejects.toMatchObject({+ code: "BAD_REQUEST",+ message: expect.stringContaining(+ "Secret headers must be re-entered when changing the remote experiment URL",+ ),+ });++ const stored = await selectSecretColumns(datasetId, projectId);+ expect(stored.remoteExperimentUrl).toBe("https://example.com/hook");+ expect(stored.remoteExperimentRequestHeaders).toBeDefined();+ expect(+ decrypt(+ (+ stored.remoteExperimentRequestHeaders as Record<+ string,+ { secret: boolean; value: string }+ >+ ).authorization.value,+ ),+ ).toBe("Bearer token-123");+ },+ );++ it("allows a URL change when secret headers are replaced", async () => {+ const { caller, projectId } = await prepare("MEMBER");+ const datasetId = await createDataset(projectId);++ await caller.datasets.upsertRemoteExperiment({+ projectId,+ datasetId,+ url: "https://example.com/hook",+ defaultPayload: "{}",+ enabled: true,+ requestHeaders: {+ authorization: { secret: true, value: "Bearer token-123" },+ },+ });++ await caller.datasets.upsertRemoteExperiment({+ projectId,+ datasetId,+ url: "https://example.com/replacement",+ defaultPayload: "{}",+ enabled: true,+ requestHeaders: {+ authorization: { secret: true, value: "Bearer replacement-token" },+ },+ });++ const stored = await selectSecretColumns(datasetId, projectId);+ expect(stored.remoteExperimentUrl).toBe("https://example.com/replacement");+ expect(+ decrypt(+ (+ stored.remoteExperimentRequestHeaders as Record<+ string,+ { secret: boolean; value: string }+ >+ ).authorization.value,+ ),+ ).toBe("Bearer replacement-token");+ });+it("excludes secret columns from generic dataset reads via the global Prisma omit", async () => {const { caller, projectId } = await prepare();const datasetId = await createDataset(projectId);diff --git a/web/src/features/datasets/server/dataset-router.ts b/web/src/features/datasets/server/dataset-router.tsindex 24dcef7e2a..5ca59fd096 100644--- a/web/src/features/datasets/server/dataset-router.ts+++ b/web/src/features/datasets/server/dataset-router.ts@@ -2174,11 +2174,47 @@ export const datasetRouter = createTRPCRouter({});}+ const existingHeaders = parseStoredRemoteExperimentHeaders(+ dataset.remoteExperimentRequestHeaders,+ );+ const submittedHeadersByLowerKey = input.requestHeaders+ ? Object.fromEntries(+ Object.entries(input.requestHeaders).map(([key, value]) => [+ key.trim().toLowerCase(),+ value,+ ]),+ )+ : undefined;+ const reusesSecretHeader = Object.entries(existingHeaders).some(+ ([key, existingHeader]) => {+ if (!existingHeader.secret) {+ return false;+ }++ // Omitting requestHeaders preserves every existing header.+ if (input.requestHeaders === undefined) {+ return true;+ }++ const normalizedKey = key.trim().toLowerCase();+ const submittedHeader = submittedHeadersByLowerKey?.[normalizedKey];++ // An empty submitted value preserves the existing secret.+ return submittedHeader?.value.trim() === "";+ },+ );++ if (input.url !== dataset.remoteExperimentUrl && reusesSecretHeader) {+ throw new TRPCError({+ code: "BAD_REQUEST",+ message:+ "Secret headers must be re-entered when changing the remote experiment URL",+ });+ }+const { requestHeaders, displayHeaders } = processRemoteExperimentHeaders(input.requestHeaders,- parseStoredRemoteExperimentHeaders(- dataset.remoteExperimentRequestHeaders,- ),+ existingHeaders,);const signingSecret =