Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(datasets): bind secret headers to remote URL (#16307)

Hassieb Pakzad58cc50069c7e
diff --git a/web/src/__tests__/server/datasets-remote-experiment.servertest.ts b/web/src/__tests__/server/datasets-remote-experiment.servertest.ts
index 14d317ca0a..7214c35399 100644
--- a/web/src/__tests__/server/datasets-remote-experiment.servertest.ts
+++ b/web/src/__tests__/server/datasets-remote-experiment.servertest.ts
@@ -8,7 +8,7 @@ import { randomUUID } from "crypto";
const orgIds: string[] = [];
-const prepare = async () => {
+const prepare = async (projectRole: "ADMIN" | "MEMBER" = "ADMIN") => {
const setup = await createOrgProjectAndApiKey();
orgIds.push(setup.orgId);
@@ -31,7 +31,7 @@ const prepare = async () => {
projects: [
{
id: setup.projectId,
- role: "ADMIN",
+ role: projectRole,
name: "Test Project",
deletedAt: null,
retentionDays: null,
@@ -50,7 +50,7 @@ const prepare = async () => {
observationEvals: false,
experimentsV4Enabled: false,
},
- admin: true,
+ admin: false,
},
environment: {} as any,
};
@@ -157,6 +157,7 @@ describe("remote experiment auth headers and signing secret", () => {
prisma.dataset.findUniqueOrThrow({
where: { id_projectId: { id: datasetId, projectId } },
select: {
+ remoteExperimentUrl: true,
remoteExperimentSecretKey: true,
remoteExperimentDisplaySecretKey: true,
remoteExperimentRequestHeaders: true,
@@ -270,11 +271,11 @@ describe("remote experiment auth headers and signing secret", () => {
// The plaintext secret must not appear anywhere in the safe read
expect(JSON.stringify(config)).not.toContain("token-123");
- // URL-only update (no requestHeaders) preserves the headers
+ // Same-URL update (no requestHeaders) preserves the headers
await caller.datasets.upsertRemoteExperiment({
projectId,
datasetId,
- url: "https://example.com/hook-updated",
+ url: "https://example.com/hook",
defaultPayload: "{}",
enabled: true,
});
@@ -282,6 +283,106 @@ describe("remote experiment auth headers and signing secret", () => {
expect(preserved.remoteExperimentRequestHeaders).toEqual(storedHeaders);
});
+ it.each([
+ {
+ name: "omitted headers",
+ requestHeaders: undefined,
+ },
+ {
+ name: "an empty masked value",
+ requestHeaders: {
+ authorization: { secret: true, value: "" },
+ },
+ },
+ ])(
+ "rejects reusing secret headers across a URL change with $name",
+ async ({ requestHeaders }) => {
+ const { caller, projectId } = await prepare("MEMBER");
+ const datasetId = await createDataset(projectId);
+
+ await caller.datasets.upsertRemoteExperiment({
+ projectId,
+ datasetId,
+ url: "https://example.com/hook",
+ defaultPayload: "{}",
+ enabled: true,
+ requestHeaders: {
+ authorization: { secret: true, value: "Bearer token-123" },
+ },
+ });
+
+ await expect(
+ caller.datasets.upsertRemoteExperiment({
+ projectId,
+ datasetId,
+ url: "https://example.com/collect",
+ defaultPayload: "{}",
+ enabled: true,
+ requestHeaders,
+ }),
+ ).rejects.toMatchObject({
+ code: "BAD_REQUEST",
+ message: expect.stringContaining(
+ "Secret headers must be re-entered when changing the remote experiment URL",
+ ),
+ });
+
+ const stored = await selectSecretColumns(datasetId, projectId);
+ expect(stored.remoteExperimentUrl).toBe("https://example.com/hook");
+ expect(stored.remoteExperimentRequestHeaders).toBeDefined();
+ expect(
+ decrypt(
+ (
+ stored.remoteExperimentRequestHeaders as Record<
+ string,
+ { secret: boolean; value: string }
+ >
+ ).authorization.value,
+ ),
+ ).toBe("Bearer token-123");
+ },
+ );
+
+ it("allows a URL change when secret headers are replaced", async () => {
+ const { caller, projectId } = await prepare("MEMBER");
+ const datasetId = await createDataset(projectId);
+
+ await caller.datasets.upsertRemoteExperiment({
+ projectId,
+ datasetId,
+ url: "https://example.com/hook",
+ defaultPayload: "{}",
+ enabled: true,
+ requestHeaders: {
+ authorization: { secret: true, value: "Bearer token-123" },
+ },
+ });
+
+ await caller.datasets.upsertRemoteExperiment({
+ projectId,
+ datasetId,
+ url: "https://example.com/replacement",
+ defaultPayload: "{}",
+ enabled: true,
+ requestHeaders: {
+ authorization: { secret: true, value: "Bearer replacement-token" },
+ },
+ });
+
+ const stored = await selectSecretColumns(datasetId, projectId);
+ expect(stored.remoteExperimentUrl).toBe("https://example.com/replacement");
+ expect(
+ decrypt(
+ (
+ stored.remoteExperimentRequestHeaders as Record<
+ string,
+ { secret: boolean; value: string }
+ >
+ ).authorization.value,
+ ),
+ ).toBe("Bearer replacement-token");
+ });
+
it("excludes secret columns from generic dataset reads via the global Prisma omit", async () => {
const { caller, projectId } = await prepare();
const datasetId = await createDataset(projectId);
diff --git a/web/src/features/datasets/server/dataset-router.ts b/web/src/features/datasets/server/dataset-router.ts
index 24dcef7e2a..5ca59fd096 100644
--- a/web/src/features/datasets/server/dataset-router.ts
+++ b/web/src/features/datasets/server/dataset-router.ts
@@ -2174,11 +2174,47 @@ export const datasetRouter = createTRPCRouter({
});
}
+ const existingHeaders = parseStoredRemoteExperimentHeaders(
+ dataset.remoteExperimentRequestHeaders,
+ );
+ const submittedHeadersByLowerKey = input.requestHeaders
+ ? Object.fromEntries(
+ Object.entries(input.requestHeaders).map(([key, value]) => [
+ key.trim().toLowerCase(),
+ value,
+ ]),
+ )
+ : undefined;
+ const reusesSecretHeader = Object.entries(existingHeaders).some(
+ ([key, existingHeader]) => {
+ if (!existingHeader.secret) {
+ return false;
+ }
+
+ // Omitting requestHeaders preserves every existing header.
+ if (input.requestHeaders === undefined) {
+ return true;
+ }
+
+ const normalizedKey = key.trim().toLowerCase();
+ const submittedHeader = submittedHeadersByLowerKey?.[normalizedKey];
+
+ // An empty submitted value preserves the existing secret.
+ return submittedHeader?.value.trim() === "";
+ },
+ );
+
+ if (input.url !== dataset.remoteExperimentUrl && reusesSecretHeader) {
+ throw new TRPCError({
+ code: "BAD_REQUEST",
+ message:
+ "Secret headers must be re-entered when changing the remote experiment URL",
+ });
+ }
+
const { requestHeaders, displayHeaders } = processRemoteExperimentHeaders(
input.requestHeaders,
- parseStoredRemoteExperimentHeaders(
- dataset.remoteExperimentRequestHeaders,
- ),
+ existingHeaders,
);
const signingSecret =