Skip to content

The open-source app everyone uses to manage agents at work

fix(runner): scope live eval tokens to eval repo (#12911)

Dotta165ca56a22ad
diff --git a/.github/workflows/runner-protocol-live-evals.yml b/.github/workflows/runner-protocol-live-evals.yml
index 9f6352b033..060a7309bd 100644
--- a/.github/workflows/runner-protocol-live-evals.yml
+++ b/.github/workflows/runner-protocol-live-evals.yml
@@ -105,7 +105,7 @@ jobs:
id: evals_token
env:
COMMITPERCLIP_KEY: ${{ secrets.COMMITPERCLIP_KEY }}
- GH_REPO: ${{ github.repository }}
+ GH_REPO: paperclipai/paperclip-evals
run: |
set -euo pipefail
token="$(node .github/scripts/get-bot-token.mjs)"
@@ -177,7 +177,7 @@ jobs:
id: evals_token
env:
COMMITPERCLIP_KEY: ${{ secrets.COMMITPERCLIP_KEY }}
- GH_REPO: ${{ github.repository }}
+ GH_REPO: paperclipai/paperclip-evals
run: |
set -euo pipefail
token="$(node .github/scripts/get-bot-token.mjs)"
@@ -330,7 +330,7 @@ jobs:
id: evals_token
env:
COMMITPERCLIP_KEY: ${{ secrets.COMMITPERCLIP_KEY }}
- GH_REPO: ${{ github.repository }}
+ GH_REPO: paperclipai/paperclip-evals
run: |
set -euo pipefail
token="$(node .github/scripts/get-bot-token.mjs)"
@@ -500,7 +500,7 @@ jobs:
id: evals_token
env:
COMMITPERCLIP_KEY: ${{ secrets.COMMITPERCLIP_KEY }}
- GH_REPO: ${{ github.repository }}
+ GH_REPO: paperclipai/paperclip-evals
run: |
set -euo pipefail
token="$(node .github/scripts/get-bot-token.mjs)"
diff --git a/packages/paperclip-runner/scripts/runner-protocol-eval-workflow-security.test.mjs b/packages/paperclip-runner/scripts/runner-protocol-eval-workflow-security.test.mjs
index 5e9f990cb8..d90a69e58e 100644
--- a/packages/paperclip-runner/scripts/runner-protocol-eval-workflow-security.test.mjs
+++ b/packages/paperclip-runner/scripts/runner-protocol-eval-workflow-security.test.mjs
@@ -45,6 +45,7 @@ test("resolves both repositories immutably and bounds total matrix concurrency",
/repos\/paperclipai\/paperclip-evals\/commits\/\$EVALS_SHA/u,
);
assert.match(authorize, /COMMITPERCLIP_KEY/u);
+ assert.match(authorize, /GH_REPO: paperclipai\/paperclip-evals/u);
assert.match(
authorize,
/GH_TOKEN: \$\{\{ steps\.evals_token\.outputs\.value \}\}/u,
@@ -66,6 +67,19 @@ test("resolves both repositories immutably and bounds total matrix concurrency",
),
];
assert.equal(privateCheckouts.length, 3);
+ const privateTokenSteps = [
+ ...workflow.matchAll(
+ /^ - name: Generate private eval-repository token\n(?<body>(?:^ {8,}.*\n?)*)/gmu,
+ ),
+ ];
+ assert.equal(privateTokenSteps.length, 4);
+ for (const tokenStep of privateTokenSteps) {
+ assert.match(
+ tokenStep.groups.body,
+ /^ {10}GH_REPO: paperclipai\/paperclip-evals$/mu,
+ "every private-eval token must be minted from the eval repository installation",
+ );
+ }
for (const checkout of privateCheckouts) {
assert.match(
checkout[0],