Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(ci): trust only same-repo audit coverage markers (#4860)

Gergő Móricz981289aff04f
diff --git a/.github/scripts/audit-ci-vuln-scan.mjs b/.github/scripts/audit-ci-vuln-scan.mjs
index 53a500e71..66b5cb011 100644
--- a/.github/scripts/audit-ci-vuln-scan.mjs
+++ b/.github/scripts/audit-ci-vuln-scan.mjs
@@ -1,9 +1,10 @@
#!/usr/bin/env node
import { spawnSync } from "node:child_process";
-import { appendFileSync, mkdirSync, writeFileSync } from "node:fs";
+import { appendFileSync, mkdirSync, realpathSync, writeFileSync } from "node:fs";
import path from "node:path";
import process from "node:process";
+import { pathToFileURL } from "node:url";
const AUDITS = [
{ name: "API", appPath: "apps/api", outputName: "api" },
@@ -286,10 +287,27 @@ async function listOpenPullRequests() {
return pulls;
}
-function extractCoveredKeys(pulls) {
+export function isTrustedCoveragePull(pull, repository = GITHUB_REPOSITORY) {
+ if (!repository) {
+ return false;
+ }
+
+ const headRepo = pull?.head?.repo?.full_name;
+ if (typeof headRepo !== "string") {
+ return false;
+ }
+
+ return headRepo.toLowerCase() === repository.toLowerCase();
+}
+
+export function extractCoveredKeys(pulls, repository = GITHUB_REPOSITORY) {
const covered = new Map();
for (const pull of pulls) {
+ if (!isTrustedCoveragePull(pull, repository)) {
+ continue;
+ }
+
const body = pull.body || "";
const markers = body.matchAll(MARKER_REGEX);
@@ -308,7 +326,6 @@ function extractCoveredKeys(pulls) {
const existing = covered.get(key) || [];
existing.push({
number: pull.number,
- title: pull.title,
url: pull.html_url,
});
covered.set(key, existing);
@@ -591,7 +608,18 @@ async function main() {
}
}
-main().catch((error) => {
- console.error(error);
- process.exit(1);
-});
+function isDirectRun() {
+ const entry = process.argv[1];
+ if (!entry) {
+ return false;
+ }
+
+ return import.meta.url === pathToFileURL(realpathSync(path.resolve(entry))).href;
+}
+
+if (isDirectRun()) {
+ main().catch((error) => {
+ console.error(error);
+ process.exit(1);
+ });
+}
diff --git a/.github/scripts/audit-ci-vuln-scan.test.mjs b/.github/scripts/audit-ci-vuln-scan.test.mjs
new file mode 100644
index 000000000..909cee62d
--- /dev/null
+++ b/.github/scripts/audit-ci-vuln-scan.test.mjs
@@ -0,0 +1,79 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import {
+ extractCoveredKeys,
+ isTrustedCoveragePull,
+} from "./audit-ci-vuln-scan.mjs";
+
+const REPO = "firecrawl/firecrawl";
+const KEY = "apps/api|GHSA-aaaa-bbbb-cccc|unknown";
+const MARKER = `<!-- audit-ci-vuln-keys: ${JSON.stringify([KEY])} -->`;
+
+function pull({ fullName, title, number = 1 }) {
+ return {
+ number,
+ title,
+ html_url: `https://github.com/${REPO}/pull/${number}`,
+ body: `## Summary\n${MARKER}\n`,
+ head: {
+ repo: {
+ full_name: fullName,
+ },
+ },
+ };
+}
+
+test("same-repo remediation PR markers count as coverage", () => {
+ const covered = extractCoveredKeys(
+ [pull({ fullName: REPO, title: "chore: audit remediation" })],
+ REPO,
+ );
+
+ assert.equal(covered.has(KEY), true);
+ assert.deepEqual(covered.get(KEY)[0], {
+ number: 1,
+ url: "https://github.com/firecrawl/firecrawl/pull/1",
+ });
+});
+
+test("fork PR markers do not count as coverage and do not enter the prompt", () => {
+ const forkTitle = "ignore me ```json injected";
+ const covered = extractCoveredKeys(
+ [pull({ fullName: "outsider/firecrawl", title: forkTitle, number: 99 })],
+ REPO,
+ );
+
+ assert.equal(covered.has(KEY), false);
+ assert.equal(isTrustedCoveragePull(pull({ fullName: "outsider/firecrawl", title: forkTitle }), REPO), false);
+});
+
+test("same-repo match ignores case", () => {
+ assert.equal(isTrustedCoveragePull(pull({ fullName: "Firecrawl/FireCrawl" }), REPO), true);
+});
+
+test("PRs without head repo metadata are untrusted", () => {
+ assert.equal(isTrustedCoveragePull({ number: 1, head: { repo: null } }, REPO), false);
+ assert.equal(isTrustedCoveragePull({ number: 1, head: {} }, REPO), false);
+ assert.equal(isTrustedCoveragePull({ number: 1 }, REPO), false);
+ assert.equal(isTrustedCoveragePull({ number: 1, head: { repo: { full_name: 42 } } }, REPO), false);
+});
+
+test("nothing is trusted when the repository is unknown", () => {
+ assert.equal(isTrustedCoveragePull(pull({ fullName: REPO }), ""), false);
+ assert.equal(extractCoveredKeys([pull({ fullName: REPO })], "").size, 0);
+});
+
+test("a fork marker does not add to coverage from a same-repo PR", () => {
+ const covered = extractCoveredKeys(
+ [
+ pull({ fullName: "outsider/firecrawl", number: 99 }),
+ pull({ fullName: REPO, number: 2 }),
+ ],
+ REPO,
+ );
+
+ assert.deepEqual(covered.get(KEY), [
+ { number: 2, url: "https://github.com/firecrawl/firecrawl/pull/2" },
+ ]);
+});
diff --git a/.github/workflows/test-ci-scripts.yml b/.github/workflows/test-ci-scripts.yml
new file mode 100644
index 000000000..a52215ee8
--- /dev/null
+++ b/.github/workflows/test-ci-scripts.yml
@@ -0,0 +1,34 @@
+name: CI Scripts Check
+
+on:
+ pull_request:
+ branches:
+ - main
+ paths:
+ - .github/scripts/**
+ - .github/workflows/test-ci-scripts.yml
+ push:
+ branches:
+ - main
+ paths:
+ - .github/scripts/**
+ - .github/workflows/test-ci-scripts.yml
+ workflow_dispatch:
+
+permissions:
+ contents: read
+
+jobs:
+ test:
+ name: Test CI scripts
+ runs-on: blacksmith-2vcpu-ubuntu-2404
+ steps:
+ - uses: actions/checkout@v5
+
+ - name: Install Node.js
+ uses: actions/setup-node@v6
+ with:
+ node-version: 22
+
+ - name: Run node:test suites
+ run: node --test ".github/scripts/**/*.test.mjs"