Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

test(ci): allow only git and echo in the PHP token job, pin the scan env

Yudhi Armyndharis3b36bf1b5a02
diff --git a/src/common/ci-privilege-gate.spec.ts b/src/common/ci-privilege-gate.spec.ts
index aa1834244..59863d502 100644
--- a/src/common/ci-privilege-gate.spec.ts
+++ b/src/common/ci-privilege-gate.spec.ts
@@ -791,44 +791,55 @@ describe.each([
// Any secret but the run's own token: `secrets['X']` and `toJSON(secrets)` hand the job the mirror token as well.
const ANY_SECRET = /\bsecrets(?:\.(?!GITHUB_TOKEN\b)|\s*\[|\s*\))/;
const holdsSecret = (job: PhpJob): boolean => ANY_SECRET.test(JSON.stringify([workflowEnv(), job]));
- // The command each segment of a line starts with, after env assignments and the usual wrappers, so that
- // `cd sdk/php && composer install` and `(cd sdk/php; phpunit)` are read as the commands they run, and
- // a word inside an echoed message is not.
- const WRAPPERS = new Set([
- ...['sudo', 'exec', 'command', 'time', 'env', 'nohup', 'xargs', 'nice', 'timeout', 'stdbuf', 'setsid', 'eval'],
- ...['!', '{', 'if', 'then', 'do', 'else', 'elif', 'while', 'until'],
- ]);
- const SHELLS = new Set(['sh', 'bash', 'dash', 'zsh']);
- const unquote = (word: string): string => word.replace(/^[\\"']+|["']+$/g, '');
- const startsWith = (run: string): string[] =>
- run
- .split('\n')
- .flatMap(line => line.split(/&&|\|\||[;|&()`]|\$\(/))
- .flatMap(segment => {
+ // What the job that holds the token may run: git, echo, `exit` and the shell's own words for an `if` or a
+ // `case`. A wrapper (`timeout`, `sudo`), a shell (`sh -c`), an interpreter or a path puts its own word in
+ // the list, so it is refused whatever it launches: this is a list of what is allowed, not of what is
+ // known to be bad.
+ const ALLOWED = new Set(['set', '[', '[[', 'echo', 'exit', 'fi', 'esac', 'git']);
+ const FLOW = new Set(['if', 'then', 'else', 'elif', '!', '{', '}', 'case', 'in']);
+ const unquote = (word: string): string => word.replace(/^[\\"'$]+|["']+$/g, '');
+ // Every command of a run, at every position of a line: after `&&`, `;`, `|`, `&` or a parenthesis,
+ // behind assignments and the flow words of an `if`, with a continued line joined and a redirection such as
+ // `2>&1` not read as a `&`. A command substitution is a command of its own wherever it sits, quoted or
+ // not; what is left of a quoted string is no command, so a `;` inside an echoed message splits nothing.
+ const commands = (run: string): string[] => {
+ let text = run.replace(/\\\n/g, ' ');
+ const nested: string[] = [];
+ for (let found = true; found;) {
+ found = false;
+ text = text.replace(
+ /\$\(([^()]*)\)|`([^`]*)`/g,
+ (_whole, parens: string | undefined, ticks: string | undefined) => {
+ nested.push(parens ?? ticks ?? '');
+ found = true;
+ return ' ';
+ },
+ );
+ }
+ return [text, ...nested]
+ .flatMap(part => part.split('\n'))
+ .map(line =>
+ line
+ .replace(/"(?:[^"\\]|\\.)*"|'[^']*'/g, ' ')
+ .replace(/\d*[<>]&\d*-?/g, ' ')
+ .replace(/&>>?/g, '>')
+ .replace(/^\s*[^\s()]+\)/, ' '),
+ )
+ .flatMap(line => line.split(/&&|\|\||[;|&()]/))
+ .map(segment => {
const words = segment.trim().split(/\s+/).filter(Boolean);
- let wrapped = false;
- for (;;) {
- const word = words[0] ?? '';
- if (WRAPPERS.has(word)) wrapped = true;
- else if (!/^[A-Za-z_]\w*=/.test(word) && !(wrapped && (word.startsWith('-') || /^\d+[smhd]?$/.test(word))))
- break;
- words.shift();
- }
- const word = unquote(words[0] ?? '');
- // `sh -c "composer install"`: the string is a command line of its own.
- if (SHELLS.has(word) && words[1] === '-c') return [word, ...startsWith(unquote(words.slice(2).join(' ')))];
- return word ? [word] : [];
- });
- const isProjectCommand = (word: string): boolean =>
- /(?:^|\/)(?:composer\d*|php[\d.]*(?:-[a-z]+)?|phpunit)(?:\.phar)?$/.test(word) ||
- /vendor\/bin|scripts\//.test(word);
- // A repository script or a vendored binary handed to an interpreter (`bash ./scripts/x.sh`) is no command word.
- const SCRIPT_WORD = /vendor\/bin|(?:^|[\s'"=])(?:\.\/)?scripts\//;
+ while (words.length > 0 && (/^[A-Za-z_]\w*=/.test(words[0]) || FLOW.has(words[0]))) words.shift();
+ return unquote(words[0] ?? '');
+ })
+ .filter(Boolean);
+ };
+ const refused = (run: string): string[] => commands(run).filter(word => !ALLOWED.has(word));
it('runs the tests in a job that holds no token, and the push in a job that needs it', () => {
const entries = Object.entries(jobs());
const testers = entries.filter(([, job]) => /\bphpunit\b/.test(runsOf(job)));
- const pushers = entries.filter(([, job]) => usesToken(job));
+ // A job that receives any secret (`secrets['X']`, `toJSON(secrets)`) holds the mirror token as well.
+ const pushers = entries.filter(([, job]) => usesToken(job) || holdsSecret(job));
expect(testers.map(([name]) => name)).toEqual(['test']);
expect(pushers.map(([name]) => name)).toEqual([pusher]);
expect(usesToken(jobs().test)).toBe(false);
@@ -836,9 +847,11 @@ describe.each([
expect([jobs()[pusher].needs].flat()).toContain('test');
});
- it('runs no PHP, Composer or project script, at any command position, in the job that holds the token', () => {
- expect(startsWith(runsOf(jobs()[pusher])).filter(isProjectCommand)).toEqual([]);
- expect(runsOf(jobs()[pusher])).not.toMatch(SCRIPT_WORD);
+ it('runs nothing but git and echo, at any command position, in the job that holds the token', () => {
+ const run = runsOf(jobs()[pusher]);
+ // Non-vacuity: the job does run commands, and the reader finds them.
+ expect(commands(run)).toContain('git');
+ expect(refused(run)).toEqual([]);
});
it('uses no action but the checkout in the job that holds the token', () => {
@@ -856,7 +869,7 @@ describe.each([
for (const line of pushes) expect(line).toContain('core.hooksPath=/dev/null');
});
- it('finds Composer, PHP and the token in the shapes the pins read', () => {
+ it('refuses every shape that runs Composer, PHP or a project script, and allows what the job runs', () => {
for (const line of [
'cd sdk/php && composer install --no-interaction',
'php -v',
@@ -865,7 +878,14 @@ describe.each([
'sudo -n composer.phar update',
'./scripts/anything.sh',
'timeout 60 composer install',
+ 'timeout -s KILL 60 composer install',
+ 'sudo -u runner composer install',
'sh -c "composer install"',
+ 'bash -lc "composer install"',
+ "sh -ec 'composer install'",
+ '/bin/sh -c "composer install"',
+ 'bash --noprofile -c "composer install"',
+ 'echo composer install | sh',
'if true; then composer install; fi',
'{ php -v; }',
'sleep 1 & composer install',
@@ -873,16 +893,17 @@ describe.each([
'! composer install',
'"composer" install',
'php8.2 -v',
- ]) {
- expect(startsWith(line).some(isProjectCommand)).toBe(true);
- }
- for (const line of [
'bash ./scripts/anything.sh',
+ 'bash "$GITHUB_WORKSPACE/scripts/x.sh"',
'source ./scripts/x.sh',
'. ./scripts/x.sh',
- 'sh vendor/bin/phpunit',
+ 'node sdk/javascript/scripts/smoke.mjs',
+ 'make -C sdk/php test',
+ '$(which composer) install',
+ 'x=$(true)#; composer install',
+ 'ionice -c3 composer install',
]) {
- expect(SCRIPT_WORD.test(line)).toBe(true);
+ expect(refused(line)).not.toEqual([]);
}
for (const line of [
'git push origin main',
@@ -891,13 +912,18 @@ describe.each([
'git -c core.hooksPath=/dev/null push',
'SPLIT_SHA="$(git subtree split --prefix=sdk/php HEAD)"',
'echo "Mirrored sdk/php -> rmyndharis/openwa-php@main"',
+ 'if git ls-remote --tags --exit-code "$MIRROR" "refs/tags/${VERSION}" >/dev/null 2>&1; then',
+ 'git -c core.hooksPath=/dev/null push --force \\\n "https://[email protected]/o/r.git" \\\n "${SHA}:refs/heads/main"',
]) {
- expect(startsWith(line).some(isProjectCommand)).toBe(false);
- expect(SCRIPT_WORD.test(line)).toBe(false);
+ expect(refused(line)).toEqual([]);
}
expect(usesToken({ env: { SPLIT: `\${{ secrets.${TOKEN} }}` }, steps: [] })).toBe(true);
expect(usesToken({ steps: [{ run: 'x', env: { SPLIT: `\${{ secrets.${TOKEN} }}` } }] })).toBe(true);
expect(usesToken({ steps: [{ run: 'x', with: { token: `\${{ secrets.${TOKEN} }}` } }] })).toBe(true);
expect(usesToken({ steps: [{ run: 'x' }] })).toBe(false);
+ // A job that is handed every secret holds the mirror token, though its text never names it.
+ expect(holdsSecret({ env: { ALL: '${{ toJSON(secrets) }}' }, steps: [{ run: 'composer install' }] })).toBe(true);
+ expect(holdsSecret({ steps: [{ run: 'x', env: { T: "${{ secrets['X'] }}" } }] })).toBe(true);
+ expect(holdsSecret({ steps: [{ run: 'x', env: { T: '${{ secrets.GITHUB_TOKEN }}' } }] })).toBe(false);
});
});
diff --git a/src/common/release-gate-parity.spec.ts b/src/common/release-gate-parity.spec.ts
index 65ce275a9..b90ef8401 100644
--- a/src/common/release-gate-parity.spec.ts
+++ b/src/common/release-gate-parity.spec.ts
@@ -95,9 +95,13 @@ describe('release gate jobs can stop the publish jobs', () => {
needs?: string | string[];
if?: unknown;
'continue-on-error'?: unknown;
+ env?: Record<string, unknown>;
strategy?: { matrix?: { platform?: string[] } };
steps?: GateStep[];
};
+ const workflowEnv = (): Record<string, unknown> =>
+ (yaml.load(fs.readFileSync(path.join(workflowDir, 'release.yml'), 'utf8')) as { env?: Record<string, unknown> })
+ .env ?? {};
const jobs = (): Record<string, GateJob> =>
(yaml.load(fs.readFileSync(path.join(workflowDir, 'release.yml'), 'utf8')) as { jobs?: Record<string, GateJob> })
.jobs ?? {};
@@ -168,6 +172,23 @@ describe('release gate jobs can stop the publish jobs', () => {
expect(jobs()['image-scan'].strategy?.matrix?.platform).toEqual(['linux/amd64', 'linux/arm64']);
});
+ // trivy-action leaves a TRIVY_* variable the caller set in force whenever the matching input is empty or
+ // the action's default, so an environment value can skip a path, a scanner or a package type, retarget the
+ // architecture or the image, while every input above still reads right.
+ it('image-scan sets no scanner variable but the architecture, and scans the image the build staged', () => {
+ const scanJob = jobs()['image-scan'];
+ const scan = (scanJob.steps ?? []).find(step => (step.uses ?? '').startsWith('aquasecurity/'));
+ expect(scan?.env).toEqual({ TRIVY_PLATFORM: '${{ matrix.platform }}' });
+ expect(Object.keys({ ...workflowEnv(), ...(scanJob.env ?? {}) }).filter(key => key.startsWith('TRIVY_'))).toEqual(
+ [],
+ );
+ expect(scan?.with?.['image-ref']).toBe('${{ steps.image.outputs.ref }}');
+ const resolve = (scanJob.steps ?? []).find(step => step.name === 'Resolve staging image ref');
+ expect(executableLines(resolve?.run ?? '')).toContain(
+ 'echo "ref=ghcr.io/${REPO,,}:smoke-${{ github.run_id }}" >> "$GITHUB_OUTPUT"',
+ );
+ });
+
// Promotion re-points tags, and is safe only because every one of them is checked to resolve to the
// digest the build job produced and the gates tested. A comparison that no longer runs, or a digest
// taken from anywhere else, keeps the step green for content nobody tested.
@@ -186,6 +207,25 @@ describe('release gate jobs can stop the publish jobs', () => {
const verifyLines = executableLines(verify?.run ?? '');
expect(verifyLines).toContain('for want in linux/amd64 linux/arm64; do');
expect(verifyLines).toContain('grep -q "$want" /tmp/inspect.txt || { echo "MISSING PLATFORM: $want"; exit 1; }');
+
+ // The comparisons end the step when they fail, the steps cannot fail softly, and the digest they
+ // compare against is assigned once, from the build job.
+ const body = (text: string, start: string): string => {
+ const lines = text.split('\n').map(line => line.trim());
+ const from = lines.indexOf(start);
+ const to = lines.indexOf('fi', from);
+ return from === -1 || to === -1 ? '' : lines.slice(from, to).join('\n');
+ };
+ expect(body(lines, 'if [ "$actual" != "$EXPECTED_DIGEST" ]; then')).toContain('exit 1');
+ expect(body(lines, 'if [ "$promoted_digest" != "$EXPECTED_DIGEST" ]; then')).toContain('exit 1');
+ expect(lines.match(/\bEXPECTED_DIGEST=/g) ?? []).toEqual([]);
+ expect(lines.match(/^\s*actual=/gm) ?? []).toHaveLength(1);
+ for (const step of [tag, verify]) {
+ expect(step?.['continue-on-error']).toBeUndefined();
+ expect(step?.if).toBeUndefined();
+ }
+ expect(verifyLines).toContain('ref="ghcr.io/${REPO,,}:${GITHUB_REF#refs/tags/v}"');
+ expect(verifyLines).not.toMatch(/\|\|\s*true|;\s*true\b/);
});
it.each(['boot-smoke', 'image-scan', 'verify-published'])(
diff --git a/src/common/workflow-lines.ts b/src/common/workflow-lines.ts
index ea8f32c66..a03983bbc 100644
--- a/src/common/workflow-lines.ts
+++ b/src/common/workflow-lines.ts
@@ -18,10 +18,11 @@ export function executableLines(run: string): string {
}
/**
- * Where a shell comment starts in `line`, or -1. A `#` begins one only at the start of the line or after
- * whitespace, and outside quotes: in `npm --version#; curl x | sh` the shell runs the curl, and `$#` and
- * `${#list[@]}` are not comments either. Cutting the line at such a `#` would hide the command behind it
- * from every check that reads these lines.
+ * Where a shell comment starts in `line`, or -1. A `#` begins one at the start of the line, after
+ * whitespace, or after `;`, `&`, `|` or `(`, which end a word, and outside quotes: in
+ * `npm --version#; curl x | sh` the shell runs the curl, `x=$(true)#; curl y` continues the word its `)`
+ * closed, and `$#` and `${#list[@]}` are not comments either. Cutting the line at such a `#` would hide
+ * the command behind it from every check that reads these lines.
*/
function commentStart(line: string): number {
let quote: '"' | "'" | null = null;
@@ -34,7 +35,7 @@ function commentStart(line: string): number {
i++;
} else if (char === '"' || char === "'") {
quote = char;
- } else if (char === '#' && (i === 0 || /[\s;&|()]/.test(line[i - 1]))) {
+ } else if (char === '#' && (i === 0 || /[\s;&|(]/.test(line[i - 1]))) {
return i;
}
}