Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

test(ci): read exactly where a gate counts or orders steps

Yudhi Armyndharisacf876e7ece9
diff --git a/src/common/ci-privilege-gate.spec.ts b/src/common/ci-privilege-gate.spec.ts
index 5221b6231..41afd5d38 100644
--- a/src/common/ci-privilege-gate.spec.ts
+++ b/src/common/ci-privilege-gate.spec.ts
@@ -1778,12 +1778,32 @@ describe('a job that can mint a publish credential pins global npm installs and
// npm reads `dir/file.tgz` as the GitHub shorthand user/repo (hosted-git-info is tried before the file
// check), and the pinned npm refuses git specs by default, so a tarball is named by an absolute path.
- it('js-sdk-release.yml: the publish step names the tarball in a form npm reads as a file', () => {
- const runs = oidcJobRuns('js-sdk-release.yml', executableLinesConservative)
+ const publishTarget = (source: string | OidcWorkflow): string => {
+ // The first publish the shell may run is the one held to the form, so the lines are read exactly.
+ const runs = oidcJobRuns(source)
.map(({ run }) => run)
.join('\n');
- const target = /\bnpm publish\s+(?:-\S+\s+)*(\S+)/.exec(runs)?.[1] ?? '';
- expect(target).toMatch(/^(?:"?\$\{?GITHUB_WORKSPACE\}?"?|)\/[^\s]*\.tgz$/);
+ return /\bnpm publish\s+(?:-\S+\s+)*(\S+)/.exec(runs)?.[1] ?? '';
+ };
+ const FILE_FORM = /^(?:"?\$\{?GITHUB_WORKSPACE\}?"?|)\/[^\s]*\.tgz$/;
+
+ it('js-sdk-release.yml: the publish step names the tarball in a form npm reads as a file', () => {
+ expect(publishTarget('js-sdk-release.yml')).toMatch(FILE_FORM);
+ });
+
+ it('js-sdk-release.yml: refuses a first publish that names a path npm reads as a repository', () => {
+ const copy = (run: string): OidcWorkflow => {
+ const workflow = JSON.parse(JSON.stringify(loadOidcWorkflow('js-sdk-release.yml'))) as OidcWorkflow;
+ stepsOf(workflow, 'publish').unshift({ run });
+ return workflow;
+ };
+ // A `#` after a blank the shell does not skip is no comment, so the publish after the `;` runs first.
+ for (const blank of [...BLANK_LOOKALIKES, String.fromCharCode(0x2028)]) {
+ expect(publishTarget(copy(`echo x${blank}#; npm publish package/x.tgz`))).not.toMatch(FILE_FORM);
+ }
+ expect(publishTarget(copy('npm publish package/x.tgz'))).not.toMatch(FILE_FORM);
+ // Behind a space it is a comment, which is not a publish.
+ expect(publishTarget(copy('echo x # npm publish package/x.tgz'))).toMatch(FILE_FORM);
});
// The caches a setup action restores are written by jobs that run third-party test code. A job that
@@ -1897,9 +1917,8 @@ describe('a publish job takes the artifact its build job uploads', () => {
const indexOfUses = (job: FlowJob, action: string): number =>
(job.steps ?? []).findIndex(step => (step.uses ?? '').startsWith(`${action}@`));
const dirOf = (target: string | undefined): string => (target ?? '').replace(/\/+$/, '');
- // Only a presence reading is made of it: the publish line is read for the tarball it names.
- const runsOf = (job: FlowJob): string =>
- (job.steps ?? []).map(step => executableLinesConservative(step.run ?? '')).join('\n');
+ // Read exactly: the first publish line the shell may run is the one that has to name the artifact.
+ const runsOf = (job: FlowJob): string => (job.steps ?? []).map(step => executableLines(step.run ?? '')).join('\n');
it('js-sdk-release.yml: the tarball npm pack writes is the one the publish step names', () => {
const { build, publish } = jobsOf('js-sdk-release.yml');
@@ -1920,9 +1939,7 @@ describe('a publish job takes the artifact its build job uploads', () => {
expect(target).toBe(
`"$GITHUB_WORKSPACE"/${dirOf(download.with?.path)}/${path.posix.basename(upload.with?.path ?? '')}`,
);
- const publishAt = (publish.steps ?? []).findIndex(step =>
- /^npm publish\b/m.test(executableLinesConservative(step.run ?? '')),
- );
+ const publishAt = (publish.steps ?? []).findIndex(step => /^npm publish\b/m.test(executableLines(step.run ?? '')));
expect(indexOfUses(publish, 'actions/download-artifact')).toBeLessThan(publishAt);
});
@@ -2060,21 +2077,74 @@ describe('ci.yml moves the branch image tag only for the tested branch head', ()
expect(executableLines(tagRules.join('\n'))).not.toContain('type=ref,event=branch');
});
- it('re-points the branch tag as the last step, after the smoke, on push, only at the branch head', () => {
- const steps = dockerSteps();
- const smoke = steps.findIndex(step =>
- executableLinesConservative(step.run ?? '').includes('smoke-test-non-root.sh'),
- );
- const retag = steps.findIndex(step => executableLinesConservative(step.run ?? '').includes('imagetools create'));
- expect(smoke).toBeGreaterThan(-1);
- expect(retag).toBeGreaterThan(smoke);
+ // What is wrong with the order of a docker job's steps. Two readings, because the two halves are different
+ // questions: the smoke has to be a step that CERTAINLY runs the script (conservative), and every step the
+ // shell MAY run `imagetools create` in has to be the last one (exact), or a re-point written as
+ // `echo x<U+00A0>#; docker buildx imagetools create ...` runs ahead of the smoke while this reads green.
+ const branchTagProblems = (steps: CiStep[]): string[] => {
+ const last = steps.length - 1;
+ const runs = (read: (run: string) => string): string[] => steps.map(step => read(step.run ?? ''));
+ const smoke = runs(executableLinesConservative).findIndex(run => run.includes('smoke-test-non-root.sh'));
+ const mayRepoint = runs(executableLines).flatMap((run, i) => (run.includes('imagetools create') ? [i] : []));
+ const found: string[] = [];
+ if (smoke === -1) found.push('no step runs the non-root smoke');
// Last, so every check added to this job gates the branch tag, not only the non-root smoke.
- expect(retag).toBe(steps.length - 1);
- const step = steps[retag];
- expect(step.if ?? '').toContain("github.event_name == 'push'");
- const run = executableLinesConservative(step.run ?? '');
- expect(run).toMatch(/git ls-remote/);
- expect(run).toMatch(/GITHUB_SHA/);
+ for (const i of mayRepoint) if (i !== last) found.push(`step ${i + 1} may re-point the branch tag, before the end`);
+ const retag = steps[last];
+ if (!executableLinesConservative(retag?.run ?? '').includes('imagetools create')) {
+ found.push('the last step does not re-point the branch tag');
+ } else if (smoke > -1 && smoke >= last) {
+ found.push('the smoke does not run before the re-point');
+ }
+ if (!(retag?.if ?? '').includes("github.event_name == 'push'")) found.push('the re-point is not push only');
+ const run = executableLinesConservative(retag?.run ?? '');
+ if (!/git ls-remote/.test(run)) found.push('the re-point does not look up the branch head');
+ if (!/GITHUB_SHA/.test(run)) found.push('the re-point does not compare the commit');
+ return found;
+ };
+
+ it('re-points the branch tag as the last step, after the smoke, on push, only at the branch head', () => {
+ expect(branchTagProblems(dockerSteps())).toEqual([]);
+ });
+
+ // The same checks on copies of the steps as they ship, each edited the way a change would be.
+ it('refuses a re-point that runs before the smoke or is only named behind a #', () => {
+ const copy = (): CiStep[] => JSON.parse(JSON.stringify(dockerSteps())) as CiStep[];
+ const smokeAt = dockerSteps().findIndex(step => (step.run ?? '').includes('smoke-test-non-root.sh'));
+ const REPOINT = 'docker buildx imagetools create --tag ghcr.io/x/openwa:main ghcr.io/x/openwa@sha256:0';
+ const early = (run: string): CiStep[] => {
+ const steps = copy();
+ steps.splice(smokeAt, 0, { run });
+ return steps;
+ };
+ // Each blank the shell reads as part of a word, and the line separator of YAML 1.1: the `#` after one is no
+ // comment, so the re-point after the `;` is a command that runs before the smoke.
+ for (const blank of [...BLANK_LOOKALIKES, String.fromCharCode(0x2028)]) {
+ const code = `U+${blank.charCodeAt(0).toString(16)}`;
+ expect([code, branchTagProblems(early(`echo x${blank}#; ${REPOINT}`))]).toEqual([
+ code,
+ [`step ${smokeAt + 1} may re-point the branch tag, before the end`],
+ ]);
+ }
+ // The same line behind a space is a comment: nothing runs, so the job is as it ships.
+ expect(branchTagProblems(early(`echo x # ${REPOINT}`))).toEqual([]);
+ // Only the exact reading finds a re-point that is named and not run, so the last step must run it.
+ const mention = copy();
+ mention[mention.length - 1].run = `echo x${BLANK_LOOKALIKES[0]}# ${REPOINT}\ngit ls-remote\n$GITHUB_SHA`;
+ expect(branchTagProblems(mention)).toEqual(['the last step does not re-point the branch tag']);
+ // A step after the re-point, and a re-point that no longer follows the smoke.
+ const appended = copy();
+ appended.push({ run: 'echo done' });
+ expect(branchTagProblems(appended)).toEqual([
+ `step ${appended.length - 1} may re-point the branch tag, before the end`,
+ 'the last step does not re-point the branch tag',
+ 'the re-point is not push only',
+ 'the re-point does not look up the branch head',
+ 'the re-point does not compare the commit',
+ ]);
+ const swapped = copy();
+ swapped[smokeAt].run = 'echo skipped';
+ expect(branchTagProblems(swapped)).toEqual(['no step runs the non-root smoke']);
});
});
@@ -2128,16 +2198,53 @@ describe('the dashboard unit tests are bounded', () => {
expect(scripts.test).toBe('npm run test:unit');
});
+ type TimedStep = Step & { 'timeout-minutes'?: unknown };
+ type TimedWorkflow = { jobs?: Record<string, { steps?: TimedStep[] }> };
+ // One step runs the tests, and its bound is a number. Both readings, since each half is its own question:
+ // a step that certainly runs them has to be there once, and every step the shell may run them in has to
+ // carry the bound, or a second one written as `echo x<U+00A0>#; npm run test:unit` runs with none.
+ const unitTestProblems = (workflow: TimedWorkflow): string[] => {
+ const steps = Object.values(workflow.jobs ?? {}).flatMap(job => job.steps ?? []);
+ const running = (read: (run: string) => string): TimedStep[] =>
+ steps.filter(step => read(step.run ?? '').includes('npm run test:unit'));
+ const found: string[] = [];
+ const certain = running(executableLinesConservative).length;
+ if (certain !== 1) found.push(`${certain} steps certainly run the unit tests, expected 1`);
+ const may = running(executableLines);
+ if (may.length > 1) found.push(`${may.length} steps may run the unit tests, expected 1`);
+ for (const step of may) {
+ if (typeof step['timeout-minutes'] !== 'number') found.push(`a step that may run the unit tests has no bound`);
+ }
+ return found;
+ };
+
it.each(['ci.yml', 'release.yml'])('%s bounds the dashboard unit test step', file => {
- type TimedStep = Step & { 'timeout-minutes'?: unknown };
- const workflow = yaml.load(fs.readFileSync(path.join(workflowDir, file), 'utf8')) as {
- jobs?: Record<string, { steps?: TimedStep[] }>;
- };
- const steps = Object.values(workflow.jobs ?? {})
- .flatMap(job => job.steps ?? [])
- .filter(step => executableLinesConservative(step.run ?? '').includes('npm run test:unit'));
- expect(steps.length).toBe(1);
- expect(steps[0]['timeout-minutes']).toEqual(expect.any(Number));
+ expect(unitTestProblems(workflowOf(file))).toEqual([]);
+ });
+
+ it.each(['ci.yml', 'release.yml'])('%s: refuses a second unit test run or one that is not bounded', file => {
+ const copy = (): TimedWorkflow => JSON.parse(JSON.stringify(workflowOf(file))) as TimedWorkflow;
+ const dashboard = (workflow: TimedWorkflow): TimedStep[] => workflow.jobs?.dashboard?.steps ?? [];
+ for (const blank of [...BLANK_LOOKALIKES, String.fromCharCode(0x2028)]) {
+ const code = `U+${blank.charCodeAt(0).toString(16)}`;
+ const second = copy();
+ dashboard(second).push({ run: `echo x${blank}#; npm run test:unit` });
+ expect([code, unitTestProblems(second)]).toEqual([
+ code,
+ ['2 steps may run the unit tests, expected 1', 'a step that may run the unit tests has no bound'],
+ ]);
+ }
+ // The step that runs them, with its bound taken off, and with the command only named behind a `#`.
+ const unbounded = copy();
+ for (const step of dashboard(unbounded)) {
+ if ((step.run ?? '').includes('npm run test:unit')) delete step['timeout-minutes'];
+ }
+ expect(unitTestProblems(unbounded)).toEqual(['a step that may run the unit tests has no bound']);
+ const named = copy();
+ for (const step of dashboard(named)) {
+ if ((step.run ?? '').includes('npm run test:unit')) step.run = `echo x${BLANK_LOOKALIKES[0]}# npm run test:unit`;
+ }
+ expect(unitTestProblems(named)).toEqual(['0 steps certainly run the unit tests, expected 1']);
});
});
diff --git a/src/common/release-gate-parity.spec.ts b/src/common/release-gate-parity.spec.ts
index 6f9b075cf..f42926d27 100644
--- a/src/common/release-gate-parity.spec.ts
+++ b/src/common/release-gate-parity.spec.ts
@@ -191,6 +191,8 @@ describe('release gate jobs can stop the publish jobs', () => {
expect(executableLinesConservative(resolve?.run ?? '')).toContain(
'echo "ref=ghcr.io/${REPO,,}:smoke-${{ github.run_id }}" >> "$GITHUB_OUTPUT"',
);
+ // Written once: a later write to the same output wins, so none may be hidden behind a '#'.
+ expect(executableLines(resolve?.run ?? '').split('GITHUB_OUTPUT')).toHaveLength(2);
});
// Promotion re-points tags, and is safe only because every one of them is checked to resolve to the
@@ -310,19 +312,50 @@ describe('one prerelease rule across the GitHub Release and the image channels',
*/
describe('the release and weekly audits fail when the audit cannot run', () => {
type AuditStep = { run?: string; env?: Record<string, unknown> };
- const auditSteps = (file: string): AuditStep[] => {
- const workflow = yaml.load(fs.readFileSync(path.join(workflowDir, file), 'utf8')) as {
- jobs?: Record<string, { steps?: AuditStep[] }>;
- };
- return Object.values(workflow.jobs ?? {})
+ type AuditWorkflow = { jobs?: Record<string, { steps?: AuditStep[] }> };
+ const auditWorkflow = (file: string): AuditWorkflow =>
+ yaml.load(fs.readFileSync(path.join(workflowDir, file), 'utf8')) as AuditWorkflow;
+ // The steps that run the audit as a reading sees it. Both are used, for different questions: a step that
+ // CERTAINLY runs it (conservative) has to be there, and every step that MAY run it (exact) is held to the
+ // rule, or an extra one written as `echo x<U+00A0>#; npm run check:audit` takes the skip unseen.
+ const auditSteps = (workflow: AuditWorkflow, read: (run: string) => string): AuditStep[] =>
+ Object.values(workflow.jobs ?? {})
.flatMap(job => job.steps ?? [])
- .filter(step => executableLinesConservative(step.run ?? '').includes('npm run check:audit'));
- };
+ .filter(step => read(step.run ?? '').includes('npm run check:audit'));
+ const unrequired = (workflow: AuditWorkflow): string[] => [
+ ...(auditSteps(workflow, executableLinesConservative).length === 0 ? ['no step certainly runs the audit'] : []),
+ ...auditSteps(workflow, executableLines)
+ .filter(step => step.env?.CHECK_AUDIT_REQUIRED !== '1')
+ .map(() => 'a step that may run the audit does not require it'),
+ ];
it.each(['release.yml', 'security-scan.yml'])('%s requires the audit', file => {
- const steps = auditSteps(file);
- expect(steps.length).toBeGreaterThan(0);
- expect(steps.filter(step => step.env?.CHECK_AUDIT_REQUIRED !== '1')).toEqual([]);
+ expect(unrequired(auditWorkflow(file))).toEqual([]);
+ });
+
+ it.each(['release.yml', 'security-scan.yml'])('%s: refuses an audit step that does not require it', file => {
+ const copy = (): AuditWorkflow => JSON.parse(JSON.stringify(auditWorkflow(file))) as AuditWorkflow;
+ const anyJob = (workflow: AuditWorkflow): AuditStep[] => Object.values(workflow.jobs ?? {})[0].steps ?? [];
+ // Each blank the shell reads as part of a word: the `#` after it is no comment, so the audit after the `;`
+ // runs, with the skip.
+ for (const code of [0xa0, 0x3000, 0xfeff, 0x2028]) {
+ const blank = String.fromCharCode(code);
+ const extra = copy();
+ anyJob(extra).push({ run: `echo x${blank}#; npm run check:audit` });
+ expect([code, unrequired(extra)]).toEqual([code, ['a step that may run the audit does not require it']]);
+ }
+ // The same line behind a space is a comment, and the real step with its requirement taken off is not.
+ const comment = copy();
+ anyJob(comment).push({ run: 'echo x # npm run check:audit' });
+ expect(unrequired(comment)).toEqual([]);
+ const lax = copy();
+ for (const step of auditSteps(lax, executableLines)) step.env = { CHECK_AUDIT_REQUIRED: '0' };
+ expect(unrequired(lax)).toEqual(['a step that may run the audit does not require it']);
+ // A step that only names the audit behind such a blank and a `#` is no audit: one must really run.
+ const named = copy();
+ for (const step of auditSteps(named, executableLines))
+ step.run = `echo x${String.fromCharCode(0xa0)}# npm run check:audit`;
+ expect(unrequired(named)).toContain('no step certainly runs the audit');
});
it('docs/10 describes the weekly audit as required, not as the merge job', () => {
@@ -333,9 +366,10 @@ describe('the release and weekly audits fail when the audit cannot run', () => {
});
it('ci.yml keeps the skip for merges', () => {
- const steps = auditSteps('ci.yml');
- expect(steps.length).toBeGreaterThan(0);
- expect(steps.some(step => step.env?.CHECK_AUDIT_REQUIRED !== undefined)).toBe(false);
+ const ci = auditWorkflow('ci.yml');
+ expect(auditSteps(ci, executableLinesConservative).length).toBeGreaterThan(0);
+ // Every step that may run it, not only one that certainly does.
+ expect(auditSteps(ci, executableLines).some(step => step.env?.CHECK_AUDIT_REQUIRED !== undefined)).toBe(false);
});
});
@@ -472,22 +506,63 @@ describe('released images carry a verifiable build-provenance attestation', () =
const grant = (perms: Perms, key: string): string | undefined =>
typeof perms === 'object' && perms !== null ? perms[key] : undefined;
- // Attesting first means a failed attestation leaves no release tag published.
+ // Attesting first means a failed attestation leaves no release tag published. Every step that MAY re-point
+ // a tag (exact) follows the attestation, and one CERTAINLY does (conservative); a re-point written as
+ // `echo x<U+00A0>#; docker buildx imagetools create ...` is a command that runs, so it is held to the order.
+ const attestProblems = (steps: AttestStep[]): string[] => {
+ const attest = steps.findIndex(step =>
+ /^actions\/attest(?:-build-provenance)?@[0-9a-f]{40}\b/.test(step.uses ?? ''),
+ );
+ const certain = steps.some(step => executableLinesConservative(step.run ?? '').includes('imagetools create'));
+ const may = steps.flatMap((step, i) => (executableLines(step.run ?? '').includes('imagetools create') ? [i] : []));
+ return [
+ ...(attest === -1 ? ['no attest step'] : []),
+ ...(certain ? [] : ['no step certainly re-points the tags']),
+ ...may.filter(i => i <= attest).map(i => `step ${i + 1} may re-point a tag before the attestation`),
+ ];
+ };
+
it('promote attests the tested digest before re-pointing the release tags', () => {
const promote = jobs().promote;
const steps = promote?.steps ?? [];
expect(steps.length).toBeGreaterThan(1);
+ expect(attestProblems(steps)).toEqual([]);
const attest = steps.findIndex(step =>
/^actions\/attest(?:-build-provenance)?@[0-9a-f]{40}\b/.test(step.uses ?? ''),
);
- const retag = steps.findIndex(step => executableLinesConservative(step.run ?? '').includes('imagetools create'));
- expect(attest).toBeGreaterThan(-1);
- expect(retag).toBeGreaterThan(attest);
expect(steps[attest].with?.['subject-digest']).toContain('needs.docker.outputs.digest');
expect(grant(promote?.permissions, 'id-token')).toBe('write');
expect(grant(promote?.permissions, 'attestations')).toBe('write');
});
+ it('refuses a re-point that may run before the attestation, or that is only named', () => {
+ const copy = (): AttestStep[] => JSON.parse(JSON.stringify(jobs().promote?.steps ?? [])) as AttestStep[];
+ const attestAt = copy().findIndex(step => (step.uses ?? '').startsWith('actions/attest'));
+ const REPOINT = 'docker buildx imagetools create --tag t s';
+ for (const code of [0xa0, 0x3000, 0xfeff, 0x2028]) {
+ const steps = copy();
+ steps.splice(attestAt, 0, { run: `echo x${String.fromCharCode(code)}#; ${REPOINT}` });
+ expect([code, attestProblems(steps)]).toEqual([
+ code,
+ [`step ${attestAt + 1} may re-point a tag before the attestation`],
+ ]);
+ }
+ // The attestation moved behind the re-point.
+ const moved = copy();
+ moved.push(...moved.splice(attestAt, 1));
+ expect(attestProblems(moved)).toEqual([
+ expect.stringMatching(/^step \d+ may re-point a tag before the attestation$/),
+ ]);
+ // A re-point that is only named behind a blank and a `#` is no re-point at all.
+ const named = copy();
+ for (const step of named) {
+ if (executableLines(step.run ?? '').includes('imagetools create')) {
+ step.run = `echo x${String.fromCharCode(0xa0)}# ${REPOINT}`;
+ }
+ }
+ expect(attestProblems(named)).toEqual(['no step certainly re-points the tags']);
+ });
+
// The signing grant stays on the one job that needs it, away from the build and test jobs.
it.each(['docker', 'boot-smoke', 'image-scan'])('%s cannot mint an OIDC token', job => {
expect(jobs()[job]).toBeDefined();
diff --git a/src/common/workflow-lines.ts b/src/common/workflow-lines.ts
index f37c4282e..eb14f4163 100644
--- a/src/common/workflow-lines.ts
+++ b/src/common/workflow-lines.ts
@@ -20,6 +20,30 @@
* found. Use it for a check that something IS run, by substring or by a pattern that is not anchored
* at the end. It is never a stand-in for the exact reading elsewhere: it would hide the `curl` in
* `echo x<U+00A0>#; curl y | sh`, and turn `npm test<U+00A0># || true` into a clean `npm test`.
+ *
+ * The question a site asks picks the reading. "Is it run" reads conservatively; a count, a first occurrence,
+ * an order, a rule every occurrence must obey and a line compared whole read exactly, and a check that asks
+ * both says so by running both. A site that is added or changed goes in this table.
+ *
+ * ci-privilege-gate.spec.ts
+ * invoked scripts: non-root smoke, backup smoke, a Python installer conservative is it run
+ * id-token jobs hold `npm publish` and `imagetools create` conservative is it run
+ * id-token allow-lists, templates, installers, `${{` in a comment exact every line it may run
+ * publish tarball, `npm pack` and `npm publish` order, `python -m build` exact first occurrence
+ * java deploy line both is there, every one
+ * branch tag: the smoke runs, the last step runs the re-point conservative is it run
+ * branch tag: which steps may re-point exact every one that may
+ * branch tag: the metadata-action tag rules exact must not be there
+ * dashboard unit tests: one step runs them, all that may are bounded both is there, every one
+ * PHP mirror: who runs phpunit, the push line both is there, every one
+ * PHP mirror: what the job holding the token may run exact every one that may
+ * release-gate-parity.spec.ts
+ * gate commands and whole steps compared with ci.yml exact line equality
+ * postgres lane names its spec, verify-published, engine libraries conservative is it run
+ * image-scan ref output, promote decision lines both is there, is the only one
+ * check:audit requires the audit both is there, every one
+ * audit trees exact order, uniqueness
+ * attest before every re-point both is there, every one
*/
export function executableLines(run: string): string {
return withoutComments(run, /[ \t;&|(]/);