Free, Open Source, Self-Hosted WhatsApp API Gateway
TypeScript apibotgatewayself-hostedwhatsappwhatsapp-apiwhatsapp-automationwhatsapp-botwhatsapp-gatewaywhatsapp-gateway-apiwhatsapp-webwhatsapp-web-api
test(ci): read exactly where a gate counts or orders steps
Yudhi Armyndharisacf876e7ece9
diff --git a/src/common/ci-privilege-gate.spec.ts b/src/common/ci-privilege-gate.spec.tsindex 5221b6231..41afd5d38 100644--- a/src/common/ci-privilege-gate.spec.ts+++ b/src/common/ci-privilege-gate.spec.ts@@ -1778,12 +1778,32 @@ describe('a job that can mint a publish credential pins global npm installs and// npm reads `dir/file.tgz` as the GitHub shorthand user/repo (hosted-git-info is tried before the file// check), and the pinned npm refuses git specs by default, so a tarball is named by an absolute path.- it('js-sdk-release.yml: the publish step names the tarball in a form npm reads as a file', () => {- const runs = oidcJobRuns('js-sdk-release.yml', executableLinesConservative)+ const publishTarget = (source: string | OidcWorkflow): string => {+ // The first publish the shell may run is the one held to the form, so the lines are read exactly.+ const runs = oidcJobRuns(source).map(({ run }) => run).join('\n');- const target = /\bnpm publish\s+(?:-\S+\s+)*(\S+)/.exec(runs)?.[1] ?? '';- expect(target).toMatch(/^(?:"?\$\{?GITHUB_WORKSPACE\}?"?|)\/[^\s]*\.tgz$/);+ return /\bnpm publish\s+(?:-\S+\s+)*(\S+)/.exec(runs)?.[1] ?? '';+ };+ const FILE_FORM = /^(?:"?\$\{?GITHUB_WORKSPACE\}?"?|)\/[^\s]*\.tgz$/;++ it('js-sdk-release.yml: the publish step names the tarball in a form npm reads as a file', () => {+ expect(publishTarget('js-sdk-release.yml')).toMatch(FILE_FORM);+ });++ it('js-sdk-release.yml: refuses a first publish that names a path npm reads as a repository', () => {+ const copy = (run: string): OidcWorkflow => {+ const workflow = JSON.parse(JSON.stringify(loadOidcWorkflow('js-sdk-release.yml'))) as OidcWorkflow;+ stepsOf(workflow, 'publish').unshift({ run });+ return workflow;+ };+ // A `#` after a blank the shell does not skip is no comment, so the publish after the `;` runs first.+ for (const blank of [...BLANK_LOOKALIKES, String.fromCharCode(0x2028)]) {+ expect(publishTarget(copy(`echo x${blank}#; npm publish package/x.tgz`))).not.toMatch(FILE_FORM);+ }+ expect(publishTarget(copy('npm publish package/x.tgz'))).not.toMatch(FILE_FORM);+ // Behind a space it is a comment, which is not a publish.+ expect(publishTarget(copy('echo x # npm publish package/x.tgz'))).toMatch(FILE_FORM);});// The caches a setup action restores are written by jobs that run third-party test code. A job that@@ -1897,9 +1917,8 @@ describe('a publish job takes the artifact its build job uploads', () => {const indexOfUses = (job: FlowJob, action: string): number =>(job.steps ?? []).findIndex(step => (step.uses ?? '').startsWith(`${action}@`));const dirOf = (target: string | undefined): string => (target ?? '').replace(/\/+$/, '');- // Only a presence reading is made of it: the publish line is read for the tarball it names.- const runsOf = (job: FlowJob): string =>- (job.steps ?? []).map(step => executableLinesConservative(step.run ?? '')).join('\n');+ // Read exactly: the first publish line the shell may run is the one that has to name the artifact.+ const runsOf = (job: FlowJob): string => (job.steps ?? []).map(step => executableLines(step.run ?? '')).join('\n');it('js-sdk-release.yml: the tarball npm pack writes is the one the publish step names', () => {const { build, publish } = jobsOf('js-sdk-release.yml');@@ -1920,9 +1939,7 @@ describe('a publish job takes the artifact its build job uploads', () => {expect(target).toBe(`"$GITHUB_WORKSPACE"/${dirOf(download.with?.path)}/${path.posix.basename(upload.with?.path ?? '')}`,);- const publishAt = (publish.steps ?? []).findIndex(step =>- /^npm publish\b/m.test(executableLinesConservative(step.run ?? '')),- );+ const publishAt = (publish.steps ?? []).findIndex(step => /^npm publish\b/m.test(executableLines(step.run ?? '')));expect(indexOfUses(publish, 'actions/download-artifact')).toBeLessThan(publishAt);});@@ -2060,21 +2077,74 @@ describe('ci.yml moves the branch image tag only for the tested branch head', ()expect(executableLines(tagRules.join('\n'))).not.toContain('type=ref,event=branch');});- it('re-points the branch tag as the last step, after the smoke, on push, only at the branch head', () => {- const steps = dockerSteps();- const smoke = steps.findIndex(step =>- executableLinesConservative(step.run ?? '').includes('smoke-test-non-root.sh'),- );- const retag = steps.findIndex(step => executableLinesConservative(step.run ?? '').includes('imagetools create'));- expect(smoke).toBeGreaterThan(-1);- expect(retag).toBeGreaterThan(smoke);+ // What is wrong with the order of a docker job's steps. Two readings, because the two halves are different+ // questions: the smoke has to be a step that CERTAINLY runs the script (conservative), and every step the+ // shell MAY run `imagetools create` in has to be the last one (exact), or a re-point written as+ // `echo x<U+00A0>#; docker buildx imagetools create ...` runs ahead of the smoke while this reads green.+ const branchTagProblems = (steps: CiStep[]): string[] => {+ const last = steps.length - 1;+ const runs = (read: (run: string) => string): string[] => steps.map(step => read(step.run ?? ''));+ const smoke = runs(executableLinesConservative).findIndex(run => run.includes('smoke-test-non-root.sh'));+ const mayRepoint = runs(executableLines).flatMap((run, i) => (run.includes('imagetools create') ? [i] : []));+ const found: string[] = [];+ if (smoke === -1) found.push('no step runs the non-root smoke');// Last, so every check added to this job gates the branch tag, not only the non-root smoke.- expect(retag).toBe(steps.length - 1);- const step = steps[retag];- expect(step.if ?? '').toContain("github.event_name == 'push'");- const run = executableLinesConservative(step.run ?? '');- expect(run).toMatch(/git ls-remote/);- expect(run).toMatch(/GITHUB_SHA/);+ for (const i of mayRepoint) if (i !== last) found.push(`step ${i + 1} may re-point the branch tag, before the end`);+ const retag = steps[last];+ if (!executableLinesConservative(retag?.run ?? '').includes('imagetools create')) {+ found.push('the last step does not re-point the branch tag');+ } else if (smoke > -1 && smoke >= last) {+ found.push('the smoke does not run before the re-point');+ }+ if (!(retag?.if ?? '').includes("github.event_name == 'push'")) found.push('the re-point is not push only');+ const run = executableLinesConservative(retag?.run ?? '');+ if (!/git ls-remote/.test(run)) found.push('the re-point does not look up the branch head');+ if (!/GITHUB_SHA/.test(run)) found.push('the re-point does not compare the commit');+ return found;+ };++ it('re-points the branch tag as the last step, after the smoke, on push, only at the branch head', () => {+ expect(branchTagProblems(dockerSteps())).toEqual([]);+ });++ // The same checks on copies of the steps as they ship, each edited the way a change would be.+ it('refuses a re-point that runs before the smoke or is only named behind a #', () => {+ const copy = (): CiStep[] => JSON.parse(JSON.stringify(dockerSteps())) as CiStep[];+ const smokeAt = dockerSteps().findIndex(step => (step.run ?? '').includes('smoke-test-non-root.sh'));+ const REPOINT = 'docker buildx imagetools create --tag ghcr.io/x/openwa:main ghcr.io/x/openwa@sha256:0';+ const early = (run: string): CiStep[] => {+ const steps = copy();+ steps.splice(smokeAt, 0, { run });+ return steps;+ };+ // Each blank the shell reads as part of a word, and the line separator of YAML 1.1: the `#` after one is no+ // comment, so the re-point after the `;` is a command that runs before the smoke.+ for (const blank of [...BLANK_LOOKALIKES, String.fromCharCode(0x2028)]) {+ const code = `U+${blank.charCodeAt(0).toString(16)}`;+ expect([code, branchTagProblems(early(`echo x${blank}#; ${REPOINT}`))]).toEqual([+ code,+ [`step ${smokeAt + 1} may re-point the branch tag, before the end`],+ ]);+ }+ // The same line behind a space is a comment: nothing runs, so the job is as it ships.+ expect(branchTagProblems(early(`echo x # ${REPOINT}`))).toEqual([]);+ // Only the exact reading finds a re-point that is named and not run, so the last step must run it.+ const mention = copy();+ mention[mention.length - 1].run = `echo x${BLANK_LOOKALIKES[0]}# ${REPOINT}\ngit ls-remote\n$GITHUB_SHA`;+ expect(branchTagProblems(mention)).toEqual(['the last step does not re-point the branch tag']);+ // A step after the re-point, and a re-point that no longer follows the smoke.+ const appended = copy();+ appended.push({ run: 'echo done' });+ expect(branchTagProblems(appended)).toEqual([+ `step ${appended.length - 1} may re-point the branch tag, before the end`,+ 'the last step does not re-point the branch tag',+ 'the re-point is not push only',+ 'the re-point does not look up the branch head',+ 'the re-point does not compare the commit',+ ]);+ const swapped = copy();+ swapped[smokeAt].run = 'echo skipped';+ expect(branchTagProblems(swapped)).toEqual(['no step runs the non-root smoke']);});});@@ -2128,16 +2198,53 @@ describe('the dashboard unit tests are bounded', () => {expect(scripts.test).toBe('npm run test:unit');});+ type TimedStep = Step & { 'timeout-minutes'?: unknown };+ type TimedWorkflow = { jobs?: Record<string, { steps?: TimedStep[] }> };+ // One step runs the tests, and its bound is a number. Both readings, since each half is its own question:+ // a step that certainly runs them has to be there once, and every step the shell may run them in has to+ // carry the bound, or a second one written as `echo x<U+00A0>#; npm run test:unit` runs with none.+ const unitTestProblems = (workflow: TimedWorkflow): string[] => {+ const steps = Object.values(workflow.jobs ?? {}).flatMap(job => job.steps ?? []);+ const running = (read: (run: string) => string): TimedStep[] =>+ steps.filter(step => read(step.run ?? '').includes('npm run test:unit'));+ const found: string[] = [];+ const certain = running(executableLinesConservative).length;+ if (certain !== 1) found.push(`${certain} steps certainly run the unit tests, expected 1`);+ const may = running(executableLines);+ if (may.length > 1) found.push(`${may.length} steps may run the unit tests, expected 1`);+ for (const step of may) {+ if (typeof step['timeout-minutes'] !== 'number') found.push(`a step that may run the unit tests has no bound`);+ }+ return found;+ };+it.each(['ci.yml', 'release.yml'])('%s bounds the dashboard unit test step', file => {- type TimedStep = Step & { 'timeout-minutes'?: unknown };- const workflow = yaml.load(fs.readFileSync(path.join(workflowDir, file), 'utf8')) as {- jobs?: Record<string, { steps?: TimedStep[] }>;- };- const steps = Object.values(workflow.jobs ?? {})- .flatMap(job => job.steps ?? [])- .filter(step => executableLinesConservative(step.run ?? '').includes('npm run test:unit'));- expect(steps.length).toBe(1);- expect(steps[0]['timeout-minutes']).toEqual(expect.any(Number));+ expect(unitTestProblems(workflowOf(file))).toEqual([]);+ });++ it.each(['ci.yml', 'release.yml'])('%s: refuses a second unit test run or one that is not bounded', file => {+ const copy = (): TimedWorkflow => JSON.parse(JSON.stringify(workflowOf(file))) as TimedWorkflow;+ const dashboard = (workflow: TimedWorkflow): TimedStep[] => workflow.jobs?.dashboard?.steps ?? [];+ for (const blank of [...BLANK_LOOKALIKES, String.fromCharCode(0x2028)]) {+ const code = `U+${blank.charCodeAt(0).toString(16)}`;+ const second = copy();+ dashboard(second).push({ run: `echo x${blank}#; npm run test:unit` });+ expect([code, unitTestProblems(second)]).toEqual([+ code,+ ['2 steps may run the unit tests, expected 1', 'a step that may run the unit tests has no bound'],+ ]);+ }+ // The step that runs them, with its bound taken off, and with the command only named behind a `#`.+ const unbounded = copy();+ for (const step of dashboard(unbounded)) {+ if ((step.run ?? '').includes('npm run test:unit')) delete step['timeout-minutes'];+ }+ expect(unitTestProblems(unbounded)).toEqual(['a step that may run the unit tests has no bound']);+ const named = copy();+ for (const step of dashboard(named)) {+ if ((step.run ?? '').includes('npm run test:unit')) step.run = `echo x${BLANK_LOOKALIKES[0]}# npm run test:unit`;+ }+ expect(unitTestProblems(named)).toEqual(['0 steps certainly run the unit tests, expected 1']);});});diff --git a/src/common/release-gate-parity.spec.ts b/src/common/release-gate-parity.spec.tsindex 6f9b075cf..f42926d27 100644--- a/src/common/release-gate-parity.spec.ts+++ b/src/common/release-gate-parity.spec.ts@@ -191,6 +191,8 @@ describe('release gate jobs can stop the publish jobs', () => {expect(executableLinesConservative(resolve?.run ?? '')).toContain('echo "ref=ghcr.io/${REPO,,}:smoke-${{ github.run_id }}" >> "$GITHUB_OUTPUT"',);+ // Written once: a later write to the same output wins, so none may be hidden behind a '#'.+ expect(executableLines(resolve?.run ?? '').split('GITHUB_OUTPUT')).toHaveLength(2);});// Promotion re-points tags, and is safe only because every one of them is checked to resolve to the@@ -310,19 +312,50 @@ describe('one prerelease rule across the GitHub Release and the image channels',*/describe('the release and weekly audits fail when the audit cannot run', () => {type AuditStep = { run?: string; env?: Record<string, unknown> };- const auditSteps = (file: string): AuditStep[] => {- const workflow = yaml.load(fs.readFileSync(path.join(workflowDir, file), 'utf8')) as {- jobs?: Record<string, { steps?: AuditStep[] }>;- };- return Object.values(workflow.jobs ?? {})+ type AuditWorkflow = { jobs?: Record<string, { steps?: AuditStep[] }> };+ const auditWorkflow = (file: string): AuditWorkflow =>+ yaml.load(fs.readFileSync(path.join(workflowDir, file), 'utf8')) as AuditWorkflow;+ // The steps that run the audit as a reading sees it. Both are used, for different questions: a step that+ // CERTAINLY runs it (conservative) has to be there, and every step that MAY run it (exact) is held to the+ // rule, or an extra one written as `echo x<U+00A0>#; npm run check:audit` takes the skip unseen.+ const auditSteps = (workflow: AuditWorkflow, read: (run: string) => string): AuditStep[] =>+ Object.values(workflow.jobs ?? {}).flatMap(job => job.steps ?? [])- .filter(step => executableLinesConservative(step.run ?? '').includes('npm run check:audit'));- };+ .filter(step => read(step.run ?? '').includes('npm run check:audit'));+ const unrequired = (workflow: AuditWorkflow): string[] => [+ ...(auditSteps(workflow, executableLinesConservative).length === 0 ? ['no step certainly runs the audit'] : []),+ ...auditSteps(workflow, executableLines)+ .filter(step => step.env?.CHECK_AUDIT_REQUIRED !== '1')+ .map(() => 'a step that may run the audit does not require it'),+ ];it.each(['release.yml', 'security-scan.yml'])('%s requires the audit', file => {- const steps = auditSteps(file);- expect(steps.length).toBeGreaterThan(0);- expect(steps.filter(step => step.env?.CHECK_AUDIT_REQUIRED !== '1')).toEqual([]);+ expect(unrequired(auditWorkflow(file))).toEqual([]);+ });++ it.each(['release.yml', 'security-scan.yml'])('%s: refuses an audit step that does not require it', file => {+ const copy = (): AuditWorkflow => JSON.parse(JSON.stringify(auditWorkflow(file))) as AuditWorkflow;+ const anyJob = (workflow: AuditWorkflow): AuditStep[] => Object.values(workflow.jobs ?? {})[0].steps ?? [];+ // Each blank the shell reads as part of a word: the `#` after it is no comment, so the audit after the `;`+ // runs, with the skip.+ for (const code of [0xa0, 0x3000, 0xfeff, 0x2028]) {+ const blank = String.fromCharCode(code);+ const extra = copy();+ anyJob(extra).push({ run: `echo x${blank}#; npm run check:audit` });+ expect([code, unrequired(extra)]).toEqual([code, ['a step that may run the audit does not require it']]);+ }+ // The same line behind a space is a comment, and the real step with its requirement taken off is not.+ const comment = copy();+ anyJob(comment).push({ run: 'echo x # npm run check:audit' });+ expect(unrequired(comment)).toEqual([]);+ const lax = copy();+ for (const step of auditSteps(lax, executableLines)) step.env = { CHECK_AUDIT_REQUIRED: '0' };+ expect(unrequired(lax)).toEqual(['a step that may run the audit does not require it']);+ // A step that only names the audit behind such a blank and a `#` is no audit: one must really run.+ const named = copy();+ for (const step of auditSteps(named, executableLines))+ step.run = `echo x${String.fromCharCode(0xa0)}# npm run check:audit`;+ expect(unrequired(named)).toContain('no step certainly runs the audit');});it('docs/10 describes the weekly audit as required, not as the merge job', () => {@@ -333,9 +366,10 @@ describe('the release and weekly audits fail when the audit cannot run', () => {});it('ci.yml keeps the skip for merges', () => {- const steps = auditSteps('ci.yml');- expect(steps.length).toBeGreaterThan(0);- expect(steps.some(step => step.env?.CHECK_AUDIT_REQUIRED !== undefined)).toBe(false);+ const ci = auditWorkflow('ci.yml');+ expect(auditSteps(ci, executableLinesConservative).length).toBeGreaterThan(0);+ // Every step that may run it, not only one that certainly does.+ expect(auditSteps(ci, executableLines).some(step => step.env?.CHECK_AUDIT_REQUIRED !== undefined)).toBe(false);});});@@ -472,22 +506,63 @@ describe('released images carry a verifiable build-provenance attestation', () =const grant = (perms: Perms, key: string): string | undefined =>typeof perms === 'object' && perms !== null ? perms[key] : undefined;- // Attesting first means a failed attestation leaves no release tag published.+ // Attesting first means a failed attestation leaves no release tag published. Every step that MAY re-point+ // a tag (exact) follows the attestation, and one CERTAINLY does (conservative); a re-point written as+ // `echo x<U+00A0>#; docker buildx imagetools create ...` is a command that runs, so it is held to the order.+ const attestProblems = (steps: AttestStep[]): string[] => {+ const attest = steps.findIndex(step =>+ /^actions\/attest(?:-build-provenance)?@[0-9a-f]{40}\b/.test(step.uses ?? ''),+ );+ const certain = steps.some(step => executableLinesConservative(step.run ?? '').includes('imagetools create'));+ const may = steps.flatMap((step, i) => (executableLines(step.run ?? '').includes('imagetools create') ? [i] : []));+ return [+ ...(attest === -1 ? ['no attest step'] : []),+ ...(certain ? [] : ['no step certainly re-points the tags']),+ ...may.filter(i => i <= attest).map(i => `step ${i + 1} may re-point a tag before the attestation`),+ ];+ };+it('promote attests the tested digest before re-pointing the release tags', () => {const promote = jobs().promote;const steps = promote?.steps ?? [];expect(steps.length).toBeGreaterThan(1);+ expect(attestProblems(steps)).toEqual([]);const attest = steps.findIndex(step =>/^actions\/attest(?:-build-provenance)?@[0-9a-f]{40}\b/.test(step.uses ?? ''),);- const retag = steps.findIndex(step => executableLinesConservative(step.run ?? '').includes('imagetools create'));- expect(attest).toBeGreaterThan(-1);- expect(retag).toBeGreaterThan(attest);expect(steps[attest].with?.['subject-digest']).toContain('needs.docker.outputs.digest');expect(grant(promote?.permissions, 'id-token')).toBe('write');expect(grant(promote?.permissions, 'attestations')).toBe('write');});+ it('refuses a re-point that may run before the attestation, or that is only named', () => {+ const copy = (): AttestStep[] => JSON.parse(JSON.stringify(jobs().promote?.steps ?? [])) as AttestStep[];+ const attestAt = copy().findIndex(step => (step.uses ?? '').startsWith('actions/attest'));+ const REPOINT = 'docker buildx imagetools create --tag t s';+ for (const code of [0xa0, 0x3000, 0xfeff, 0x2028]) {+ const steps = copy();+ steps.splice(attestAt, 0, { run: `echo x${String.fromCharCode(code)}#; ${REPOINT}` });+ expect([code, attestProblems(steps)]).toEqual([+ code,+ [`step ${attestAt + 1} may re-point a tag before the attestation`],+ ]);+ }+ // The attestation moved behind the re-point.+ const moved = copy();+ moved.push(...moved.splice(attestAt, 1));+ expect(attestProblems(moved)).toEqual([+ expect.stringMatching(/^step \d+ may re-point a tag before the attestation$/),+ ]);+ // A re-point that is only named behind a blank and a `#` is no re-point at all.+ const named = copy();+ for (const step of named) {+ if (executableLines(step.run ?? '').includes('imagetools create')) {+ step.run = `echo x${String.fromCharCode(0xa0)}# ${REPOINT}`;+ }+ }+ expect(attestProblems(named)).toEqual(['no step certainly re-points the tags']);+ });+// The signing grant stays on the one job that needs it, away from the build and test jobs.it.each(['docker', 'boot-smoke', 'image-scan'])('%s cannot mint an OIDC token', job => {expect(jobs()[job]).toBeDefined();diff --git a/src/common/workflow-lines.ts b/src/common/workflow-lines.tsindex f37c4282e..eb14f4163 100644--- a/src/common/workflow-lines.ts+++ b/src/common/workflow-lines.ts@@ -20,6 +20,30 @@* found. Use it for a check that something IS run, by substring or by a pattern that is not anchored* at the end. It is never a stand-in for the exact reading elsewhere: it would hide the `curl` in* `echo x<U+00A0>#; curl y | sh`, and turn `npm test<U+00A0># || true` into a clean `npm test`.+ *+ * The question a site asks picks the reading. "Is it run" reads conservatively; a count, a first occurrence,+ * an order, a rule every occurrence must obey and a line compared whole read exactly, and a check that asks+ * both says so by running both. A site that is added or changed goes in this table.+ *+ * ci-privilege-gate.spec.ts+ * invoked scripts: non-root smoke, backup smoke, a Python installer conservative is it run+ * id-token jobs hold `npm publish` and `imagetools create` conservative is it run+ * id-token allow-lists, templates, installers, `${{` in a comment exact every line it may run+ * publish tarball, `npm pack` and `npm publish` order, `python -m build` exact first occurrence+ * java deploy line both is there, every one+ * branch tag: the smoke runs, the last step runs the re-point conservative is it run+ * branch tag: which steps may re-point exact every one that may+ * branch tag: the metadata-action tag rules exact must not be there+ * dashboard unit tests: one step runs them, all that may are bounded both is there, every one+ * PHP mirror: who runs phpunit, the push line both is there, every one+ * PHP mirror: what the job holding the token may run exact every one that may+ * release-gate-parity.spec.ts+ * gate commands and whole steps compared with ci.yml exact line equality+ * postgres lane names its spec, verify-published, engine libraries conservative is it run+ * image-scan ref output, promote decision lines both is there, is the only one+ * check:audit requires the audit both is there, every one+ * audit trees exact order, uniqueness+ * attest before every re-point both is there, every one*/export function executableLines(run: string): string {return withoutComments(run, /[ \t;&|(]/);