Skip to content

Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active

🐙 ci: Spread Playwright Coverage Across Isolated Shards (#16358)

lia-by-librechat[bot]24570b68cfb2
diff --git a/.github/scripts/playwright-matrix.test.mjs b/.github/scripts/playwright-matrix.test.mjs
new file mode 100644
index 0000000000..346741a6d8
--- /dev/null
+++ b/.github/scripts/playwright-matrix.test.mjs
@@ -0,0 +1,200 @@
+import yaml from 'js-yaml';
+import { test } from 'node:test';
+import { join } from 'node:path';
+import assert from 'node:assert/strict';
+import { fileURLToPath } from 'node:url';
+import { spawnSync } from 'node:child_process';
+import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
+
+const scriptsDir = fileURLToPath(new URL('.', import.meta.url));
+const workflow = yaml.load(
+ readFileSync(new URL('../workflows/playwright-mock.yml', import.meta.url), 'utf8'),
+);
+const selector = workflow.jobs.codegraph_select.steps.find((step) => step.id === 'sel');
+const shards = workflow.jobs.e2e_shards;
+const defaults = [...shards.strategy.matrix.matchAll(/fromJSON\('([^']+)'\)/g)].map((match) =>
+ JSON.parse(match[1]),
+);
+const prMatrix = JSON.parse(selector.env.FULL_INCLUDE);
+
+function assertPartition(matrix, store, suite, count) {
+ const lanes = matrix.include.filter(
+ (lane) => lane.stream_store === store && lane.suite === suite,
+ );
+ assert.equal(lanes.length, count);
+ assert.deepEqual(
+ lanes.map((lane) => lane.shard),
+ Array.from({ length: count }, (_, index) => `${index + 1}/${count}`),
+ );
+ assert.ok(
+ lanes.every((lane) => lane.redis_image === (store === 'redis' ? 'redis:7-alpine' : '')),
+ );
+}
+
+function select(t, response, overrides = {}) {
+ const scratch = mkdtempSync(join(scriptsDir, '.playwright-matrix-test-'));
+ t.after(() => rmSync(scratch, { recursive: true, force: true }));
+ writeFileSync(join(scratch, 'response.json'), JSON.stringify(response));
+ writeFileSync(join(scratch, 'output'), '');
+ writeFileSync(join(scratch, 'selector.sh'), selector.run);
+ writeFileSync(
+ join(scratch, 'gh'),
+ '#!/usr/bin/env bash\nprintf \'%s\\n\' \'{"path":"client/src/example.ts","status":"modified"}\'\nexit "${GH_EXIT:-0}"\n',
+ { mode: 0o700 },
+ );
+ writeFileSync(
+ join(scratch, 'curl'),
+ '#!/usr/bin/env bash\ncat response.json\nexit "${CURL_EXIT:-0}"\n',
+ { mode: 0o700 },
+ );
+ const result = spawnSync('bash', ['selector.sh'], {
+ cwd: scratch,
+ encoding: 'utf8',
+ timeout: 10000,
+ env: {
+ PATH: `${scratch}:${process.env.PATH}`,
+ FULL_INCLUDE: selector.env.FULL_INCLUDE,
+ URL: 'https://selector.invalid',
+ TOKEN: 'test-fixture',
+ REPO: 'fixture/repo',
+ PR: '1',
+ BASE_SHA: 'base',
+ HEAD_SHA: 'head',
+ CHANGED: '1',
+ E2E_SKIP_ARMED: '',
+ GITHUB_OUTPUT: join(scratch, 'output'),
+ GITHUB_STEP_SUMMARY: join(scratch, 'summary'),
+ ...overrides,
+ },
+ });
+ assert.equal(result.status, 0, result.stdout + result.stderr);
+ const outputs = Object.fromEntries(
+ readFileSync(join(scratch, 'output'), 'utf8')
+ .trim()
+ .split('\n')
+ .filter(Boolean)
+ .map((line) => {
+ const separator = line.indexOf('=');
+ return [line.slice(0, separator), line.slice(separator + 1)];
+ }),
+ );
+ return { outputs, matrix: outputs.e2e_include ? JSON.parse(outputs.e2e_include) : defaults[0] };
+}
+
+function decision(redis, mcp = true) {
+ return { matrix: { 'playwright-mock': { redis_transport: redis, mcp_tool_list_changed: mcp } } };
+}
+
+test('PR selection and fail-open defaults use six memory shards and two Redis transport shards', () => {
+ assert.equal(defaults.length, 2);
+ assert.deepEqual(defaults[0], prMatrix);
+ assert.equal(prMatrix.include.length, 8);
+ assertPartition(prMatrix, 'memory', 'full', 6);
+ assertPartition(prMatrix, 'redis', 'transport', 2);
+});
+
+test('nightly and default manual runs retain full-suite coverage on both stores', () => {
+ assert.equal(workflow.on.workflow_dispatch.inputs.coverage.default, 'full');
+ assert.equal(defaults[1].include.length, 4);
+ assertPartition(defaults[1], 'memory', 'full', 2);
+ assertPartition(defaults[1], 'redis', 'full', 2);
+});
+
+test('manual PR coverage opts into the PR fallback matrix without bypassing the PR author gate', () => {
+ assert.deepEqual(workflow.on.workflow_dispatch.inputs.coverage.options, ['full', 'pr']);
+ assert.match(
+ shards.strategy.matrix,
+ /github\.event_name == 'pull_request' \|\| inputs\.coverage == 'pr'/,
+ );
+ assert.match(shards.if, /github\.event_name == 'workflow_dispatch'/);
+ assert.match(shards.if, /github\.event\.pull_request\.author_association/);
+});
+
+test('every shard has a unique job name and artifact namespace', () => {
+ for (const matrix of [prMatrix, ...defaults]) {
+ for (const key of ['name', 'artifact']) {
+ assert.equal(new Set(matrix.include.map((lane) => lane[key])).size, matrix.include.length);
+ }
+ }
+});
+
+test('both suite commands consume the matrix shard without increasing worker concurrency', () => {
+ for (const suite of ['full', 'transport']) {
+ const step = shards.steps.find((entry) => entry.if === `matrix.suite == '${suite}'`);
+ assert.match(step.run, /--shard=\$\{\{ matrix\.shard \}\}/);
+ assert.doesNotMatch(step.run, /--workers/);
+ }
+ const config = readFileSync(
+ new URL('../../e2e/playwright.config.mock.ts', import.meta.url),
+ 'utf8',
+ );
+ assert.match(config, /workers: 1,/);
+ assert.equal(shards['runs-on'], 'ubuntu-latest');
+});
+
+test('an explicit Redis skip drops both transport shards and no memory shard', (t) => {
+ const { matrix, outputs } = select(t, decision(false, false));
+ assert.equal(outputs.decided, 'true');
+ assert.equal(outputs.mcp_run, 'false');
+ assert.equal(matrix.include.length, 6);
+ assertPartition(matrix, 'memory', 'full', 6);
+});
+
+test('missing or non-boolean Redis decisions retain both transport shards', (t) => {
+ for (const redis of [true, 'false', null, undefined]) {
+ const { matrix } = select(t, decision(redis));
+ assert.deepEqual(matrix, prMatrix);
+ assertPartition(matrix, 'redis', 'transport', 2);
+ }
+});
+
+test('unavailable configuration, failed requests, and incomplete file lists fall back to the full PR matrix', (t) => {
+ for (const overrides of [
+ { URL: '' },
+ { TOKEN: '' },
+ { CURL_EXIT: '28' },
+ { GH_EXIT: '1' },
+ { CHANGED: '2' },
+ ]) {
+ const { matrix, outputs } = select(t, decision(false, false), overrides);
+ assert.equal(outputs.e2e_include, undefined);
+ assert.equal(outputs.mcp_run, undefined);
+ assert.deepEqual(matrix, prMatrix);
+ }
+ const { matrix, outputs } = select(t, {});
+ assert.equal(outputs.e2e_include, undefined);
+ assert.deepEqual(matrix, prMatrix);
+});
+
+test('graduated spec skips keep the same pool across shards and remain opt-in', (t) => {
+ const response = {
+ ...decision(true),
+ e2e: { fail_open: false, graduated: ['e2e/specs/mock/sidebar.spec.ts'] },
+ };
+ const dark = select(t, response);
+ assert.equal(dark.outputs.e2e_skip, '');
+ const armed = select(t, response, { E2E_SKIP_ARMED: 'on' });
+ assert.equal(armed.outputs.e2e_skip, 'specs/mock/sidebar.spec.ts');
+ assert.deepEqual(armed.matrix, prMatrix);
+ const fallback = select(
+ t,
+ { ...response, e2e: { ...response.e2e, fail_open: true } },
+ { E2E_SKIP_ARMED: 'on' },
+ );
+ assert.equal(fallback.outputs.e2e_skip, '');
+ assert.deepEqual(fallback.matrix, prMatrix);
+});
+
+test('the required gate still waits for all shards and rejects failures without fighting cancellation', () => {
+ const gate = workflow.jobs.e2e;
+ assert.equal(gate.name, 'e2e');
+ assert.deepEqual(gate.needs, ['codegraph_select', 'e2e_shards', 'mcp_tool_list_changed']);
+ assert.equal(shards.strategy['fail-fast'], false);
+ assert.match(gate.if, /!cancelled\(\)/);
+ assert.match(shards.if, /!cancelled\(\)/);
+ assert.match(gate.steps[0].if, /needs\.e2e_shards\.result != 'success'/);
+ assert.match(gate.steps[0].if, /needs\.mcp_tool_list_changed\.result != 'success'/);
+ assert.match(gate.steps[0].if, /needs\.codegraph_select\.outputs\.mcp_run == 'false'/);
+ assert.equal(gate.steps[0].run, 'exit 1');
+ assert.equal(workflow.concurrency['cancel-in-progress'], true);
+});
diff --git a/.github/workflows/playwright-mock.yml b/.github/workflows/playwright-mock.yml
index acb62732b1..9713220892 100644
--- a/.github/workflows/playwright-mock.yml
+++ b/.github/workflows/playwright-mock.yml
@@ -15,6 +15,13 @@ on:
description: 'Reason for manual trigger'
required: false
default: 'Manual e2e run'
+ coverage:
+ description: 'Coverage profile: full suite on both stores, or the PR shards'
+ type: choice
+ options:
+ - full
+ - pr
+ default: full
permissions:
contents: read
@@ -29,7 +36,7 @@ env:
jobs:
# Stage 2 of codegraph gating (stage 1 = backend jest in backend-review.yml). The graph decides
- # two matrix lanes whose relevance is a reachability question: the redis-transport lane (the ten
+ # two matrix lanes whose relevance is a reachability question: the redis-transport shards (the
# stream-boundary specs under a real Redis round-trip) and the MCP list_changed lanes. Memory
# shards always run. Monotone and fail-open: a lane is dropped ONLY on an explicit `false` from
# the service; unavailable/unconfigured/non-synchronize events keep the full matrix. Lock
@@ -63,7 +70,7 @@ jobs:
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
CHANGED: ${{ github.event.pull_request.changed_files }}
E2E_SKIP_ARMED: ${{ vars.CODEGRAPH_E2E_SKIP }}
- FULL_INCLUDE: '{"include":[{"name":"memory, shard 1/3","stream_store":"memory","redis_image":"","suite":"full","shard":"1/3","artifact":"memory-1-of-3"},{"name":"memory, shard 2/3","stream_store":"memory","redis_image":"","suite":"full","shard":"2/3","artifact":"memory-2-of-3"},{"name":"memory, shard 3/3","stream_store":"memory","redis_image":"","suite":"full","shard":"3/3","artifact":"memory-3-of-3"},{"name":"redis transport","stream_store":"redis","redis_image":"redis:7-alpine","suite":"transport","shard":"","artifact":"redis-transport"}]}'
+ FULL_INCLUDE: '{"include":[{"name":"memory, shard 1/6","stream_store":"memory","redis_image":"","suite":"full","shard":"1/6","artifact":"memory-1-of-6"},{"name":"memory, shard 2/6","stream_store":"memory","redis_image":"","suite":"full","shard":"2/6","artifact":"memory-2-of-6"},{"name":"memory, shard 3/6","stream_store":"memory","redis_image":"","suite":"full","shard":"3/6","artifact":"memory-3-of-6"},{"name":"memory, shard 4/6","stream_store":"memory","redis_image":"","suite":"full","shard":"4/6","artifact":"memory-4-of-6"},{"name":"memory, shard 5/6","stream_store":"memory","redis_image":"","suite":"full","shard":"5/6","artifact":"memory-5-of-6"},{"name":"memory, shard 6/6","stream_store":"memory","redis_image":"","suite":"full","shard":"6/6","artifact":"memory-6-of-6"},{"name":"redis transport, shard 1/2","stream_store":"redis","redis_image":"redis:7-alpine","suite":"transport","shard":"1/2","artifact":"redis-transport-1-of-2"},{"name":"redis transport, shard 2/2","stream_store":"redis","redis_image":"redis:7-alpine","suite":"transport","shard":"2/2","artifact":"redis-transport-2-of-2"}]}'
run: |
set +e
note() { echo "$1" >> "$GITHUB_STEP_SUMMARY"; }
@@ -112,7 +119,7 @@ jobs:
INCLUDE="$FULL_INCLUDE"
note "| redis transport | run |"
fi
- if ! echo "$INCLUDE" | jq -e '.include | length >= 3' >/dev/null 2>&1; then
+ if ! echo "$INCLUDE" | jq -e '.include | length >= 6' >/dev/null 2>&1; then
note "_matrix assembly failed; running FULL_"
exit 0
fi
@@ -161,14 +168,16 @@ jobs:
contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.pull_request.author_association)))
runs-on: ubuntu-latest
timeout-minutes: 30
+ # Separate runners isolate each shard's user, MongoDB, Redis and fixture state.
+ # Keep workers: 1 in Playwright; shared-user cleanup is not worker-safe.
strategy:
fail-fast: false
matrix: >-
${{
(github.event_name == 'pull_request' && needs.codegraph_select.outputs.e2e_include != '' &&
fromJSON(needs.codegraph_select.outputs.e2e_include)) ||
- (github.event_name == 'pull_request' &&
- fromJSON('{"include":[{"name":"memory, shard 1/3","stream_store":"memory","redis_image":"","suite":"full","shard":"1/3","artifact":"memory-1-of-3"},{"name":"memory, shard 2/3","stream_store":"memory","redis_image":"","suite":"full","shard":"2/3","artifact":"memory-2-of-3"},{"name":"memory, shard 3/3","stream_store":"memory","redis_image":"","suite":"full","shard":"3/3","artifact":"memory-3-of-3"},{"name":"redis transport","stream_store":"redis","redis_image":"redis:7-alpine","suite":"transport","shard":"","artifact":"redis-transport"}]}')) ||
+ ((github.event_name == 'pull_request' || inputs.coverage == 'pr') &&
+ fromJSON('{"include":[{"name":"memory, shard 1/6","stream_store":"memory","redis_image":"","suite":"full","shard":"1/6","artifact":"memory-1-of-6"},{"name":"memory, shard 2/6","stream_store":"memory","redis_image":"","suite":"full","shard":"2/6","artifact":"memory-2-of-6"},{"name":"memory, shard 3/6","stream_store":"memory","redis_image":"","suite":"full","shard":"3/6","artifact":"memory-3-of-6"},{"name":"memory, shard 4/6","stream_store":"memory","redis_image":"","suite":"full","shard":"4/6","artifact":"memory-4-of-6"},{"name":"memory, shard 5/6","stream_store":"memory","redis_image":"","suite":"full","shard":"5/6","artifact":"memory-5-of-6"},{"name":"memory, shard 6/6","stream_store":"memory","redis_image":"","suite":"full","shard":"6/6","artifact":"memory-6-of-6"},{"name":"redis transport, shard 1/2","stream_store":"redis","redis_image":"redis:7-alpine","suite":"transport","shard":"1/2","artifact":"redis-transport-1-of-2"},{"name":"redis transport, shard 2/2","stream_store":"redis","redis_image":"redis:7-alpine","suite":"transport","shard":"2/2","artifact":"redis-transport-2-of-2"}]}')) ||
fromJSON('{"include":[{"name":"memory, shard 1/2","stream_store":"memory","redis_image":"","suite":"full","shard":"1/2","artifact":"memory-1-of-2"},{"name":"memory, shard 2/2","stream_store":"memory","redis_image":"","suite":"full","shard":"2/2","artifact":"memory-2-of-2"},{"name":"redis, shard 1/2","stream_store":"redis","redis_image":"redis:7-alpine","suite":"full","shard":"1/2","artifact":"redis-1-of-2"},{"name":"redis, shard 2/2","stream_store":"redis","redis_image":"redis:7-alpine","suite":"full","shard":"2/2","artifact":"redis-2-of-2"}]}')
}}
services:
@@ -211,6 +220,9 @@ jobs:
if: steps.cache-node-modules.outputs.cache-hit != 'true'
run: npm ci
+ - name: Test Playwright matrix selection
+ run: node --test .github/scripts/playwright-matrix.test.mjs
+
- name: Restore data-provider build cache
id: cache-data-provider
uses: actions/cache@v5
@@ -356,7 +368,7 @@ jobs:
- name: Run Redis stream transport e2e
if: matrix.suite == 'transport'
- run: npx playwright test --config=e2e/playwright.config.redis.ts
+ run: npx playwright test --config=e2e/playwright.config.redis.ts --shard=${{ matrix.shard }}
env:
CI: 'true'