Skip to content

Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active

🥼 ci: Gate MCP Apps on Executed Browser Tests (#16283)

lia-by-librechat[bot]627ded714baf
diff --git a/.github/workflows/playwright-mock.yml b/.github/workflows/playwright-mock.yml
index aa56ad71d3..43122021e9 100644
--- a/.github/workflows/playwright-mock.yml
+++ b/.github/workflows/playwright-mock.yml
@@ -555,6 +555,81 @@ jobs:
./redis-config/stop-cluster.sh || true
redis-cli -p 6379 shutdown || true
+ mcp_apps:
+ name: MCP Apps (browser and server)
+ # Always run the dedicated fixture on eligible PRs, schedule, and manual runs. The regular
+ # mock shards skip it unless E2E_MCP_APPS=true; codegraph gating must not omit this lane.
+ if: >-
+ github.event_name == 'schedule' ||
+ github.event_name == 'workflow_dispatch' ||
+ (github.event_name == 'pull_request' &&
+ github.event.pull_request != null &&
+ contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.pull_request.author_association))
+ runs-on: ubuntu-latest
+ timeout-minutes: 45
+ env:
+ CI: 'true'
+ E2E_CHROMIUM_CHANNEL: chrome
+ steps:
+ - uses: actions/checkout@v5
+
+ - name: Use Node.js 24.16.0
+ uses: actions/setup-node@v5
+ with:
+ node-version: '24.16.0'
+
+ - name: Restore node_modules cache
+ id: cache-node-modules
+ uses: actions/cache@v5
+ with:
+ path: |
+ node_modules
+ client/node_modules
+ packages/client/node_modules
+ packages/data-provider/node_modules
+ packages/data-schemas/node_modules
+ packages/api/node_modules
+ api/node_modules
+ key: node-modules-e2e-${{ runner.os }}-24.16.0-${{ hashFiles('package-lock.json') }}
+
+ - name: Install dependencies
+ if: steps.cache-node-modules.outputs.cache-hit != 'true'
+ run: npm ci
+
+ - name: Verify Chrome is present
+ run: google-chrome --version
+
+ - name: Check MCP Apps result gating
+ run: node --test e2e/mcp-apps/results.test.mjs
+
+ - name: Check real SDK server fixture
+ run: node --test e2e/mcp-apps/server.test.mjs
+
+ # The runner builds once, then runs all four browser policy phases against one disposable
+ # MongoDB. It also rejects a Playwright-successful phase if every test was skipped.
+ - name: Run MCP Apps browser fixture (enabled, disabled, omitted, quota)
+ run: node e2e/mcp-apps/run.mjs
+
+ - name: Upload per-phase Playwright reports
+ if: ${{ !cancelled() }}
+ uses: actions/upload-artifact@v6
+ with:
+ name: playwright-mcp-apps-reports
+ path: |
+ e2e/playwright-report/mcp-apps-*/**
+ e2e/.generated/mcp-apps-result-*.json
+ retention-days: 7
+ if-no-files-found: warn
+
+ - name: Upload MCP Apps traces and screenshots
+ if: failure()
+ uses: actions/upload-artifact@v6
+ with:
+ name: playwright-mcp-apps-results
+ path: e2e/specs/.test-results/mcp-apps-*/**
+ retention-days: 7
+ if-no-files-found: ignore
+
e2e:
name: e2e
# `!cancelled()`, not `always()`: the gate has to survive a failed shard to report it, but a
@@ -569,13 +644,14 @@ jobs:
(github.event_name == 'pull_request' &&
github.event.pull_request != null &&
contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.pull_request.author_association)))
- needs: [codegraph_select, e2e_shards, mcp_tool_list_changed]
+ needs: [codegraph_select, e2e_shards, mcp_tool_list_changed, mcp_apps]
runs-on: ubuntu-latest
steps:
# A codegraph-skipped MCP lane reports `skipped`; that is a decision, not a failure.
- name: Verify every Playwright job passed
if: >-
needs.e2e_shards.result != 'success' ||
+ needs.mcp_apps.result != 'success' ||
(needs.mcp_tool_list_changed.result != 'success' &&
!(needs.mcp_tool_list_changed.result == 'skipped' && needs.codegraph_select.outputs.mcp_run == 'false'))
run: exit 1
diff --git a/docs/mcp-apps.md b/docs/mcp-apps.md
index d1991b4770..2d0ff3995e 100644
--- a/docs/mcp-apps.md
+++ b/docs/mcp-apps.md
@@ -205,9 +205,12 @@ Use Node 24 with the exact lockfile installed and Playwright Chromium available.
root, run the profile with one command:
```bash
+node --test e2e/mcp-apps/server.test.mjs
node e2e/mcp-apps/run.mjs
```
+The Playwright E2E workflow runs these checks on eligible pull requests, the nightly schedule, and manual dispatch, regardless of codegraph selection. Its required `e2e` gate fails if this dedicated job fails. The runner verifies that enabled, disabled, omitted-policy, and quota phases each executed passing browser tests; a successful Playwright exit with only skipped tests does not pass. CI uploads separate reports for each phase and preserves failing traces and screenshots.
+
The runner builds the client with the chat at `http://127.0.0.1:3080` and the Sandbox Proxy at
`http://localhost:3080`. Those URLs reach the same disposable LibreChat process but have distinct
browser origins. The aliases are only a disposable local substitute for separate deployment
diff --git a/e2e/mcp-apps/results.mjs b/e2e/mcp-apps/results.mjs
new file mode 100644
index 0000000000..628d60efb7
--- /dev/null
+++ b/e2e/mcp-apps/results.mjs
@@ -0,0 +1,35 @@
+import assert from 'node:assert/strict';
+
+/** Playwright exits successfully when every test is skipped. The dedicated CI lane must not. */
+export function assertMCPAppsPhaseResults(report, phase, minimumPassed) {
+ assert.ok(
+ Array.isArray(report?.suites) && Array.isArray(report.errors),
+ `MCP Apps ${phase}: missing Playwright JSON results`,
+ );
+ assert.equal(report.errors.length, 0, `MCP Apps ${phase}: Playwright reported errors`);
+
+ const collect = (suites) =>
+ (suites ?? []).reduce(
+ (passed, suite) =>
+ passed +
+ (suite.specs ?? []).reduce(
+ (count, spec) =>
+ count +
+ (spec.tests ?? []).filter(
+ (test) =>
+ test.expectedStatus === 'passed' &&
+ test.results?.some((result) => result.status === 'passed'),
+ ).length,
+ 0,
+ ) +
+ collect(suite.suites),
+ 0,
+ );
+
+ const passed = collect(report.suites);
+ assert.ok(
+ passed >= minimumPassed,
+ `MCP Apps ${phase}: expected at least ${minimumPassed} executed passing tests, got ${passed}`,
+ );
+ return passed;
+}
diff --git a/e2e/mcp-apps/results.test.mjs b/e2e/mcp-apps/results.test.mjs
new file mode 100644
index 0000000000..2fd9c79b76
--- /dev/null
+++ b/e2e/mcp-apps/results.test.mjs
@@ -0,0 +1,49 @@
+import assert from 'node:assert/strict';
+import { test } from 'node:test';
+import { assertMCPAppsPhaseResults } from './results.mjs';
+
+const result = (status, expectedStatus = 'passed') => ({
+ expectedStatus,
+ results: [{ status }],
+});
+const report = (...tests) => ({
+ errors: [],
+ suites: [{ suites: [{ specs: [{ tests }] }] }],
+});
+
+test('requires the expected number of executed, passing tests for each phase', () => {
+ assert.equal(
+ assertMCPAppsPhaseResults(
+ report(result('passed'), result('passed'), result('skipped', 'skipped')),
+ 'true',
+ 2,
+ ),
+ 2,
+ );
+ assert.equal(assertMCPAppsPhaseResults(report(result('passed')), 'quota', 1), 1);
+});
+
+test('rejects skipped-only and missing-test reports even when Playwright exits successfully', () => {
+ assert.throws(
+ () => assertMCPAppsPhaseResults(report(result('skipped', 'skipped')), 'false', 1),
+ /expected at least 1 executed passing tests, got 0/,
+ );
+ assert.throws(() => assertMCPAppsPhaseResults(report(), 'omitted', 1), /got 0/);
+ assert.throws(() => assertMCPAppsPhaseResults({}, 'quota', 1), /missing Playwright JSON results/);
+});
+
+test('does not count a skipped test as a passing test on retry', () => {
+ assert.throws(
+ () => assertMCPAppsPhaseResults(report(result('passed', 'skipped')), 'true', 1),
+ /got 0/,
+ );
+ assert.throws(
+ () =>
+ assertMCPAppsPhaseResults(
+ { ...report(result('passed')), errors: [{ message: 'failed' }] },
+ 'quota',
+ 1,
+ ),
+ /Playwright reported errors/,
+ );
+});
diff --git a/e2e/mcp-apps/run.mjs b/e2e/mcp-apps/run.mjs
index 4276a338f8..1c22481269 100644
--- a/e2e/mcp-apps/run.mjs
+++ b/e2e/mcp-apps/run.mjs
@@ -1,10 +1,12 @@
#!/usr/bin/env node
+import fs from 'node:fs';
import path from 'node:path';
import { fileURLToPath } from 'node:url';
import { randomBytes } from 'node:crypto';
import { spawn } from 'node:child_process';
import { createRequire } from 'node:module';
+import { assertMCPAppsPhaseResults } from './results.mjs';
const baseUrl = new URL(process.env.E2E_BASE_URL || 'http://127.0.0.1:3080');
const sandboxHost = baseUrl.hostname === 'localhost' ? '127.0.0.1' : 'localhost';
@@ -52,12 +54,13 @@ function run(executable, args, overrides = {}) {
}
const phases = [
- { name: 'true', policy: 'true' },
- { name: 'false', policy: 'false' },
- { name: 'omitted', policy: 'omitted' },
+ { name: 'true', policy: 'true', minimumPassed: 2 },
+ { name: 'false', policy: 'false', minimumPassed: 1 },
+ { name: 'omitted', policy: 'omitted', minimumPassed: 1 },
{
name: 'quota',
policy: 'true',
+ minimumPassed: 1,
E2E_MCP_APP_RESOURCE_LIMIT: '2',
E2E_MCP_APP_TOOL_CALL_LIMIT: '2',
},
@@ -74,12 +77,32 @@ try {
env.MONGO_URI = mongo.getUri('LibreChat-e2e');
for (const phase of phases) {
- await run('npx', ['playwright', 'test', '--config=e2e/playwright.config.mcp-apps.ts'], {
- E2E_MCP_APPS_PHASE: phase.name,
- E2E_MCP_APPS_POLICY: phase.policy,
- E2E_MCP_APP_RESOURCE_LIMIT: phase.E2E_MCP_APP_RESOURCE_LIMIT || '',
- E2E_MCP_APP_TOOL_CALL_LIMIT: phase.E2E_MCP_APP_TOOL_CALL_LIMIT || '',
- });
+ const resultsFile = path.join(root, `e2e/.generated/mcp-apps-result-${phase.name}.json`);
+ fs.rmSync(resultsFile, { force: true });
+ await run(
+ 'npx',
+ [
+ 'playwright',
+ 'test',
+ '--config=e2e/playwright.config.mcp-apps.ts',
+ '--reporter=line,html,json',
+ ],
+ {
+ E2E_MCP_APPS_PHASE: phase.name,
+ E2E_MCP_APPS_POLICY: phase.policy,
+ E2E_MCP_APP_RESOURCE_LIMIT: phase.E2E_MCP_APP_RESOURCE_LIMIT || '',
+ E2E_MCP_APP_TOOL_CALL_LIMIT: phase.E2E_MCP_APP_TOOL_CALL_LIMIT || '',
+ PLAYWRIGHT_HTML_OPEN: 'never',
+ PLAYWRIGHT_HTML_OUTPUT_DIR: path.join(root, `e2e/playwright-report/mcp-apps-${phase.name}`),
+ PLAYWRIGHT_JSON_OUTPUT_FILE: resultsFile,
+ },
+ );
+ const passed = assertMCPAppsPhaseResults(
+ JSON.parse(fs.readFileSync(resultsFile, 'utf8')),
+ phase.name,
+ phase.minimumPassed,
+ );
+ console.log(`MCP Apps ${phase.name}: ${passed} browser test(s) passed`);
}
} finally {
await mongo?.stop();