Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active
TypeScript aianthropicartifactsawsazurechatgptchatgpt-cloneclaudeclonedeepseekgeminigooglegpt-5librechatmcpo1openairesponses-apivisionwebui
🥼 ci: Gate MCP Apps on Executed Browser Tests (#16283)
lia-by-librechat[bot]627ded714baf
diff --git a/.github/workflows/playwright-mock.yml b/.github/workflows/playwright-mock.ymlindex aa56ad71d3..43122021e9 100644--- a/.github/workflows/playwright-mock.yml+++ b/.github/workflows/playwright-mock.yml@@ -555,6 +555,81 @@ jobs:./redis-config/stop-cluster.sh || trueredis-cli -p 6379 shutdown || true+ mcp_apps:+ name: MCP Apps (browser and server)+ # Always run the dedicated fixture on eligible PRs, schedule, and manual runs. The regular+ # mock shards skip it unless E2E_MCP_APPS=true; codegraph gating must not omit this lane.+ if: >-+ github.event_name == 'schedule' ||+ github.event_name == 'workflow_dispatch' ||+ (github.event_name == 'pull_request' &&+ github.event.pull_request != null &&+ contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.pull_request.author_association))+ runs-on: ubuntu-latest+ timeout-minutes: 45+ env:+ CI: 'true'+ E2E_CHROMIUM_CHANNEL: chrome+ steps:+ - uses: actions/checkout@v5++ - name: Use Node.js 24.16.0+ uses: actions/setup-node@v5+ with:+ node-version: '24.16.0'++ - name: Restore node_modules cache+ id: cache-node-modules+ uses: actions/cache@v5+ with:+ path: |+ node_modules+ client/node_modules+ packages/client/node_modules+ packages/data-provider/node_modules+ packages/data-schemas/node_modules+ packages/api/node_modules+ api/node_modules+ key: node-modules-e2e-${{ runner.os }}-24.16.0-${{ hashFiles('package-lock.json') }}++ - name: Install dependencies+ if: steps.cache-node-modules.outputs.cache-hit != 'true'+ run: npm ci++ - name: Verify Chrome is present+ run: google-chrome --version++ - name: Check MCP Apps result gating+ run: node --test e2e/mcp-apps/results.test.mjs++ - name: Check real SDK server fixture+ run: node --test e2e/mcp-apps/server.test.mjs++ # The runner builds once, then runs all four browser policy phases against one disposable+ # MongoDB. It also rejects a Playwright-successful phase if every test was skipped.+ - name: Run MCP Apps browser fixture (enabled, disabled, omitted, quota)+ run: node e2e/mcp-apps/run.mjs++ - name: Upload per-phase Playwright reports+ if: ${{ !cancelled() }}+ uses: actions/upload-artifact@v6+ with:+ name: playwright-mcp-apps-reports+ path: |+ e2e/playwright-report/mcp-apps-*/**+ e2e/.generated/mcp-apps-result-*.json+ retention-days: 7+ if-no-files-found: warn++ - name: Upload MCP Apps traces and screenshots+ if: failure()+ uses: actions/upload-artifact@v6+ with:+ name: playwright-mcp-apps-results+ path: e2e/specs/.test-results/mcp-apps-*/**+ retention-days: 7+ if-no-files-found: ignore+e2e:name: e2e# `!cancelled()`, not `always()`: the gate has to survive a failed shard to report it, but a@@ -569,13 +644,14 @@ jobs:(github.event_name == 'pull_request' &&github.event.pull_request != null &&contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.pull_request.author_association)))- needs: [codegraph_select, e2e_shards, mcp_tool_list_changed]+ needs: [codegraph_select, e2e_shards, mcp_tool_list_changed, mcp_apps]runs-on: ubuntu-lateststeps:# A codegraph-skipped MCP lane reports `skipped`; that is a decision, not a failure.- name: Verify every Playwright job passedif: >-needs.e2e_shards.result != 'success' ||+ needs.mcp_apps.result != 'success' ||(needs.mcp_tool_list_changed.result != 'success' &&!(needs.mcp_tool_list_changed.result == 'skipped' && needs.codegraph_select.outputs.mcp_run == 'false'))run: exit 1diff --git a/docs/mcp-apps.md b/docs/mcp-apps.mdindex d1991b4770..2d0ff3995e 100644--- a/docs/mcp-apps.md+++ b/docs/mcp-apps.md@@ -205,9 +205,12 @@ Use Node 24 with the exact lockfile installed and Playwright Chromium available.root, run the profile with one command:```bash+node --test e2e/mcp-apps/server.test.mjsnode e2e/mcp-apps/run.mjs```+The Playwright E2E workflow runs these checks on eligible pull requests, the nightly schedule, and manual dispatch, regardless of codegraph selection. Its required `e2e` gate fails if this dedicated job fails. The runner verifies that enabled, disabled, omitted-policy, and quota phases each executed passing browser tests; a successful Playwright exit with only skipped tests does not pass. CI uploads separate reports for each phase and preserves failing traces and screenshots.+The runner builds the client with the chat at `http://127.0.0.1:3080` and the Sandbox Proxy at`http://localhost:3080`. Those URLs reach the same disposable LibreChat process but have distinctbrowser origins. The aliases are only a disposable local substitute for separate deploymentdiff --git a/e2e/mcp-apps/results.mjs b/e2e/mcp-apps/results.mjsnew file mode 100644index 0000000000..628d60efb7--- /dev/null+++ b/e2e/mcp-apps/results.mjs@@ -0,0 +1,35 @@+import assert from 'node:assert/strict';++/** Playwright exits successfully when every test is skipped. The dedicated CI lane must not. */+export function assertMCPAppsPhaseResults(report, phase, minimumPassed) {+ assert.ok(+ Array.isArray(report?.suites) && Array.isArray(report.errors),+ `MCP Apps ${phase}: missing Playwright JSON results`,+ );+ assert.equal(report.errors.length, 0, `MCP Apps ${phase}: Playwright reported errors`);++ const collect = (suites) =>+ (suites ?? []).reduce(+ (passed, suite) =>+ passed ++ (suite.specs ?? []).reduce(+ (count, spec) =>+ count ++ (spec.tests ?? []).filter(+ (test) =>+ test.expectedStatus === 'passed' &&+ test.results?.some((result) => result.status === 'passed'),+ ).length,+ 0,+ ) ++ collect(suite.suites),+ 0,+ );++ const passed = collect(report.suites);+ assert.ok(+ passed >= minimumPassed,+ `MCP Apps ${phase}: expected at least ${minimumPassed} executed passing tests, got ${passed}`,+ );+ return passed;+}diff --git a/e2e/mcp-apps/results.test.mjs b/e2e/mcp-apps/results.test.mjsnew file mode 100644index 0000000000..2fd9c79b76--- /dev/null+++ b/e2e/mcp-apps/results.test.mjs@@ -0,0 +1,49 @@+import assert from 'node:assert/strict';+import { test } from 'node:test';+import { assertMCPAppsPhaseResults } from './results.mjs';++const result = (status, expectedStatus = 'passed') => ({+ expectedStatus,+ results: [{ status }],+});+const report = (...tests) => ({+ errors: [],+ suites: [{ suites: [{ specs: [{ tests }] }] }],+});++test('requires the expected number of executed, passing tests for each phase', () => {+ assert.equal(+ assertMCPAppsPhaseResults(+ report(result('passed'), result('passed'), result('skipped', 'skipped')),+ 'true',+ 2,+ ),+ 2,+ );+ assert.equal(assertMCPAppsPhaseResults(report(result('passed')), 'quota', 1), 1);+});++test('rejects skipped-only and missing-test reports even when Playwright exits successfully', () => {+ assert.throws(+ () => assertMCPAppsPhaseResults(report(result('skipped', 'skipped')), 'false', 1),+ /expected at least 1 executed passing tests, got 0/,+ );+ assert.throws(() => assertMCPAppsPhaseResults(report(), 'omitted', 1), /got 0/);+ assert.throws(() => assertMCPAppsPhaseResults({}, 'quota', 1), /missing Playwright JSON results/);+});++test('does not count a skipped test as a passing test on retry', () => {+ assert.throws(+ () => assertMCPAppsPhaseResults(report(result('passed', 'skipped')), 'true', 1),+ /got 0/,+ );+ assert.throws(+ () =>+ assertMCPAppsPhaseResults(+ { ...report(result('passed')), errors: [{ message: 'failed' }] },+ 'quota',+ 1,+ ),+ /Playwright reported errors/,+ );+});diff --git a/e2e/mcp-apps/run.mjs b/e2e/mcp-apps/run.mjsindex 4276a338f8..1c22481269 100644--- a/e2e/mcp-apps/run.mjs+++ b/e2e/mcp-apps/run.mjs@@ -1,10 +1,12 @@#!/usr/bin/env node+import fs from 'node:fs';import path from 'node:path';import { fileURLToPath } from 'node:url';import { randomBytes } from 'node:crypto';import { spawn } from 'node:child_process';import { createRequire } from 'node:module';+import { assertMCPAppsPhaseResults } from './results.mjs';const baseUrl = new URL(process.env.E2E_BASE_URL || 'http://127.0.0.1:3080');const sandboxHost = baseUrl.hostname === 'localhost' ? '127.0.0.1' : 'localhost';@@ -52,12 +54,13 @@ function run(executable, args, overrides = {}) {}const phases = [- { name: 'true', policy: 'true' },- { name: 'false', policy: 'false' },- { name: 'omitted', policy: 'omitted' },+ { name: 'true', policy: 'true', minimumPassed: 2 },+ { name: 'false', policy: 'false', minimumPassed: 1 },+ { name: 'omitted', policy: 'omitted', minimumPassed: 1 },{name: 'quota',policy: 'true',+ minimumPassed: 1,E2E_MCP_APP_RESOURCE_LIMIT: '2',E2E_MCP_APP_TOOL_CALL_LIMIT: '2',},@@ -74,12 +77,32 @@ try {env.MONGO_URI = mongo.getUri('LibreChat-e2e');for (const phase of phases) {- await run('npx', ['playwright', 'test', '--config=e2e/playwright.config.mcp-apps.ts'], {- E2E_MCP_APPS_PHASE: phase.name,- E2E_MCP_APPS_POLICY: phase.policy,- E2E_MCP_APP_RESOURCE_LIMIT: phase.E2E_MCP_APP_RESOURCE_LIMIT || '',- E2E_MCP_APP_TOOL_CALL_LIMIT: phase.E2E_MCP_APP_TOOL_CALL_LIMIT || '',- });+ const resultsFile = path.join(root, `e2e/.generated/mcp-apps-result-${phase.name}.json`);+ fs.rmSync(resultsFile, { force: true });+ await run(+ 'npx',+ [+ 'playwright',+ 'test',+ '--config=e2e/playwright.config.mcp-apps.ts',+ '--reporter=line,html,json',+ ],+ {+ E2E_MCP_APPS_PHASE: phase.name,+ E2E_MCP_APPS_POLICY: phase.policy,+ E2E_MCP_APP_RESOURCE_LIMIT: phase.E2E_MCP_APP_RESOURCE_LIMIT || '',+ E2E_MCP_APP_TOOL_CALL_LIMIT: phase.E2E_MCP_APP_TOOL_CALL_LIMIT || '',+ PLAYWRIGHT_HTML_OPEN: 'never',+ PLAYWRIGHT_HTML_OUTPUT_DIR: path.join(root, `e2e/playwright-report/mcp-apps-${phase.name}`),+ PLAYWRIGHT_JSON_OUTPUT_FILE: resultsFile,+ },+ );+ const passed = assertMCPAppsPhaseResults(+ JSON.parse(fs.readFileSync(resultsFile, 'utf8')),+ phase.name,+ phase.minimumPassed,+ );+ console.log(`MCP Apps ${phase.name}: ${passed} browser test(s) passed`);}} finally {await mongo?.stop();