A free and open source instant messaging and VoIP chat app built for friends, groups, and communities.
test(config): expand the shipped stack on another port (#2612)
Hampus3e3241484974
diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yamlindex 545dcd8d5..7ab669fef 100644--- a/.github/workflows/tests.yaml+++ b/.github/workflows/tests.yaml@@ -426,6 +426,28 @@ jobs:'packages/markdown_parser/rust/Cargo.toml', 'packages/markdown_parser/rust/.cargo/config.toml','packages/markdown_parser/rust/src/**') }}+ lint:+ runs-on: ubuntu-24.04+ timeout-minutes: 15+ steps:+ - name: Checkout code+ uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0++ - name: Install pnpm+ uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271++ - name: Install Node.js+ uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e+ with:+ node-version: '24'+ cache: 'pnpm'++ - name: Install dependencies+ run: pnpm install --frozen-lockfile++ - name: Check formatting and lint+ run: pnpm exec biome ci .+i18n:runs-on: ubuntu-24.04timeout-minutes: 25diff --git a/packages/config/src/__tests__/DeployEnvCoverage.test.ts b/packages/config/src/__tests__/DeployEnvCoverage.test.tsindex 8f8c1e7c6..23b76de74 100644--- a/packages/config/src/__tests__/DeployEnvCoverage.test.ts+++ b/packages/config/src/__tests__/DeployEnvCoverage.test.ts@@ -1,6 +1,6 @@// SPDX-License-Identifier: AGPL-3.0-or-later-import {readFileSync} from 'node:fs';+import {readdirSync, readFileSync} from 'node:fs';import path from 'node:path';import {fileURLToPath} from 'node:url';import {describe, expect, test} from 'vitest';@@ -11,9 +11,15 @@ const INTERPOLATION_PATTERN = /\$\{([A-Z][A-Z0-9_]*)[:?}-]/g;const CADDY_PLACEHOLDER_PATTERN = /\{\$([A-Z][A-Z0-9_]*)[:}]/g;const DECLARATION_PATTERN = /^#?([A-Z][A-Z0-9_]*)=/gm;const ACTIVE_DECLARATION_PATTERN = /^([A-Z][A-Z0-9_]*)=/gm;+const LINE_DECLARATION_PATTERN = /^#?([A-Z][A-Z0-9_]*)=(.*)$/u;+const LINE_REFERENCE_PATTERN = /\$\{?([A-Z][A-Z0-9_]*)/gu;const read = (name: string) => readFileSync(path.join(SELF_HOSTING, name), 'utf8');+const composeFiles = readdirSync(SELF_HOSTING)+ .filter((name) => name.endsWith('.yml'))+ .sort();+const namesMatching = (source: string, pattern: RegExp) =>new Set([...source.matchAll(pattern)].map(([, name]) => name));@@ -23,7 +29,7 @@ const example = read('.env.example');const declared = new Set([...example.matchAll(DECLARATION_PATTERN)].map(([, name]) => name));describe('.env.example covers every name the compose files interpolate', () => {- for (const file of ['docker-compose.yml', 'docker-compose.proxy.yml']) {+ for (const file of composeFiles) {test(`every \${NAME} in ${file} has a line in .env.example`, () => {const missing = [...interpolatedNames(read(file))].filter((name) => !declared.has(name)).sort();expect(missing).toEqual([]);@@ -48,4 +54,29 @@ describe('.env.example covers every name the compose files interpolate', () => {}expect([...repeated].sort()).toEqual([]);});++ test('every name a .env.example line reads is declared above it', () => {+ const lines = example.split('\n');+ const declaredAt = new Map<string, number>();+ lines.forEach((line, index) => {+ const match = LINE_DECLARATION_PATTERN.exec(line);+ if (match && !declaredAt.has(match[1])) {+ declaredAt.set(match[1], index);+ }+ });+ const unresolved: Array<string> = [];+ lines.forEach((line, index) => {+ const match = LINE_DECLARATION_PATTERN.exec(line);+ if (!match) {+ return;+ }+ for (const [, name] of match[2].matchAll(LINE_REFERENCE_PATTERN)) {+ const at = declaredAt.get(name);+ if (at === undefined || at > index) {+ unresolved.push(`${match[1]} reads ${name}`);+ }+ }+ });+ expect(unresolved.sort()).toEqual([]);+ });});diff --git a/packages/config/src/__tests__/NonDefaultPortCompose.test.ts b/packages/config/src/__tests__/NonDefaultPortCompose.test.tsnew file mode 100644index 000000000..a44de0a3b--- /dev/null+++ b/packages/config/src/__tests__/NonDefaultPortCompose.test.ts@@ -0,0 +1,341 @@+// SPDX-License-Identifier: AGPL-3.0-or-later++import {readFileSync} from 'node:fs';+import path from 'node:path';+import {fileURLToPath} from 'node:url';+import {loadConfig, resetConfig} from '@fluxer/config/src/ConfigLoader';+import {normalizePublicEndpoint} from '@fluxer/config/src/EndpointDerivation';+import type {MasterConfig} from '@fluxer/config/src/MasterConfig';+import {afterEach, beforeEach, describe, expect, test, vi} from 'vitest';++const SELF_HOSTING = path.join(fileURLToPath(new URL('../../../../', import.meta.url)), 'deploy/self-hosting');++const compose = readFileSync(path.join(SELF_HOSTING, 'docker-compose.yml'), 'utf8');+const lines = compose.split('\n');++const DOMAIN = 'chat.example.com';+const PUBLIC_PORT = '19080';+const ORIGIN_PORT = '29080';++const SECRETS: Record<string, string> = {+ FLUXER_DOMAIN: DOMAIN,+ POSTGRES_PASSWORD: 'postgres-password',+ MEILI_MASTER_KEY: 'meili-master-key',+ FLUXER_S3_ACCESS_KEY: 's3-access-key',+ FLUXER_S3_SECRET_KEY: 's3-secret-key',+ LIVEKIT_API_KEY: 'livekit-api-key',+ LIVEKIT_API_SECRET: 'livekit-api-secret',+ FLUXER_ERLANG_COOKIE: 'erlang-cookie',+ FLUXER_SUDO_MODE_SECRET: 'sudo-mode-secret',+ FLUXER_CONNECTION_INITIATION_SECRET: 'connection-initiation-secret',+ FLUXER_GATEWAY_RPC_AUTH_TOKEN: 'gateway-rpc-auth-token',+ FLUXER_MEDIA_PROXY_SECRET_KEY: 'media-proxy-secret-key',+ FLUXER_MEDIA_PROXY_UPLOAD_RELAY_SECRET_BASE64: 'AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=',+ FLUXER_ADMIN_SECRET_KEY_BASE: 'admin-secret-key-base',+ FLUXER_ADMIN_OAUTH_CLIENT_SECRET: 'admin-oauth-client-secret',+ FLUXER_VAPID_PUBLIC_KEY: 'BB76bTFIuoqmxJtTfZX0yGTn1f_qu9H03B_nkj8OyExJFkN7Y-HBZZzShnHZoEhXKc5ZRy3jFu7OkBbnaQG-4aw',+ FLUXER_VAPID_PRIVATE_KEY: 'Xgi-3P8J-I3Q6U1HlCcXMuc_tKLGAM9nIfznX3Hz68o',+};++const PORT_ONLY_ENV: Record<string, string> = {+ ...SECRETS,+ FLUXER_PUBLIC_SCHEME: 'http',+ FLUXER_PUBLIC_PORT: PUBLIC_PORT,+};++const DOCUMENTED_RECIPE_ENV: Record<string, string> = {+ ...PORT_ONLY_ENV,+ FLUXER_PUBLIC_ORIGIN: `http://${DOMAIN}:${PUBLIC_PORT}`,+ FLUXER_HTTP_PORT: PUBLIC_PORT,+};++const ORIGIN_ONLY_ENV: Record<string, string> = {+ ...SECRETS,+ FLUXER_PUBLIC_ORIGIN: `https://${DOMAIN}:${ORIGIN_PORT}`,+ FLUXER_HTTPS_PORT: ORIGIN_PORT,+};++const indentOf = (line: string): number => line.length - line.trimStart().length;++function unquote(value: string): string {+ const trimmed = value.trim();+ const quote = trimmed.slice(0, 1);+ const quoted = (quote === '"' || quote === "'") && trimmed.length > 1 && trimmed.endsWith(quote);+ return quoted ? trimmed.slice(1, -1) : trimmed;+}++function anchorStart(anchor: string): number {+ const start = lines.findIndex((line) => line.startsWith('x-') && line.endsWith(`&${anchor}`));+ if (start === -1) {+ throw new Error(`docker-compose.yml has no &${anchor} anchor`);+ }+ return start;+}++function mapping(start: number, indent: number): Map<string, string> {+ const entries = new Map<string, string>();+ for (let index = start; index < lines.length; index += 1) {+ const line = lines[index];+ if (line.trim().length === 0 || line.trimStart().startsWith('#')) {+ continue;+ }+ if (indentOf(line) < indent) {+ break;+ }+ if (indentOf(line) > indent) {+ continue;+ }+ const merge = /^<<: \*([a-z][a-z0-9-]*)$/u.exec(line.trim());+ if (merge) {+ for (const [key, value] of mapping(anchorStart(merge[1]) + 1, 2)) {+ entries.set(key, value);+ }+ continue;+ }+ const entry = /^([A-Za-z_][A-Za-z0-9_]*): (.*)$/u.exec(line.trim());+ if (entry) {+ entries.set(entry[1], unquote(entry[2]));+ }+ }+ return entries;+}++function serviceSection(service: string, key: string): number {+ const start = lines.indexOf(` ${service}:`);+ if (start === -1) {+ throw new Error(`docker-compose.yml has no ${service} service`);+ }+ for (let index = start + 1; index < lines.length; index += 1) {+ const line = lines[index];+ if (line.trim().length > 0 && indentOf(line) <= 2) {+ break;+ }+ if (line === ` ${key}:`) {+ return index;+ }+ }+ return -1;+}++function serviceList(service: string, key: string): Array<string> {+ const start = serviceSection(service, key);+ if (start === -1) {+ return [];+ }+ const items: Array<string> = [];+ for (let index = start + 1; index < lines.length; index += 1) {+ const line = lines[index];+ if (line.trim().length === 0) {+ continue;+ }+ if (indentOf(line) <= 4) {+ break;+ }+ const item = /^- (.*)$/u.exec(line.trim());+ if (item) {+ items.push(unquote(item[1]));+ }+ }+ return items;+}++const SERVICES_WITHOUT_ENDPOINT_REPAIR = new Set(['edge']);++function serviceNames(): Array<string> {+ const section = compose.slice(compose.indexOf('\nservices:\n'), compose.indexOf('\nnetworks:\n'));+ return [...section.matchAll(/\n {2}([a-z][a-z0-9_-]*):\n/gu)].map(([, name]) => name);+}++function closingBrace(value: string, open: number): number {+ let depth = 0;+ for (let index = open; index < value.length; index += 1) {+ if (value[index] === '{') {+ depth += 1;+ } else if (value[index] === '}') {+ depth -= 1;+ if (depth === 0) {+ return index;+ }+ }+ }+ throw new Error(`unbalanced interpolation in ${value}`);+}++function expand(value: string, env: Record<string, string>): string {+ let out = '';+ let index = 0;+ while (index < value.length) {+ const dollar = value.indexOf('$', index);+ if (dollar === -1) {+ return out + value.slice(index);+ }+ out += value.slice(index, dollar);+ if (value[dollar + 1] !== '{') {+ const bare = /^[A-Za-z_][A-Za-z0-9_]*/u.exec(value.slice(dollar + 1));+ out += bare ? (env[bare[0]] ?? '') : '$';+ index = dollar + 1 + (bare ? bare[0].length : 0);+ continue;+ }+ const close = closingBrace(value, dollar + 1);+ const inner = value.slice(dollar + 2, close);+ const operator = inner.search(/:[-?+]/u);+ const name = operator === -1 ? inner : inner.slice(0, operator);+ const fallback = operator === -1 ? '' : inner.slice(operator + 2);+ const current = env[name] ?? '';+ if (current.length > 0) {+ out += inner[operator + 1] === '+' && operator !== -1 ? expand(fallback, env) : current;+ } else if (inner[operator + 1] === '-' && operator !== -1) {+ out += expand(fallback, env);+ } else if (inner[operator + 1] === '?' && operator !== -1) {+ throw new Error(`${name} is required: ${fallback}`);+ }+ index = close + 1;+ }+ return out;+}++function expandedEnvironment(service: string, env: Record<string, string>): Record<string, string> {+ const expanded: Record<string, string> = {};+ for (const [key, value] of mapping(serviceSection(service, 'environment') + 1, 6)) {+ expanded[key] = expand(value, env);+ }+ return expanded;+}++function publicUrl(value: string): URL | null {+ let parsed: URL;+ try {+ parsed = new URL(value);+ } catch {+ return null;+ }+ return parsed.hostname === DOMAIN ? parsed : null;+}++function publicUrlNames(environment: Record<string, string>): Array<string> {+ return Object.keys(environment)+ .filter((name) => publicUrl(environment[name]) !== null)+ .sort();+}++function repairedPublicUrls(service: string, env: Record<string, string>): Array<[string, string]> {+ const environment = expandedEnvironment(service, env);+ const publicPort = Number.parseInt(environment.FLUXER_PUBLIC_PORT ?? '', 10);+ return publicUrlNames(environment).map((name) => [+ `${service}.${name}`,+ normalizePublicEndpoint(+ environment[name],+ environment.FLUXER_BASE_DOMAIN ?? '',+ Number.isNaN(publicPort) ? undefined : publicPort,+ ),+ ]);+}++function withoutPort(entries: Array<[string, string]>, port: string): Array<string> {+ return entries+ .filter(([, value]) => publicUrl(value)?.port !== port)+ .map(([name, value]) => `${name}=${value}`)+ .sort();+}++function publishedPorts(env: Record<string, string>): Array<string> {+ return serviceList('edge', 'ports')+ .map((mapped) => expand(mapped, env).split(':'))+ .map((parts) => parts[parts.length - 2] ?? '');+}++function browserFacingUrls(config: MasterConfig): Array<[string, string]> {+ const entries: Array<[string, string]> = [+ ...Object.entries(config.endpoints),+ ['integrations.voice.url', config.integrations.voice.url],+ ['services.media_proxy.upload_relay.endpoint', config.services.media_proxy.upload_relay.endpoint],+ ...config.auth.passkeys.additional_allowed_origins.map((origin, index): [string, string] => [+ `auth.passkeys.additional_allowed_origins.${index}`,+ origin,+ ]),+ ];+ return entries.filter(([, value]) => publicUrl(value) !== null);+}++async function loadApiConfig(env: Record<string, string>): Promise<MasterConfig | Error> {+ for (const key of Object.keys(process.env)) {+ if (key.startsWith('FLUXER_')) {+ vi.stubEnv(key, undefined);+ }+ }+ for (const [key, value] of Object.entries(expandedEnvironment('api', env))) {+ if (key.startsWith('FLUXER_')) {+ vi.stubEnv(key, value);+ }+ }+ try {+ return await loadConfig();+ } catch (error) {+ return error instanceof Error ? error : new Error(String(error));+ }+}++describe('the shipped compose stack expanded on a non-default port', () => {+ test('every service handed a public URL is handed the base domain and port that repair it', () => {+ const starved = serviceNames()+ .filter((service) => !SERVICES_WITHOUT_ENDPOINT_REPAIR.has(service))+ .filter((service) => {+ const environment = expandedEnvironment(service, PORT_ONLY_ENV);+ return (+ publicUrlNames(environment).length > 0 && (!environment.FLUXER_BASE_DOMAIN || !environment.FLUXER_PUBLIC_PORT)+ );+ });+ expect(starved).toEqual([]);+ });++ test('every public URL the stack hands a browser carries the port', () => {+ const entries = serviceNames()+ .filter((service) => !SERVICES_WITHOUT_ENDPOINT_REPAIR.has(service))+ .flatMap((service) => repairedPublicUrls(service, PORT_ONLY_ENV));+ expect(withoutPort(entries, PUBLIC_PORT)).toEqual([]);+ });++ test('the edge listens on its scheme default whatever the public port is', () => {+ const environment = expandedEnvironment('edge', PORT_ONLY_ENV);+ expect(environment.FLUXER_EDGE_SITE_ADDRESS).toBe(`http://${DOMAIN}`);+ expect(publishedPorts(PORT_ONLY_ENV)).not.toContain(PUBLIC_PORT);+ });++ test('the documented recipe publishes the port every public URL advertises', () => {+ expect(publishedPorts(DOCUMENTED_RECIPE_ENV)).toContain(PUBLIC_PORT);+ });+});++describe('a public origin carrying a port while FLUXER_PUBLIC_PORT stays standard', () => {+ beforeEach(() => {+ resetConfig();+ });++ afterEach(() => {+ resetConfig();+ vi.unstubAllEnvs();+ });++ test('the compose overrides all carry the origin port', () => {+ const environment = expandedEnvironment('api', ORIGIN_ONLY_ENV);+ const entries = publicUrlNames(environment).map((name): [string, string] => [name, environment[name]]);+ expect(withoutPort(entries, ORIGIN_PORT)).toEqual([]);+ });++ test('the loaded config either ports every public URL or refuses the split', async () => {+ const loaded = await loadApiConfig(ORIGIN_ONLY_ENV);+ if (loaded instanceof Error) {+ expect(loaded.message).not.toBe('');+ return;+ }+ expect(withoutPort(browserFacingUrls(loaded), ORIGIN_PORT)).toEqual([]);+ });++ test('the port-only recipe ports every public URL the loaded config exposes', async () => {+ const loaded = await loadApiConfig(PORT_ONLY_ENV);+ if (loaded instanceof Error) {+ throw loaded;+ }+ expect(withoutPort(browserFacingUrls(loaded), PUBLIC_PORT)).toEqual([]);+ });+});