:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
chore(ingestion): pick e2e worker ports below the ephemeral range (#120991)
Paweł Ledwońae6e339e3756
diff --git a/nodejs/tests/helpers/service-process.ts b/nodejs/tests/helpers/service-process.tsindex 12842fb3e9a0..dd829e17c8c7 100644--- a/nodejs/tests/helpers/service-process.ts+++ b/nodejs/tests/helpers/service-process.ts@@ -26,6 +26,27 @@ export async function getFreePort(): Promise<number> {})}+// A worker needs two adjacent free ports: HTTP, and HTTP + grpcPortOffset for the stream.+export async function getFreeWorkerPort(grpcPortOffset: number): Promise<number> {+ for (let attempt = 0; attempt < 20; attempt++) {+ // Below the ephemeral range (Linux 32768+, macOS 49152+), so outbound connections+ // cannot take either port between this check and the worker's bind.+ const port = 20_000 + Math.floor(Math.random() * 12_000)+ if ((await isPortFree(port)) && (await isPortFree(port + grpcPortOffset))) {+ return port+ }+ }+ throw new Error('could not find adjacent free ports for a worker')+}++export function isPortFree(port: number): Promise<boolean> {+ return new Promise((resolve) => {+ const server = net.createServer()+ server.once('error', () => resolve(false))+ server.listen(port, '127.0.0.1', () => server.close(() => resolve(true)))+ })+}+export class ServiceProcess {private readonly child: ChildProcessWithoutNullStreamsprivate readonly exited: Promise<ProcessExit>diff --git a/nodejs/tests/service-e2e/personhog-kafka-parity.serial.test.ts b/nodejs/tests/service-e2e/personhog-kafka-parity.serial.test.tsindex dbfeba4293f4..43c7b5713bf1 100644--- a/nodejs/tests/service-e2e/personhog-kafka-parity.serial.test.ts+++ b/nodejs/tests/service-e2e/personhog-kafka-parity.serial.test.ts@@ -11,7 +11,6 @@// not compared, the publisher does not project it.import { ConsumerGlobalConfig, KafkaConsumer, LibrdKafkaError, Message, Metadata, WatermarkOffsets } from 'node-rdkafka'import { existsSync, mkdtempSync, readFileSync, rmSync } from 'node:fs'-import net from 'node:net'import { tmpdir } from 'node:os'import path from 'node:path'import { Pool } from 'pg'@@ -45,7 +44,7 @@ import { Hub } from '~/types'import { waitForExpect } from '../helpers/expectations'import { DEFAULT_TEAM, EventBuilder, createKafkaMessage } from '../helpers/ingestion-e2e'import { TEST_KAFKA_TOPICS, createKafkaTestTopicName, deleteKafkaTopics, ensureKafkaTopics } from '../helpers/kafka'-import { ServiceProcess, getFreePort } from '../helpers/service-process'+import { ServiceProcess, getFreePort, getFreeWorkerPort } from '../helpers/service-process'import { createOrganization, createTeam } from '../helpers/sql'jest.setTimeout(300_000)@@ -647,7 +646,7 @@ async function startNodeIngestionApiWorker(extraEnv: Record<string, string>,register: Register): Promise<string> {- const port = await getFreeWorkerPort()+ const port = await getFreeWorkerPort(GRPC_PORT_OFFSET)const service = new ServiceProcess('node-worker', 'pnpm', ['exec', 'tsx', 'src/index.ts'], {cwd: NODEJS_ROOT,env: {@@ -746,25 +745,6 @@ function dependencyEnv(hub: Hub): NodeJS.ProcessEnv {}}-// A worker needs two adjacent free ports: HTTP, and HTTP + GRPC_PORT_OFFSET for the stream.-async function getFreeWorkerPort(): Promise<number> {- for (let attempt = 0; attempt < 20; attempt++) {- const port = await getFreePort()- if (await isPortFree(port + GRPC_PORT_OFFSET)) {- return port- }- }- throw new Error('could not find adjacent free ports for a worker')-}--function isPortFree(port: number): Promise<boolean> {- return new Promise((resolve) => {- const server = net.createServer()- server.once('error', () => resolve(false))- server.listen(port, '127.0.0.1', () => server.close(() => resolve(true)))- })-}-function foldFeed(distinctIdRows: Buffer[], mergeEvents: Buffer[], teamId: number): FeedState {const latest = new Map<string, DistinctIdRow>()const persons = new Set<string>()diff --git a/nodejs/tests/service-e2e/rust-ingestion-consumer.serial.test.ts b/nodejs/tests/service-e2e/rust-ingestion-consumer.serial.test.tsindex 430189a2df76..a8705e18800c 100644--- a/nodejs/tests/service-e2e/rust-ingestion-consumer.serial.test.ts+++ b/nodejs/tests/service-e2e/rust-ingestion-consumer.serial.test.ts@@ -1,4 +1,3 @@-import net from 'net'import {ConsumerGlobalConfig,KafkaConsumer,@@ -20,7 +19,7 @@ import { parseJSON } from '~/common/utils/json-parse'import { UUIDT } from '~/common/utils/utils'import { waitForExpect } from '~/tests/helpers/expectations'import { TEST_KAFKA_TOPICS, createKafkaTestTopicName, ensureKafkaTopics } from '~/tests/helpers/kafka'-import { ServiceProcess, getFreePort } from '~/tests/helpers/service-process'+import { ServiceProcess, getFreePort, getFreeWorkerPort, isPortFree } from '~/tests/helpers/service-process'import { Clickhouse } from '../helpers/clickhouse'import {@@ -415,7 +414,7 @@ async function startNodeIngestionApiWorker(name: string, port?: number): Promise// worker binds it. Unpinned starts retry with a fresh pair; pinned restarts wait// for the previous holder to let go (see restartNodeIngestionApiWorker).for (let attempt = 1; ; attempt++) {- const workerPort = port ?? (await getFreeWorkerPort())+ const workerPort = port ?? (await getFreeWorkerPort(GRPC_PORT_OFFSET))try {return await spawnNodeIngestionApiWorker(name, workerPort)} catch (error) {@@ -455,25 +454,6 @@ async function spawnNodeIngestionApiWorker(name: string, workerPort: number): Prreturn { service, url }}-// A worker needs two adjacent free ports: HTTP, and HTTP + GRPC_PORT_OFFSET for the stream.-async function getFreeWorkerPort(): Promise<number> {- for (let attempt = 0; attempt < 20; attempt++) {- const port = await getFreePort()- if (await isPortFree(port + GRPC_PORT_OFFSET)) {- return port- }- }- throw new Error('could not find adjacent free ports for a worker')-}--function isPortFree(port: number): Promise<boolean> {- return new Promise((resolve) => {- const server = net.createServer()- server.once('error', () => resolve(false))- server.listen(port, '127.0.0.1', () => server.close(() => resolve(true)))- })-}-// Poll until every port can be bound. A stopped worker's node process releases its// listeners only after its graceful shutdown (gRPC drain), which outlives the pnpm// wrapper that ServiceProcess.stop() waits on.