Skip to content

Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.

chore(net): use native happy eyeballs instead of the manual implementation (#42240)

Pavel Feldmandffe62020f67
diff --git a/packages/playwright-core/src/server/fetch.ts b/packages/playwright-core/src/server/fetch.ts
index 27a520fea4..1a9718ef38 100644
--- a/packages/playwright-core/src/server/fetch.ts
+++ b/packages/playwright-core/src/server/fetch.ts
@@ -21,12 +21,11 @@ import { TLSSocket } from 'tls';
import * as zlib from 'zlib';
import { createGuid } from '@utils/crypto';
-import { httpHappyEyeballsAgent, httpsHappyEyeballsAgent, timingForSocket } from '@utils/happyEyeballs';
import { assert } from '@isomorphic/assert';
import { constructURLBasedOnBaseURL } from '@isomorphic/urlMatch';
import { eventsHelper } from '@utils/eventsHelper';
import { monotonicTime } from '@isomorphic/time';
-import { createProxyAgent } from '@utils/network';
+import { createProxyAgent, dualStackLookup, flattenAggregateError } from '@utils/network';
import { getUserAgent } from './userAgent';
import { BrowserContext, findMatchingHttpCredentials, verifyClientCertificates } from './browserContext';
import { Cookie, CookieStore, domainMatches, parseRawCookie } from './cookieStore';
@@ -342,9 +341,10 @@ export abstract class APIRequestContext extends SdkObject {
const resultPromise = new Promise<SendRequestResult>((fulfill, reject) => {
const requestConstructor: ((url: URL, options: http.RequestOptions, callback?: (res: http.IncomingMessage) => void) => http.ClientRequest)
= (url.protocol === 'https:' ? https : http).request;
- // If we have a proxy agent already, do not override it.
- const agent = options.agent || (url.protocol === 'https:' ? httpsHappyEyeballsAgent : httpHappyEyeballsAgent);
- const requestOptions = { ...options, agent };
+ // Without an explicit proxy agent, the default global agent is used, which
+ // has keep-alive enabled and connects with Happy Eyeballs (autoSelectFamily).
+ const requestOptions = { ...options };
+ requestOptions.lookup = options.__testHookLookup ? lookupWithTestHook(options.__testHookLookup) : dualStackLookup;
const startAt = monotonicTime();
const startAtWallTime = Date.now();
@@ -366,7 +366,7 @@ export abstract class APIRequestContext extends SdkObject {
// https://github.com/nodejs/undici/blob/01a912e49a50c48009ed2639d2a457a6ec26752a/lib/dispatcher/client-h1.js#L735
if (responseReceived && isNetworkConnectionError(error))
return;
- reject(error);
+ reject(flattenAggregateError(error));
};
const request = requestConstructor(url, requestOptions as any, async response => {
@@ -614,15 +614,13 @@ export abstract class APIRequestContext extends SdkObject {
return;
}
- // happy eyeballs don't emit lookup and connect events, so we use our custom ones
- const happyEyeBallsTimings = timingForSocket(socket);
- dnsLookupAt = happyEyeBallsTimings.dnsLookupAt;
- tcpConnectionAt = happyEyeBallsTimings.tcpConnectionAt;
-
- // non-happy-eyeballs sockets
listeners.push(
eventsHelper.addEventListener(socket, 'lookup', () => { dnsLookupAt = monotonicTime(); }),
- eventsHelper.addEventListener(socket, 'connect', () => { tcpConnectionAt = monotonicTime(); }),
+ eventsHelper.addEventListener(socket, 'connect', () => {
+ tcpConnectionAt = monotonicTime();
+ serverIPAddress = socket.remoteAddress;
+ serverPort = socket.remotePort;
+ }),
eventsHelper.addEventListener(socket, 'secureConnect', () => {
tlsHandshakeAt = monotonicTime();
captureSecurityDetails(socket);
@@ -865,3 +863,21 @@ function setBasicAuthorizationHeader(headers: { [name: string]: string }, creden
const encoded = Buffer.from(`${username || ''}:${password || ''}`).toString('base64');
setHeader(headers, 'authorization', `Basic ${encoded}`);
}
+
+function lookupWithTestHook(testHookLookup: (hostname: string) => LookupAddress[]): net.LookupFunction {
+ return (hostname, options, callback) => {
+ setImmediate(() => {
+ let addresses: LookupAddress[];
+ try {
+ addresses = testHookLookup(hostname);
+ } catch (error) {
+ callback(error as NodeJS.ErrnoException, '');
+ return;
+ }
+ if (options.all)
+ callback(null, addresses);
+ else
+ callback(null, addresses[0].address, addresses[0].family);
+ });
+ };
+}
diff --git a/packages/playwright-core/src/server/socksClientCertificatesInterceptor.ts b/packages/playwright-core/src/server/socksClientCertificatesInterceptor.ts
index fe39c303c8..448ebc238f 100644
--- a/packages/playwright-core/src/server/socksClientCertificatesInterceptor.ts
+++ b/packages/playwright-core/src/server/socksClientCertificatesInterceptor.ts
@@ -23,11 +23,10 @@ import tls from 'tls';
import { getProxyForUrl } from 'proxy-from-env';
import { SocksProxy } from '@utils/socksProxy';
import { debugLogger } from '@utils/debugLogger';
-import { createSocket } from '@utils/happyEyeballs';
import { escapeHTML } from '@isomorphic/stringUtils';
import { generateSelfSignedCertificate } from '@utils/crypto';
import { rewriteErrorMessage } from '@utils/stackTrace';
-import { createProxyAgent } from '@utils/network';
+import { createProxyAgent, createSocket } from '@utils/network';
import { verifyClientCertificates } from './browserContext';
import type * as types from './types';
import type { SocksSocketClosedPayload, SocksSocketDataPayload, SocksSocketRequestedPayload } from '@utils/socksProxy';
diff --git a/packages/playwright-core/src/server/transport.ts b/packages/playwright-core/src/server/transport.ts
index 36a6eb60cb..d333834efc 100644
--- a/packages/playwright-core/src/server/transport.ts
+++ b/packages/playwright-core/src/server/transport.ts
@@ -16,7 +16,7 @@
*/
import ws from 'ws';
-import { httpHappyEyeballsAgent, httpsHappyEyeballsAgent } from '@utils/happyEyeballs';
+import { dualStackLookup, flattenAggregateError } from '@utils/network';
import { makeWaitForNextTask } from '@utils/task';
import type { WebSocket } from 'ws';
import type { Progress } from './progress';
@@ -89,8 +89,9 @@ export class WebSocketTransport implements ConnectionTransport {
fulfill({});
});
transport._ws.on('error', event => {
- progress?.log(`<ws connect error> ${logUrl} ${event.message}`);
- reject(new Error('WebSocket error: ' + event.message));
+ const message = flattenAggregateError(event).message;
+ progress?.log(`<ws connect error> ${logUrl} ${message}`);
+ reject(new Error('WebSocket error: ' + message));
transport._ws.close();
});
transport._ws.on('unexpected-response', (request: ClientRequest, response: IncomingMessage) => {
@@ -137,7 +138,7 @@ export class WebSocketTransport implements ConnectionTransport {
maxPayload: 256 * 1024 * 1024, // 256Mb,
headers: options.headers,
followRedirects: options.followRedirects,
- agent: (/^(https|wss):\/\//.test(url)) ? httpsHappyEyeballsAgent : httpHappyEyeballsAgent,
+ lookup: dualStackLookup,
perMessageDeflate,
});
this._ws.on('upgrade', response => {
diff --git a/packages/playwright-core/src/server/webkit/webview/wvBrowser.ts b/packages/playwright-core/src/server/webkit/webview/wvBrowser.ts
index 0e4f37808f..b2664ad689 100644
--- a/packages/playwright-core/src/server/webkit/webview/wvBrowser.ts
+++ b/packages/playwright-core/src/server/webkit/webview/wvBrowser.ts
@@ -21,7 +21,7 @@ import path from 'path';
import ws from 'ws';
import { debugLogger, RecentLogsCollector } from '@utils/debugLogger';
import { removeFolders } from '@utils/fileUtils';
-import { httpHappyEyeballsAgent, httpsHappyEyeballsAgent } from '@utils/happyEyeballs';
+import { dualStackLookup } from '@utils/network';
import { headersArrayToObject } from '@isomorphic/headers';
import { Browser } from '../../browser';
import { helper } from '../../helper';
@@ -96,7 +96,7 @@ class DeferredWebSocketTransport implements ConnectOverCDPTransport {
maxPayload: 256 * 1024 * 1024,
headers: this._headers,
followRedirects: true,
- agent: (/^(https|wss):\/\//.test(url)) ? httpsHappyEyeballsAgent : httpHappyEyeballsAgent,
+ lookup: dualStackLookup,
perMessageDeflate,
allowSynchronousEvents: false,
});
diff --git a/packages/utils/happyEyeballs.ts b/packages/utils/happyEyeballs.ts
deleted file mode 100644
index 7bd585d069..0000000000
--- a/packages/utils/happyEyeballs.ts
+++ /dev/null
@@ -1,217 +0,0 @@
-/**
- * Copyright (c) Microsoft Corporation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import dns from 'dns';
-import http from 'http';
-import https from 'https';
-import net from 'net';
-import stream from 'stream';
-import tls from 'tls';
-
-import { assert } from '@isomorphic/assert';
-import { ManualPromise } from '@isomorphic/manualPromise';
-import { monotonicTime } from '@isomorphic/time';
-
-// Implementation(partial) of Happy Eyeballs 2 algorithm described in
-// https://www.rfc-editor.org/rfc/rfc8305
-
-// Same as in Chromium (https://source.chromium.org/chromium/chromium/src/+/5666ff4f5077a7e2f72902f3a95f5d553ea0d88d:net/socket/transport_connect_job.cc;l=102)
-const connectionAttemptDelayMs = 300;
-
-const kDNSLookupAt = Symbol('kDNSLookupAt');
-const kTCPConnectionAt = Symbol('kTCPConnectionAt');
-
-class HttpHappyEyeballsAgent extends http.Agent {
- override createConnection(options: http.ClientRequestArgs, oncreate?: (err: Error | null, socket: stream.Duplex) => void): stream.Duplex | undefined {
- // There is no ambiguity in case of IP address.
- if (net.isIP(clientRequestArgsToHostName(options)))
- return net.createConnection(options as net.NetConnectOpts);
- const cb = oncreate as unknown as ((err: Error | null, socket?: net.Socket) => void) | undefined;
- createConnectionAsync(options, cb, /* useTLS */ false).catch(err => cb?.(err));
- }
-}
-
-class HttpsHappyEyeballsAgent extends https.Agent {
- override createConnection(options: http.ClientRequestArgs, oncreate?: (err: Error | null, socket: stream.Duplex) => void): stream.Duplex | undefined {
- // There is no ambiguity in case of IP address.
- if (net.isIP(clientRequestArgsToHostName(options)))
- return tls.connect(options as tls.ConnectionOptions);
- const cb = oncreate as unknown as ((err: Error | null, socket?: tls.TLSSocket) => void) | undefined;
- createConnectionAsync(options, cb, /* useTLS */ true).catch(err => cb?.(err));
- }
-}
-
-// These options are aligned with the default Node.js globalAgent options.
-export const httpsHappyEyeballsAgent = new HttpsHappyEyeballsAgent({ keepAlive: true });
-export const httpHappyEyeballsAgent = new HttpHappyEyeballsAgent({ keepAlive: true });
-
-export async function createSocket(host: string, port: number): Promise<net.Socket> {
- return new Promise((resolve, reject) => {
- if (net.isIP(host)) {
- const socket = net.createConnection({ host, port });
- socket.on('connect', () => resolve(socket));
- socket.on('error', error => reject(error));
- } else {
- createConnectionAsync({ host, port }, (err, socket) => {
- if (err)
- reject(err);
- if (socket)
- resolve(socket);
- }, /* useTLS */ false).catch(err => reject(err));
- }
- });
-}
-
-export async function createTLSSocket(options: tls.ConnectionOptions): Promise<tls.TLSSocket> {
- return new Promise((resolve, reject) => {
- assert(options.host, 'host is required');
- if (net.isIP(options.host)) {
- const socket = tls.connect(options);
- socket.on('secureConnect', () => resolve(socket));
- socket.on('error', error => reject(error));
- } else {
- createConnectionAsync(options, (err, socket) => {
- if (err)
- reject(err);
- if (socket) {
- socket.on('secureConnect', () => resolve(socket));
- socket.on('error', error => reject(error));
- }
- }, true).catch(err => reject(err));
- }
- });
-}
-
-export async function createConnectionAsync(
- options: http.ClientRequestArgs,
- oncreate: ((err: Error | null, socket?: tls.TLSSocket) => void) | undefined,
- useTLS: true
-): Promise<void>;
-
-export async function createConnectionAsync(
- options: http.ClientRequestArgs,
- oncreate: ((err: Error | null, socket?: net.Socket) => void) | undefined,
- useTLS: false
-): Promise<void>;
-
-export async function createConnectionAsync(
- options: http.ClientRequestArgs,
- oncreate: ((err: Error | null, socket?: any) => void) | undefined,
- useTLS: boolean
-): Promise<void> {
- const lookup = (options as any).__testHookLookup || lookupAddresses;
- const hostname = clientRequestArgsToHostName(options);
- const addresses = await lookup(hostname);
- const dnsLookupAt = monotonicTime();
- const sockets = new Set<net.Socket>();
- let firstError;
- let errorCount = 0;
- const handleError = (socket: net.Socket, err: Error) => {
- if (!sockets.delete(socket))
- return;
- ++errorCount;
- firstError ??= err;
- if (errorCount === addresses.length)
- oncreate?.(firstError);
- };
-
- const connected = new ManualPromise();
- for (const { address } of addresses) {
- const socket = useTLS ?
- tls.connect({
- ...(options as tls.ConnectionOptions),
- port: options.port as number,
- host: address,
- servername: hostname }) :
- net.createConnection({
- ...options,
- port: options.port as number,
- host: address });
-
- (socket as any)[kDNSLookupAt] = dnsLookupAt;
-
- // Each socket may fire only one of 'connect', 'timeout' or 'error' events.
- // None of these events are fired after socket.destroy() is called.
- socket.on('connect', () => {
- (socket as any)[kTCPConnectionAt] = monotonicTime();
-
- connected.resolve();
- oncreate?.(null, socket);
- // TODO: Cache the result?
- // Close other outstanding sockets.
- sockets.delete(socket);
- for (const s of sockets)
- s.destroy();
- sockets.clear();
- });
- socket.on('timeout', () => {
- // Timeout is not an error, so we have to manually close the socket.
- socket.destroy();
- handleError(socket, new Error('Connection timeout'));
- });
- socket.on('error', e => handleError(socket, e));
- sockets.add(socket);
- await Promise.race([
- connected,
- new Promise(f => setTimeout(f, connectionAttemptDelayMs))
- ]);
- if (connected.isDone())
- break;
- }
-}
-
-async function lookupAddresses(hostname: string): Promise<dns.LookupAddress[]> {
- // Use separate family lookups to avoid AI_ADDRCONFIG filtering. When family: 0 is used,
- // Node.js passes AI_ADDRCONFIG to getaddrinfo(), which on macOS can filter out addresses
- // for a family that has no non-loopback interface — e.g. returning only 127.0.0.1 for
- // "localhost" when IPv6 is not available on non-loopback interfaces, even though ::1 is
- // present in /etc/hosts. Separate family: 4 and family: 6 lookups do not pass AI_ADDRCONFIG.
- const [v4Result, v6Result] = await Promise.allSettled([
- dns.promises.lookup(hostname, { all: true, family: 4 }),
- dns.promises.lookup(hostname, { all: true, family: 6 }),
- ]);
- const v4Addresses = v4Result.status === 'fulfilled' ? v4Result.value : [];
- const v6Addresses = v6Result.status === 'fulfilled' ? v6Result.value : [];
- if (!v4Addresses.length && !v6Addresses.length) {
- if (v4Result.status === 'rejected')
- throw v4Result.reason;
- throw (v6Result as PromiseRejectedResult).reason;
- }
- const result: dns.LookupAddress[] = [];
- // Alternate IPv6 and IPv4 addresses per RFC 8305 (prefer IPv6 first).
- for (let i = 0; i < Math.max(v4Addresses.length, v6Addresses.length); i++) {
- if (v6Addresses[i])
- result.push(v6Addresses[i]);
- if (v4Addresses[i])
- result.push(v4Addresses[i]);
- }
- return result;
-}
-
-function clientRequestArgsToHostName(options: http.ClientRequestArgs): string {
- if (options.hostname)
- return options.hostname;
- if (options.host)
- return options.host;
- throw new Error('Either options.hostname or options.host must be provided');
-}
-
-export function timingForSocket(socket: net.Socket | tls.TLSSocket) {
- return {
- dnsLookupAt: (socket as any)[kDNSLookupAt] as number | undefined,
- tcpConnectionAt: (socket as any)[kTCPConnectionAt] as number | undefined,
- };
-}
diff --git a/packages/utils/network.ts b/packages/utils/network.ts
index 3b3790754c..9205f225fe 100644
--- a/packages/utils/network.ts
+++ b/packages/utils/network.ts
@@ -15,17 +15,17 @@
*/
+import dns from 'dns';
import http from 'http';
import http2 from 'http2';
import https from 'https';
+import net from 'net';
import { HttpsProxyAgent } from 'https-proxy-agent';
import { SocksProxyAgent } from 'socks-proxy-agent';
import { getProxyForUrl } from 'proxy-from-env';
import { ManualPromise } from '@isomorphic/manualPromise';
-import { httpHappyEyeballsAgent, httpsHappyEyeballsAgent } from './happyEyeballs';
-
-import type net from 'net';
+import { rewriteErrorMessage } from './stackTrace';
export type ProxySettings = {
server: string,
@@ -50,6 +50,7 @@ export function httpRequest(params: HTTPRequestParams, onResponse: (r: http.Inco
const options: https.RequestOptions = {
method: params.method || 'GET',
headers: params.headers,
+ lookup: dualStackLookup,
};
if (params.rejectUnauthorized !== undefined)
options.rejectUnauthorized = params.rejectUnauthorized;
@@ -69,8 +70,6 @@ export function httpRequest(params: HTTPRequestParams, onResponse: (r: http.Inco
}
}
- options.agent ??= url.protocol === 'https:' ? httpsHappyEyeballsAgent : httpHappyEyeballsAgent;
-
let cancelRequest: (e: Error | undefined) => void;
const requestCallback = (res: http.IncomingMessage) => {
const statusCode = res.statusCode || 0;
@@ -86,7 +85,7 @@ export function httpRequest(params: HTTPRequestParams, onResponse: (r: http.Inco
const request = url.protocol === 'https:' ?
https.request(url, options, requestCallback) :
http.request(url, options, requestCallback);
- request.on('error', onError);
+ request.on('error', error => onError(flattenAggregateError(error)));
if (params.socketTimeout !== undefined) {
request.setTimeout(params.socketTimeout, () => {
onError(new Error(`Request to ${params.url} timed out after ${params.socketTimeout}ms`));
@@ -156,6 +155,52 @@ export function createProxyAgent(proxy?: ProxySettings, forUrl?: URL) {
return new HttpsProxyAgent(proxyURL);
}
+// Node.js family-agnostic lookup passes AI_ADDRCONFIG to getaddrinfo(), which can filter out
+// addresses of a family that has no non-loopback interface, and "localhost" may miss addresses
+// of a family that is not listed in /etc/hosts — e.g. resolving to only 127.0.0.1 even though
+// ::1 is served. Separate family: 4 and family: 6 lookups do not have these problems. Native
+// Happy Eyeballs (autoSelectFamily) then races connection attempts across the families.
+export const dualStackLookup: net.LookupFunction = (hostname, options, callback) => {
+ const families = options.family === 4 || options.family === 6 ? [options.family] : [6, 4];
+ void Promise.allSettled(families.map(family => dns.promises.lookup(hostname, { all: true, family }))).then(results => {
+ const perFamily = results.map(result => result.status === 'fulfilled' ? result.value : []);
+ const addresses: dns.LookupAddress[] = [];
+ // Alternate IPv6 and IPv4 addresses per RFC 8305 (prefer IPv6 first).
+ for (let i = 0; i < Math.max(...perFamily.map(list => list.length)); i++) {
+ for (const list of perFamily) {
+ if (list[i])
+ addresses.push(list[i]);
+ }
+ }
+ if (!addresses.length) {
+ const firstError = results.map(result => result.status === 'rejected' ? result.reason : undefined).find(Boolean);
+ callback(firstError ?? new Error(`Cannot resolve address for ${hostname}`), '');
+ return;
+ }
+ if (options.all)
+ callback(null, addresses);
+ else
+ callback(null, addresses[0].address, addresses[0].family);
+ });
+};
+
+// When every raced connection attempt fails, Node.js reports an AggregateError with an
+// empty message and the individual failures in the `errors` property. Surface those instead.
+export function flattenAggregateError(error: Error): Error {
+ const errors = (error as any).errors as Error[] | undefined;
+ if (error.name === 'AggregateError' && !error.message && errors?.length)
+ return rewriteErrorMessage(error, errors.map(e => e.message).join('\n'));
+ return error;
+}
+
+export async function createSocket(host: string, port: number): Promise<net.Socket> {
+ return new Promise((resolve, reject) => {
+ const socket = net.createConnection({ host, port, lookup: dualStackLookup });
+ socket.on('connect', () => resolve(socket));
+ socket.on('error', error => reject(flattenAggregateError(error)));
+ });
+}
+
export function createHttpServer(requestListener?: (req: http.IncomingMessage, res: http.ServerResponse) => void): http.Server;
export function createHttpServer(options: http.ServerOptions, requestListener?: (req: http.IncomingMessage, res: http.ServerResponse) => void): http.Server;
export function createHttpServer(...args: any[]): http.Server {
diff --git a/packages/utils/socksProxy.ts b/packages/utils/socksProxy.ts
index db629b4a12..c1002dadb6 100644
--- a/packages/utils/socksProxy.ts
+++ b/packages/utils/socksProxy.ts
@@ -20,7 +20,7 @@ import net from 'net';
import { assert } from '@isomorphic/assert';
import { createGuid } from './crypto';
import { debugLogger } from './debugLogger';
-import { createSocket } from './happyEyeballs';
+import { createSocket } from './network';
import type { AddressInfo } from 'net';