Skip to content

Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.

19.0 KB
Raw Blame
/**
 * Copyright (c) Microsoft Corporation.
 *
 * Licensed under the Apache License, Version 2.0 (the 'License');
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an 'AS IS' BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

import { eventsHelper } from '@utils/eventsHelper';
import { findMatchingHttpCredentials } from '../browserContext';
import { parseRawCookie } from '../cookieStore';
import * as network from '../network';
import * as bidi from './third_party/bidiProtocol';

import type { RegisteredListener } from '@utils/eventsHelper';
import type * as frames from '../frames';
import type { Page } from '../page';
import type * as types from '../types';
import type { HttpCredentials } from '@protocol/structs';
import type { BidiSession } from './bidiConnection';
import type { BidiPage } from './bidiPage';

const REQUEST_BODY_HEADERS = new Set(['content-encoding', 'content-language', 'content-location', 'content-type']);

export class BidiNetworkManager {
  private readonly _session: BidiSession;
  private readonly _requests: Map<string, BidiRequest>;
  private readonly _page: Page;
  private readonly _eventListeners: RegisteredListener[];
  private _userRequestInterceptionEnabled: boolean = false;
  private _protocolRequestInterceptionEnabled: boolean = false;
  private _credentials: HttpCredentials[] | undefined;
  private _proxyCredentials: HttpCredentials | undefined;
  private _attemptedAuthentications = new Set<string>();
  private _intercepId: bidi.Network.Intercept | undefined;

  constructor(bidiSession: BidiSession, page: Page) {
    this._session = bidiSession;
    this._requests = new Map();
    this._page = page;
    this._eventListeners = [
      eventsHelper.addEventListener(bidiSession, 'network.beforeRequestSent', this._onBeforeRequestSent.bind(this)),
      eventsHelper.addEventListener(bidiSession, 'network.responseStarted', this._onResponseStarted.bind(this)),
      eventsHelper.addEventListener(bidiSession, 'network.responseCompleted', this._onResponseCompleted.bind(this)),
      eventsHelper.addEventListener(bidiSession, 'network.fetchError', this._onFetchError.bind(this)),
      eventsHelper.addEventListener(bidiSession, 'network.authRequired', this._onAuthRequired.bind(this)),
    ];
  }

  dispose() {
    eventsHelper.removeEventListeners(this._eventListeners);
  }

  private _onBeforeRequestSent(param: bidi.Network.BeforeRequestSentParameters) {
    if (param.request.url.startsWith('data:'))
      return;
    const redirectedFrom = param.redirectCount ? (this._requests.get(param.request.request) || null) : null;
    const frame = redirectedFrom ? redirectedFrom.request.frame() : (param.context ? this._page.frameManager.frame(param.context) : null);
    if (!frame)
      return;
    if (redirectedFrom)
      this._deleteRequest(redirectedFrom._id);

    // handle CORS preflight requests
    if (param.request.method === 'OPTIONS') {
      // TODO: we should detect preflight requests by looking at param.initiator.type, but the Bidi spec for
      // the initiator type is incomplete and browser implementations are inconsistent, so we check for an
      // Access-Control-Request-Method header instead. See https://github.com/w3c/webdriver-bidi/issues/698.
      const requestHeaders = Object.fromEntries(param.request.headers.map(h => [h.name.toLowerCase(), bidiBytesValueToString(h.value)]));
      if (param.initiator?.type === 'preflight' || requestHeaders['access-control-request-method']) {
        if (param.intercepts) {
          // If interception is enabled, we accept all CORS options, assuming that this was intended when setting the route.
          const responseHeaders: types.HeadersArray = [
            { name: 'Access-Control-Allow-Origin', value: requestHeaders['origin'] || '*' },
            { name: 'Access-Control-Allow-Methods', value: requestHeaders['access-control-request-method'] },
            { name: 'Access-Control-Allow-Credentials', value: 'true' }
          ];
          if (requestHeaders['access-control-request-headers'])
            responseHeaders.push({ name: 'Access-Control-Allow-Headers', value: requestHeaders['access-control-request-headers'] });
          this._session.sendMayFail('network.provideResponse', {
            request: param.request.request,
            statusCode: 204,
            headers: toBidiHeaders(responseHeaders),
          });
        }
        return;
      }
    }

    let route;
    let headersOverride: types.HeadersArray | undefined;
    if (param.intercepts) {
      // We do not support intercepting redirects.
      if (redirectedFrom) {
        let params = {};
        if (redirectedFrom._originalRequestRoute?._alreadyContinuedHeaders) {
          const originalHeaders = fromBidiHeaders(param.request.headers);
          headersOverride = network.applyHeadersOverrides(originalHeaders, redirectedFrom._originalRequestRoute._alreadyContinuedHeaders);
          // If the redirect turned a POST into a GET request, remove the request body headers,
          // corresponding to step 12 of https://fetch.spec.whatwg.org/#http-redirect-fetch.
          if (redirectedFrom.request.method() === 'POST' && param.request.method === 'GET')
            headersOverride = headersOverride.filter(({ name }) => !REQUEST_BODY_HEADERS.has(name.toLowerCase()));
          params = toBidiRequestHeaders(headersOverride);
        }

        this._session.sendMayFail('network.continueRequest', {
          request: param.request.request,
          ...params,
        });
      } else {
        route = new BidiRouteImpl(this._session, param.request.request);
      }
    }
    const request = new BidiRequest(frame, redirectedFrom, param, route, headersOverride);
    this._requests.set(request._id, request);
    this._page.frameManager.requestStarted(request.request, route);
  }

  private _onResponseStarted(params: bidi.Network.ResponseStartedParameters) {
    const request = this._requests.get(params.request.request);
    if (!request)
      return;
    const getResponseBody = async () => {
      const { bytes } = await this._session.send('network.getData', { request: params.request.request, dataType: bidi.Network.DataType.Response });
      const encoding = bytes.type === 'base64' ? 'base64' : 'utf8';
      return Buffer.from(bytes.value, encoding);
    };
    const timings = params.request.timings;
    const startTime = timings.requestTime;
    function relativeToStart(time: number): number {
      if (!time)
        return -1;
      return (time - startTime);
    }
    const timing: network.ResourceTiming = {
      startTime: startTime,
      requestStart: relativeToStart(timings.requestStart),
      responseStart: relativeToStart(timings.responseStart),
      domainLookupStart: relativeToStart(timings.dnsStart),
      domainLookupEnd: relativeToStart(timings.dnsEnd),
      connectStart: relativeToStart(timings.connectStart),
      secureConnectionStart: relativeToStart(timings.tlsStart),
      connectEnd: relativeToStart(timings.connectEnd),
    };
    const response = new network.Response(request.request, params.response.status, params.response.statusText, fromBidiHeaders(params.response.headers), timing, getResponseBody, false);
    response._serverAddrFinished();
    response._securityDetailsFinished();
    response._setHttpVersion(params.response.protocol);
    // "raw" headers are the same as "provisional" headers in Bidi.
    response.setRawResponseHeaders(null);
    response.setResponseHeadersSize(params.response.headersSize);
    this._page.frameManager.requestReceivedResponse(response);
  }

  private _onResponseCompleted(params: bidi.Network.ResponseCompletedParameters) {
    const request = this._requests.get(params.request.request);
    if (!request)
      return;
    const response = request.request._existingResponse()!;
    // TODO: body size is the encoded size
    response.setTransferSize(params.response.bodySize);
    response.setEncodedBodySize(params.response.bodySize);

    // Keep redirected requests in the map for future reference as redirectedFrom.
    const isRedirected = response.status() >= 300 && response.status() <= 399;
    const responseEndTime = params.request.timings.responseEnd - response.timing().startTime;
    if (isRedirected) {
      response._requestFinished(responseEndTime);
    } else {
      this._deleteRequest(request._id);
      response._requestFinished(responseEndTime);
    }
    this._page.frameManager.reportRequestFinished(request.request, response);

  }

  private _onFetchError(params: bidi.Network.FetchErrorParameters) {
    const request = this._requests.get(params.request.request);
    if (!request)
      return;
    this._deleteRequest(request._id);
    const response = request.request._existingResponse();
    if (response) {
      response.setTransferSize(null);
      response.setEncodedBodySize(null);
      response._requestFinished(-1);
    }
    request.request._setFailureText(params.errorText);
    // TODO: support canceled flag
    this._page.frameManager.requestFailed(request.request, params.errorText === 'NS_BINDING_ABORTED');
  }

  private _onAuthRequired(params: bidi.Network.AuthRequiredParameters) {
    // Don't look at params.response.authChallenges: Chromium does not report them.
    // Proxy credentials only answer the proxy's challenge, never the website's.
    // Chromium does not report the status either, so the challenge may come from the proxy or the website.
    const siteCredentials = findMatchingHttpCredentials(this._credentials, params.request.url);
    const status = params.response.status;
    const isProxy = status === 407 || (status !== 401 && !siteCredentials);
    const credentials = isProxy ? this._proxyCredentials : siteCredentials;
    const attemptKey = authAttemptKey(params.request.request, isProxy);
    if (credentials) {
      if (this._attemptedAuthentications.has(attemptKey)) {
        this._session.sendMayFail('network.continueWithAuth', {
          request: params.request.request,
          action: 'cancel',
        });
      } else {
        this._attemptedAuthentications.add(attemptKey);
        this._session.sendMayFail('network.continueWithAuth', {
          request: params.request.request,
          action: 'provideCredentials',
          credentials: {
            type: 'password',
            username: credentials.username,
            password: credentials.password,
          }
        });
      }
    } else {
      this._session.sendMayFail('network.continueWithAuth', {
        request: params.request.request,
        action: 'cancel',
      });
    }
  }

  _deleteRequest(requestId: string) {
    this._requests.delete(requestId);
    this._attemptedAuthentications.delete(authAttemptKey(requestId, true));
    this._attemptedAuthentications.delete(authAttemptKey(requestId, false));
  }

  async setRequestInterception(value: boolean) {
    this._userRequestInterceptionEnabled = value;
    await this._updateProtocolRequestInterception();
  }

  async setCredentials(credentials: HttpCredentials[] | undefined, proxyCredentials: HttpCredentials | undefined) {
    this._credentials = credentials;
    this._proxyCredentials = proxyCredentials;
    await this._updateProtocolRequestInterception();
  }

  async _updateProtocolRequestInterception(initial?: boolean) {
    const enabled = this._userRequestInterceptionEnabled || !!this._credentials?.length || !!this._proxyCredentials;
    if (enabled === this._protocolRequestInterceptionEnabled)
      return;
    this._protocolRequestInterceptionEnabled = enabled;
    if (initial && !enabled)
      return;
    const contexts: [string] = [(this._page.delegate as BidiPage)._session.sessionId];
    const cachePromise = this._session.send('network.setCacheBehavior', { cacheBehavior: enabled ? 'bypass' : 'default', contexts });
    let interceptPromise = Promise.resolve<any>(undefined);
    if (enabled) {
      interceptPromise = this._session.send('network.addIntercept', {
        phases: [bidi.Network.InterceptPhase.BeforeRequestSent],
        contexts,
      }).then(r => {
        this._intercepId = r.intercept;
      });
    } else if (this._intercepId) {
      interceptPromise = this._session.send('network.removeIntercept', { intercept: this._intercepId });
      this._intercepId = undefined;
    }
    await Promise.all([cachePromise, interceptPromise]);
  }
}


class BidiRequest {
  readonly request: network.Request;
  readonly _id: string;
  private _redirectedTo: BidiRequest | undefined;
  // Only first request in the chain can be intercepted, so this will
  // store the first and only Route in the chain (if any).
  _originalRequestRoute: BidiRouteImpl | undefined;

  constructor(
    frame: frames.Frame,
    redirectedFrom: BidiRequest | null,
    payload: bidi.Network.BeforeRequestSentParameters,
    route: BidiRouteImpl | undefined,
    headersOverride: types.HeadersArray | undefined
  ) {
    this._id = payload.request.request;
    if (redirectedFrom)
      redirectedFrom._redirectedTo = this;
    // TODO: missing in the spec?
    const postDataBuffer = null;
    this.request = new network.Request(frame._page.browserContext, frame, null, redirectedFrom ? redirectedFrom.request : null, payload.navigation ?? undefined,
        payload.request.url, resourceTypeFromBidi(payload.request.destination, payload.request.initiatorType, payload.initiator?.type), payload.request.method,
        postDataBuffer, headersOverride || fromBidiHeaders(payload.request.headers), payload.timestamp);
    // "raw" headers are the same as "provisional" headers in Bidi.
    this.request.setRawRequestHeaders(null);
    this.request._setBodySize(payload.request.bodySize || 0);
    this._originalRequestRoute = route ?? redirectedFrom?._originalRequestRoute;
    route?._setRequest(this.request);
  }

  _finalRequest(): BidiRequest {
    let request: BidiRequest = this;
    while (request._redirectedTo)
      request = request._redirectedTo;
    return request;
  }
}

class BidiRouteImpl implements network.RouteDelegate {
  private _requestId: bidi.Network.Request;
  private _session: BidiSession;
  private _request!: network.Request;
  _alreadyContinuedHeaders: types.HeadersArray | undefined;

  constructor(session: BidiSession, requestId: bidi.Network.Request) {
    this._session = session;
    this._requestId = requestId;
  }

  _setRequest(request: network.Request) {
    this._request = request;
  }

  async continue(overrides: types.NormalizedContinueOverrides) {
    // Firefox does not update content-length header.
    let headers = overrides.headers || this._request.headers();
    if (overrides.postData && headers) {
      headers = headers.map(header => {
        if (header.name.toLowerCase() === 'content-length')
          return { name: header.name, value: overrides.postData!.byteLength.toString() };
        return header;
      });
    }
    this._alreadyContinuedHeaders = headers;
    await this._session.sendMayFail('network.continueRequest', {
      request: this._requestId,
      url: overrides.url,
      method: overrides.method,
      ...toBidiRequestHeaders(this._alreadyContinuedHeaders),
      body: overrides.postData ? { type: 'base64', value: Buffer.from(overrides.postData).toString('base64') } : undefined,
    });
  }

  async fulfill(response: types.NormalizedFulfillResponse) {
    const base64body = response.isBase64 ? response.body : Buffer.from(response.body).toString('base64');
    const headers = response.headers.filter(h => h.name.toLowerCase() !== 'content-encoding');
    await this._session.sendMayFail('network.provideResponse', {
      request: this._requestId,
      statusCode: response.status,
      reasonPhrase: network.statusText(response.status),
      ...toBidiResponseHeaders(headers),
      body: { type: 'base64', value: base64body },
    });
  }

  async abort(errorCode: string) {
    await this._session.sendMayFail('network.failRequest', {
      request: this._requestId
    });
  }
}

function fromBidiHeaders(bidiHeaders: bidi.Network.Header[]): types.HeadersArray {
  const result: types.HeadersArray = [];
  for (const { name, value } of bidiHeaders)
    result.push({ name, value: bidiBytesValueToString(value) });
  return result;
}

function toBidiRequestHeaders(allHeaders: types.HeadersArray): { headers: bidi.Network.Header[] } {
  const bidiHeaders = toBidiHeaders(allHeaders);
  return { headers: bidiHeaders };
}

function toBidiResponseHeaders(headers: types.HeadersArray): { cookies: bidi.Network.SetCookieHeader[], headers: bidi.Network.Header[] } {
  const setCookieHeaders = headers.filter(h => h.name.toLowerCase() === 'set-cookie');
  const otherHeaders = headers.filter(h => h.name.toLowerCase() !== 'set-cookie');
  const rawCookies = setCookieHeaders.map(h => parseRawCookie(h.value));
  const cookies: bidi.Network.SetCookieHeader[] = rawCookies.filter(Boolean).map(c => {
    return {
      ...c!,
      value: { type: 'string', value: c!.value },
      sameSite: toBidiSameSite(c!.sameSite),
    };
  });
  return { cookies, headers: toBidiHeaders(otherHeaders) };
}

function toBidiHeaders(headers: types.HeadersArray): bidi.Network.Header[] {
  return headers.map(({ name, value }) => ({ name, value: { type: 'string', value } }));
}

export function bidiBytesValueToString(value: bidi.Network.BytesValue): string {
  if (value.type === 'string')
    return value.value;
  if (value.type === 'base64')
    return Buffer.from(value.value, 'base64').toString('binary');
  return 'unknown value type: ' + (value as any).type;

}

function toBidiSameSite(sameSite?: 'Strict' | 'Lax' | 'None'): bidi.Network.SameSite | undefined {
  if (!sameSite)
    return undefined;
  if (sameSite === 'Strict')
    return bidi.Network.SameSite.Strict;
  if (sameSite === 'Lax')
    return bidi.Network.SameSite.Lax;
  return bidi.Network.SameSite.None;
}

function resourceTypeFromBidi(requestDestination: string, requestInitiatorType: string | null, eventInitiatorType: string | undefined): network.ResourceType {
  switch (requestDestination) {
    case 'audio': return 'media';
    case 'audioworklet': return 'script';
    case 'document': return 'document';
    case 'font': return 'font';
    case 'frame': return 'document';
    case 'iframe': return 'document';
    case 'image': return 'image';
    case 'object': return 'other';
    case 'paintworklet': return 'script';
    case 'script': return 'script';
    case 'serviceworker': return 'script';
    case 'sharedworker': return 'script';
    case 'style': return 'stylesheet';
    case 'track': return 'texttrack';
    case 'video': return 'media';
    case 'worker': return 'script';
    case '':
      switch (requestInitiatorType) {
        case 'fetch': return 'fetch';
        case 'font': return 'font';
        case 'xmlhttprequest': return 'xhr';
        case null: return eventInitiatorType === 'script' ? 'xhr' : 'document';
        default: return 'other';
      }
    default: return 'other';
  }
}

function authAttemptKey(requestId: string, isProxy: boolean) {
  return `${requestId}:${isProxy ? 'proxy' : 'server'}`;
}