Skip to content

armgit / rea

Public Imported from GitHub
0

Reverse engineer anything with agents, from app behavior down to native binaries.

17.8 KB
Raw Blame
import { isAbsolute } from "node:path";
import { CaptureFormatError } from "../CaptureFormatError.js";
import {
  CaptureRedaction,
  isCaptureCredentialHeaderBytes,
} from "../CaptureRedaction.js";
import { createHash } from "node:crypto";
import { readStableArtifact } from "../../../artifacts/readStableArtifact.js";
import {
  historicalCaptureFailure,
  withCaptureReaderCleanup,
} from "../CaptureFailures.js";
import type { ArtifactResourceScope } from "../../../artifacts/ArtifactResourceScope.js";
import {
  AnalysisInputError,
  AnalysisCapabilityUnavailableError,
} from "../../../domain/analysisErrorCore.js";
import { redactExplicitText } from "../../../domain/explicitSensitiveValues.js";
import {
  WEB_NETWORK_CAPTURE_LIMITS,
  type InspectWebNetworkCaptureInput,
  type WebNetworkCapture,
  type WebNetworkCaptureRecord,
} from "../../../domain/webNetworkCapture.js";
import type { JsonValue } from "../../../domain/jsonValue.js";
import { ProtobufBudget } from "./ProtobufWire.js";
import type { ProtobufSchema } from "./ProtobufSchema.js";
import {
  PROTOBUF_FRAMING,
  PROTOBUF_LIMITATIONS,
  decodeProtobufBody,
} from "./ProtobufBody.js";
import {
  prepareProtobufBody,
  type PreparedProtobufBody,
} from "./ProtobufFraming.js";
import type { ProtobufDecoding } from "../../../domain/protobufWire.js";

const object = (
  value: JsonValue | undefined,
): Readonly<Record<string, JsonValue>> | undefined =>
  value !== null && typeof value === "object" && !Array.isArray(value)
    ? value
    : undefined;

/** Exclude the original encoded representation as well as its decoding sidecar. */
const excludeBody = (
  record: WebNetworkCaptureRecord,
  pointer: string,
  reason: "explicit-sensitive-value" | "transport-credential",
) => {
  record.binary_fields = record.binary_fields.map((field) =>
    field.pointer !== pointer
      ? field
      : {
          pointer: field.pointer,
          representation: field.representation,
          state: "redacted",
          content_base64: null,
          bytes: null,
          sha256: null,
        },
  );
  const parts = pointer
    .split("/")
    .slice(1)
    .map((part) => part.replaceAll("~1", "/").replaceAll("~0", "~"));
  let parent: JsonValue | undefined = record.reported;
  for (const part of parts.slice(0, -1)) {
    if (
      parent === null ||
      typeof parent !== "object" ||
      !Object.hasOwn(parent, part)
    ) {
      parent = undefined;
      break;
    }
    parent = Array.isArray(parent) ? parent[Number(part)] : parent[part];
  }
  const key = parts.at(-1);
  if (
    parent !== null &&
    typeof parent === "object" &&
    key !== undefined &&
    Object.hasOwn(parent, key)
  )
    Object.defineProperty(parent, key, {
      value: null,
      writable: true,
      configurable: true,
      enumerable: true,
    });
  record.redactions.push({ pointer, reason });
};

/** A media type's case-insensitive essence; parameters do not select framing. */
const mediaTypeEssence = (value: string) =>
  value.split(";", 1)[0]!.trim().toLowerCase();

/** Format-aware owner of body metadata; callers never reinterpret producer JSON. */
const bodyMetadata = (
  record: WebNetworkCaptureRecord,
  pointer: string,
  format: string,
  binaryByPointer: ReadonlyMap<
    string,
    WebNetworkCaptureRecord["binary_fields"][number]
  >,
) => {
  const side = /^\/(request|response)\//.exec(pointer)?.[1];
  if (
    side === undefined ||
    ![
      "/request/postData/text",
      "/response/content/text",
      "/request/body",
      "/response/body",
      "/request/content",
      "/response/content",
    ].includes(pointer)
  )
    return undefined;
  const message = object(object(record.reported)?.[side]);
  if (message === undefined) return undefined;
  const headers = new Map<string, string | undefined>();
  let headerNamesKnown = Array.isArray(message.headers);
  if (Array.isArray(message.headers)) {
    for (const [index, item] of message.headers.entries()) {
      const pair = object(item);
      const text = (
        value: JsonValue | undefined,
        coordinate: string,
      ): string | undefined => {
        const binary = binaryByPointer.get(coordinate);
        if (binary !== undefined)
          return binary.state === "retained"
            ? Buffer.from(binary.content_base64, "base64").toString("latin1")
            : undefined;
        return typeof value === "string" ? value : undefined;
      };
      const name = Array.isArray(item)
        ? text(item[0], `/${side}/headers/${index}/0`)
        : pair?.name;
      const value = Array.isArray(item)
        ? text(item[1], `/${side}/headers/${index}/1`)
        : pair?.value;
      if (typeof name === "string") {
        const key = name.toLowerCase();
        const knownValue = typeof value === "string" ? value : undefined;
        const previous = headers.get(key);
        // Repeated declarations agree only when known and equal; media types by essence.
        headers.set(
          key,
          !headers.has(key)
            ? knownValue
            : previous !== undefined &&
                knownValue !== undefined &&
                (key === "content-type"
                  ? mediaTypeEssence(previous) === mediaTypeEssence(knownValue)
                  : previous === knownValue)
              ? previous
              : undefined,
        );
      } else headerNamesKnown = false;
    }
  }
  // A visible declaration selects framing. Absence is established only when every header
  // name is known; excluded or conflicting values leave framing applicability unknown.
  const header = headers.get("content-type");
  const automaticFraming = (contentType: string | undefined) => {
    const essence =
      contentType === undefined ? undefined : mediaTypeEssence(contentType);
    return essence !== undefined && Object.hasOwn(PROTOBUF_FRAMING, essence)
      ? PROTOBUF_FRAMING[essence as keyof typeof PROTOBUF_FRAMING]
      : undefined;
  };
  if (format === "har") {
    if (
      pointer !== `/${side}/${side === "request" ? "postData" : "content"}/text`
    )
      return undefined;
    // HAR requires a string mimeType; any other value is an excluded declaration.
    const mimeType = object(
      message[side === "request" ? "postData" : "content"],
    )?.mimeType;
    const contentType = typeof mimeType === "string" ? mimeType : header;
    return {
      contentType,
      automaticFraming: automaticFraming(contentType),
      contentTypeKnown: contentType !== undefined,
      httpIdentityKnown: true,
      httpCodingKnown: true,
    };
  }
  if (pointer !== `/${side}/body` && pointer !== `/${side}/content`)
    return undefined;
  return {
    contentType: header,
    automaticFraming: automaticFraming(header),
    contentTypeKnown:
      header !== undefined ||
      (headerNamesKnown && !headers.has("content-type")),
    httpCodingKnown:
      headerNamesKnown &&
      ["content-encoding", "transfer-encoding"].every(
        (name) => !headers.has(name) || headers.get(name) !== undefined,
      ),
    httpIdentityKnown:
      headerNamesKnown &&
      ["content-encoding", "transfer-encoding"].every(
        (name) =>
          // HTTP OWS is SP/HTAB; require the entire identity token, including EOF.
          !headers.has(name) ||
          /^[ \t]*identity[ \t]*(?![\s\S])/i.test(headers.get(name) ?? ""),
      ),
  };
};

type RetainedBody = Extract<
  WebNetworkCaptureRecord["binary_fields"][number],
  { state: "retained" }
>;
interface PreparedBody {
  readonly field: RetainedBody;
  readonly body: Buffer;
  readonly contentType: string | undefined;
  readonly framing: ProtobufDecoding["framing"];
  readonly prepared: PreparedProtobufBody;
}
export interface PreparedProtobufCapture {
  readonly records: readonly {
    record: WebNetworkCaptureRecord;
    bodies: readonly PreparedBody[];
  }[];
  readonly incomplete: boolean;
}

/** Sanitize every selected body before any schema acquisition or field-expansion failure.
 * Until this stage completes, the caller must not publish the original source as partial Evidence.
 */
export const prepareCaptureProtobuf = async (
  capture: WebNetworkCapture,
  input: InspectWebNetworkCaptureInput,
  signal?: AbortSignal,
): Promise<PreparedProtobufCapture | undefined> => {
  if (input.record_ordinals === undefined) return undefined;
  capture.protobuf_coverage = {
    status: "partial",
    selected_records: input.record_ordinals.length,
    inspected_records: 0,
  };
  capture.limitations.push(...PROTOBUF_LIMITATIONS);
  let incomplete = false;
  const records: { record: WebNetworkCaptureRecord; bodies: PreparedBody[] }[] =
    [];
  const options = input.protobuf;
  const pointers =
    options?.body_pointers === undefined
      ? undefined
      : new Set(options.body_pointers);
  const redactions = new CaptureRedaction(input.sensitive_values);
  const work = new ProtobufBudget(signal);
  for (const ordinal of input.record_ordinals) {
    await work.checkpoint();
    const record = capture.records[ordinal];
    if (record === undefined) continue;
    const bodies: PreparedBody[] = [];
    records.push({ record, bodies });
    const binaryByPointer = new Map(
      record.binary_fields.map((field) => [field.pointer, field]),
    );
    for (const field of record.binary_fields) {
      await work.checkpoint();
      if (pointers !== undefined && !pointers.has(field.pointer)) continue;
      const metadata = bodyMetadata(
        record,
        field.pointer,
        input.format,
        binaryByPointer,
      );
      // Framing overrides interpret bodies; header and extension bytes remain their own facts.
      if (metadata === undefined) continue;
      if (field.state !== "retained") {
        incomplete = true;
        continue;
      }
      const { contentType, automaticFraming: automatic } = metadata;
      const framing =
        options?.framing === undefined || options.framing === "auto"
          ? automatic
          : options.framing;
      if (framing === undefined) {
        // An empty body has no protobuf content under any framing.
        if (!metadata.contentTypeKnown && field.bytes > 0) {
          incomplete = true;
          record.limitations.push(
            "Selected body's MIME metadata is excluded, conflicting or unavailable, so automatic protobuf framing applicability is unknown; its retained bytes were not interpreted. Select explicit framing to decode them.",
          );
        }
        continue;
      }
      if (
        field.bytes > WEB_NETWORK_CAPTURE_LIMITS.inputBytes ||
        field.content_base64.length >
          Math.ceil(WEB_NETWORK_CAPTURE_LIMITS.inputBytes / 3) * 4
      )
        throw new CaptureFormatError(
          "limit",
          "Protobuf body exceeds the admitted 32 MiB source bound",
        );
      const body = Buffer.from(field.content_base64, "base64");
      if (
        body.length !== field.bytes ||
        createHash("sha256").update(body).digest("hex") !== field.sha256
      )
        throw new AnalysisInputError("inspect_web_network_capture", undefined, [
          {
            path: ["capture_path"],
            reason: "invalid_format",
            message:
              "Retained protobuf bytes contradict their source descriptor",
          },
        ]);
      const exclusions = {
        decoded: (data: Uint8Array) =>
          redactions.sensitiveBytes(
            Buffer.from(data.buffer, data.byteOffset, data.byteLength),
          ),
        header: isCaptureCredentialHeaderBytes,
      };
      const prepared = await prepareProtobufBody(
        body,
        framing,
        exclusions,
        signal,
      );
      // Caller overrides do not turn off exclusions for the transport declared by the source.
      const declared =
        prepared.excluded === undefined &&
        automatic !== undefined &&
        automatic !== framing
          ? await prepareProtobufBody(body, automatic, exclusions, signal)
          : prepared;
      const excluded = prepared.excluded ?? declared.excluded;
      if (excluded !== undefined) {
        excludeBody(record, field.pointer, excluded);
        binaryByPointer.delete(field.pointer);
        incomplete = true;
        continue;
      }
      // Analysis eligibility never bypasses the independent source exclusions.
      if (
        !metadata.httpCodingKnown &&
        (!prepared.credentialsChecked || !declared.credentialsChecked)
      )
        throw new AnalysisCapabilityUnavailableError(
          input.format,
          "inspect_web_network_capture",
          "Credential inspection could not complete while HTTP coding metadata was unavailable; unchecked capture bodies were withheld.",
          {
            diagnostics: { body_pointer: field.pointer },
            userMessage:
              "Use a capture with available HTTP coding metadata, or select records whose transport can be inspected. Unchecked source bytes were not published.",
          },
        );
      if (!metadata.httpIdentityKnown) {
        incomplete = true;
        record.limitations.push(
          "Selected protobuf body's HTTP content/transfer coding is encoded or unknown, so identity is not established; its exact stored bytes remain opaque.",
        );
        continue;
      }
      bodies.push({ field, body, contentType, framing, prepared });
    }
  }
  return { records, incomplete };
};

/** Interpret the sanitized source once, inside the capture adapter's owned operation. */
export const decodeCaptureProtobuf = async (
  capture: WebNetworkCapture,
  input: InspectWebNetworkCaptureInput,
  resources: ArtifactResourceScope,
  prepared: PreparedProtobufCapture | undefined,
  signal?: AbortSignal,
  encodedSourceBytes = 0,
): Promise<void> => {
  if (prepared === undefined || capture.protobuf_coverage === undefined) return;
  let incomplete = prepared.incomplete;
  const budget = new ProtobufBudget(signal);
  budget.charge(
    encodedSourceBytes +
      256 +
      Buffer.byteLength(JSON.stringify(PROTOBUF_LIMITATIONS)),
  );
  for (const entry of prepared.records)
    for (const item of entry.bodies)
      budget.charge(
        item.body.length +
          (item.prepared.data === item.body
            ? 0
            : (item.prepared.data?.length ?? 0)),
      );
  const redact = (text: string) => {
    if (input.sensitive_values.length === 0) return text;
    if (input.sensitive_values.some((literal) => text.includes(literal)))
      budget.charge(text.length * 60);
    return redactExplicitText(text, input.sensitive_values);
  };
  const options = input.protobuf;
  let schema: ProtobufSchema | undefined;
  let schemaSource: ProtobufDecoding["schema"];
  if (options?.schema !== undefined) {
    const path = options.schema.descriptor_set_path;
    if (!isAbsolute(path))
      throw new AnalysisInputError("inspect_web_network_capture", undefined, [
        {
          path: ["protobuf", "schema", "descriptor_set_path"],
          reason: "invalid_format",
          message: "Expected an absolute descriptor-set path",
        },
      ]);
    const snapshot = await readStableArtifact(
      path,
      WEB_NETWORK_CAPTURE_LIMITS.inputBytes,
      signal,
      undefined,
      resources,
    ).catch((cause: unknown) => {
      throw withCaptureReaderCleanup(
        historicalCaptureFailure(
          input,
          cause,
          "descriptor-read",
          signal === undefined ? undefined : { signal },
        ),
        cause,
      );
    });
    const { ProtobufSchema } = await import("./ProtobufSchema.js");
    try {
      schema = await ProtobufSchema.read(
        snapshot.bytes,
        options.schema.message_type,
        budget,
        redact,
      );
    } catch (cause) {
      if (signal?.aborted === true && cause === signal.reason) throw cause;
      if (cause instanceof CaptureFormatError) throw cause;
      throw new AnalysisInputError("inspect_web_network_capture", undefined, [
        {
          path: ["protobuf", "schema"],
          reason: "invalid_format",
          message:
            "The selected local FileDescriptorSet or message type is invalid or exceeds decoding budgets",
        },
      ]);
    }
    schemaSource = {
      decoder: "protobuf-es-2.16.0",
      descriptor_set_path: redact(path),
      descriptor_set_sha256: snapshot.sha256,
      descriptor_set_bytes: snapshot.bytes.length,
      message_type: redact(options.schema.message_type),
    };
  }
  for (const { record, bodies } of prepared.records) {
    const decoded: ProtobufDecoding[] = [];
    for (const {
      field,
      body,
      contentType,
      framing,
      prepared: bodySource,
    } of bodies) {
      await budget.checkpoint();
      if (
        input.sensitive_values.some(
          (value) =>
            field.pointer.includes(value) ||
            field.sha256.includes(value) ||
            schemaSource?.descriptor_set_sha256.includes(value),
        )
      ) {
        incomplete = true;
        continue;
      }
      budget.charge(
        2048 +
          8 *
            (field.pointer.length +
              (contentType?.length ?? 0) +
              (schemaSource?.descriptor_set_path.length ?? 0) +
              (schemaSource?.message_type.length ?? 0)),
      );
      schema?.reset();
      const result = await decodeProtobufBody(
        body,
        framing,
        {
          pointer: field.pointer,
          content_type: contentType ?? null,
          body_sha256: field.sha256,
        },
        redact,
        budget,
        schema,
        bodySource,
      );
      if (schemaSource !== undefined) result.schema = schemaSource;
      if (
        result.status !== "complete" ||
        result.fields.some(
          (field) =>
            field.schema_status !== undefined &&
            field.schema_status !== "decoded",
        )
      )
        incomplete = true;
      decoded.push(result);
    }
    if (decoded.length > 0) record.protobuf = decoded;
    capture.protobuf_coverage.inspected_records++;
  }
  if (
    !incomplete &&
    capture.protobuf_coverage.inspected_records ===
      capture.protobuf_coverage.selected_records
  )
    capture.protobuf_coverage.status = "complete";
};