Skip to content

armgit / rea

Public Imported from GitHub
0

Reverse engineer anything with agents, from app behavior down to native binaries.

20.1 KB
Raw Blame
import { z } from "zod";
import type { AnalysisError } from "../analysisErrorBase.js";
import { jsonValueType, parseJsonObject } from "../jsonShape.js";
import { ok, type Result } from "../result.js";
import {
  compareUnicodeCodePoints as compare,
  compositeKey,
} from "../unicodeCodePointOrder.js";
import {
  WEB_NETWORK_CAPTURE_LIMITS,
  type WebNetworkCaptureRecord,
} from "../webNetworkCapture.js";
import { completeWithinViewCoverage } from "./analysisViewCoverage.js";
import type { UnsignedAnalysisView } from "./analysisView.js";
import { WebNetworkProjectionAccumulator } from "./WebNetworkProjectionAccumulator.js";
import {
  compareTools,
  parameterNames,
  pathTemplate,
  toolName,
  toolNameRegistry,
} from "./environmentManifestNames.js";
import {
  primaryOrigin,
  type OriginStats,
} from "./environmentManifestOrigin.js";
import {
  applyTemplateRules,
  SEGMENT_LIMITED_RULES,
  SIBLING_VARIANCE_SEGMENT_LIMIT,
  type BodyShape,
  type JsonTypeName,
  type LiteralPart,
  type TemplateRule,
  type ToolGroup,
} from "./environmentManifestRules.js";
import {
  ENTRY_BYTES,
  GROUP_BYTES,
  RECORD_SCRATCH_BYTES,
  SEGMENT_BYTES,
  mediaTypeEssence,
  readQueryParameterNames,
  readRecordedRequestLine,
  routePartCount,
  scanRouteTemplate,
  textBytes,
  type RecordedRequestLine,
  type RouteRule,
  type RouteTemplate,
} from "./webNetworkRouteTemplates.js";

type Manifest = Extract<UnsignedAnalysisView, { kind: "environment-manifest" }>;
type ManifestFields = Pick<
  Manifest,
  | "origin"
  | "name"
  | "tools"
  | "records"
  | "excluded"
  | "unknowns"
  | "templated_groups"
  | "coverage"
  | "limitations"
>;
type Tool = Manifest["tools"][number];
type Exclusion = Manifest["excluded"][number];
type RequestLine = Extract<RecordedRequestLine, { kind: "line" }>;

interface RecordLine {
  readonly ordinal: number;
  readonly method: string;
  readonly path: string;
  readonly status: number | null;
  readonly contentType: string | null;
}

type MutablePart =
  | LiteralPart
  | {
      readonly kind: "param";
      readonly rule: RouteRule;
      readonly values: Set<string>;
    };

interface GroupState extends ToolGroup {
  readonly parts: readonly MutablePart[];
  readonly positions: number[];
  readonly statuses: Set<number>;
  readonly contentTypes: Set<string>;
  readonly queryNames: Set<string>;
  readonly apiPath: boolean;
  jsonLike: boolean;
  /** Every response of the write group had no body, as a 204 has. */
  bodiless: boolean;
}

const responseTextSchema = z.object({
  response: z.object({
    status: z.unknown().optional(),
    content: z
      .object({ text: z.unknown().optional(), size: z.unknown().optional() })
      .nullable()
      .optional(),
  }),
});
/** Whether a record's response had no body: a 204 or 205, an empty text, or a recorded size of zero; a missing text alone may be a body not kept. */
const emptyResponse = (reported: unknown): boolean => {
  const parsed = responseTextSchema.safeParse(reported);
  if (!parsed.success) return false;
  const { status, content } = parsed.data.response;
  return (
    status === 204 ||
    status === 205 ||
    content?.text === "" ||
    ((content?.text === undefined || content.text === null) &&
      content?.size === 0)
  );
};
/** A write group every response of which had no body, at least one of them a success. */
const answeredEmpty = (group: GroupState): boolean =>
  group.bodiless &&
  [...group.statuses].some((status) => status >= 200 && status < 300);
const postDataSchema = z.object({
  request: z.object({
    postData: z
      .object({
        mimeType: z.unknown().optional(),
        text: z.unknown().optional(),
      })
      .nullable()
      .optional(),
  }),
});
const JSON_LIKE_TEXT = new Set(["text/plain", "text/markdown"]);
const ASSET_ESSENCES = new Set([
  "text/html",
  "application/xhtml+xml",
  "text/css",
  "text/javascript",
  "application/javascript",
  "application/x-javascript",
  "application/wasm",
]);
const ASSET_PREFIXES = ["image/", "font/", "audio/", "video/"];
const ASSET_PATH =
  /\.(?:m?js|css|map|html?|png|jpe?g|gif|svg|webp|ico|woff2?|ttf|otf)$/iu;
const WRITE_METHODS = new Set(["POST", "PUT", "PATCH"]);
/** A write whose every response is empty is a tool whatever its content type: the empty reply is its answer. */
const BODILESS_WRITE_METHODS = new Set([...WRITE_METHODS, "DELETE"]);
/** Quotes, separators and brackets a four-part composite key adds around its texts. */
const KEY_OVERHEAD_CHARS = 32;
const UNKNOWNS: Manifest["unknowns"] = [
  {
    kind: "authorization-model",
    statement:
      "Authorization headers and cookies are structurally excluded from captures; the manifest states no authorization model.",
  },
  {
    kind: "response-semantics",
    statement:
      "Recorded statuses and bodies are retained only as pointers to parent records; the manifest states no response semantics.",
  },
  {
    kind: "state-transition",
    statement:
      "Recorded exchanges are independent observations; the manifest states no ordering, side effects or state transitions between tools.",
  },
];

/** `application/json` or `application/*+json`; a `+json` suffix on another type is not JSON here. */
const isJson = (essence: string | null): boolean =>
  essence === "application/json" ||
  (essence !== null &&
    essence.startsWith("application/") &&
    essence.endsWith("+json"));

/** JSON, `+json`, `text/plain` or `text/markdown`: the content types a replay parses as JSON. */
export const isJsonLike = (essence: string | null): boolean =>
  isJson(essence) || (essence !== null && JSON_LIKE_TEXT.has(essence));

const isApiPath = (path: string): boolean =>
  path.startsWith("/api/") || path.startsWith("/apis/");

const isAsset = (group: GroupState): boolean =>
  [...group.contentTypes].some(
    (essence) =>
      ASSET_ESSENCES.has(essence) ||
      ASSET_PREFIXES.some((prefix) => essence.startsWith(prefix)),
  ) || ASSET_PATH.test(group.api_surface_template);

/** Kebab-case name from the origin host; dots and ports become hyphens. */
const environmentName = (origin: string): string =>
  new URL(origin).host
    .toLowerCase()
    .replaceAll(/[^a-z0-9]+/gu, "-")
    .replace(/^-/u, "")
    .replace(/-$/u, "");

const routeParts = (route: RouteTemplate): MutablePart[] => {
  const byIndex = new Map(
    route.segments.map((segment) => [segment.segment_index, segment]),
  );
  return route.template
    .slice(1)
    .split("/")
    .map((text, index): MutablePart => {
      const segment = byIndex.get(index);
      return segment === undefined
        ? { kind: "literal", value: text }
        : {
            kind: "param",
            rule: segment.rule,
            values: new Set([segment.value]),
          };
    });
};

const hasParam = (parts: readonly MutablePart[]): boolean =>
  parts.some((part) => part.kind === "param");

/** Own tool grouping, record pointers and admission for the entire manifest. */
export class EnvironmentManifestAccumulator extends WebNetworkProjectionAccumulator {
  readonly #groups = new Map<string, GroupState>();
  readonly #records: RecordLine[] = [];
  readonly #origins = new Map<string, OriginStats>();
  readonly #rules: ReadonlySet<TemplateRule>;
  #unrepresented = 0;
  #unrepresentableBodyKey = false;

  constructor(
    maximumBytes = WEB_NETWORK_CAPTURE_LIMITS.outputBytes,
    readonly includeQueryParameterNames = true,
    rules: readonly TemplateRule[] = [],
    budgetSubject = "Environment-manifest",
  ) {
    super(maximumBytes, budgetSubject);
    this.#rules = new Set(rules);
  }

  /** Whether the finished projection publishes body shapes, and so states their limitation. */
  protected get reportsBodyShapes(): boolean {
    return true;
  }

  add(record: WebNetworkCaptureRecord): Result<void, AnalysisError> {
    const limitations = this.addLimitations(record.limitations);
    if (!limitations.ok) return limitations;
    try {
      const line = readRecordedRequestLine(
        record.reported,
        this.includeQueryParameterNames,
        this.scratch,
      );
      if (line === null) return this.overBudget();
      if (line.kind === "excluded") {
        this.#unrepresented += 1;
        return ok(undefined);
      }
      const route = scanRouteTemplate(line.path, this.scratch);
      if (route === null) return this.overBudget();
      const queryNames = readQueryParameterNames(line.query, this.scratch);
      if (queryNames === null) return this.overBudget();
      // The key copies the method, origin, template and rules; charge that copy before it exists.
      if (
        !this.scratch(
          textBytes(
            line.method.length +
              line.origin.length +
              route.template.length +
              route.rules.length +
              KEY_OVERHEAD_CHARS,
          ),
        )
      )
        return this.overBudget();
      const key = compositeKey([
        line.method,
        line.origin,
        route.template,
        route.rules,
      ]);
      const group = this.#groups.get(key);
      const bodyShape =
        group === undefined && WRITE_METHODS.has(line.method)
          ? this.#bodyShape(record.reported)
          : ok(null);
      if (!bodyShape.ok) return bodyShape;
      return this.#append(
        record.ordinal,
        line,
        route,
        queryNames,
        { key, group },
        { shape: bodyShape.value, empty: emptyResponse(record.reported) },
      );
    } finally {
      this.releaseTemporary();
    }
  }

  /** Read the top-level key types of the group's first recorded JSON-typed request body; nested values are never walked. */
  #bodyShape(reported: unknown): Result<BodyShape | null, AnalysisError> {
    const parsed = postDataSchema.safeParse(reported);
    const body = parsed.success ? parsed.data.request.postData : undefined;
    if (
      body === undefined ||
      body === null ||
      typeof body.text !== "string" ||
      typeof body.mimeType !== "string"
    )
      return ok(null);
    // The media type is scanned and lowercased before it is known to be JSON; charge it first.
    if (!this.scratch(textBytes(body.mimeType.length)))
      return this.overBudget();
    if (!isJson(mediaTypeEssence(body.mimeType))) return ok(null);
    // The parsed tree is linear in the text and coexists with it and the key entries.
    if (!this.scratch(RECORD_SCRATCH_BYTES + textBytes(body.text.length) * 3))
      return this.overBudget();
    const root = parseJsonObject(body.text);
    if (root === null) return ok(null);
    const entries: [string, JsonTypeName][] = [];
    for (const [name, value] of Object.entries(root)) {
      // JSON parsers and schema clones drop a __proto__ own property; omit it explicitly.
      if (name === "__proto__") this.#unrepresentableBodyKey = true;
      else entries.push([name, jsonValueType(value)]);
    }
    return ok({ type: "object", properties: Object.fromEntries(entries) });
  }

  #append(
    ordinal: number,
    line: RequestLine,
    route: RouteTemplate,
    queryNames: ReadonlySet<string> | undefined,
    bucket: { readonly key: string; readonly group: GroupState | undefined },
    exchange: { readonly shape: BodyShape | null; readonly empty: boolean },
  ): Result<void, AnalysisError> {
    const bodyShape = exchange.shape;
    const mime = mediaTypeEssence(line.rawMime);
    let group = bucket.group;
    // One retained record pointer, including its projected output.
    let bytes =
      ENTRY_BYTES * 2 +
      textBytes(line.path.length) +
      textBytes(line.method.length);
    const origin = this.#origins.get(line.origin);
    if (origin === undefined)
      bytes += ENTRY_BYTES * 2 + textBytes(line.origin.length);
    if (group === undefined) {
      bytes +=
        GROUP_BYTES +
        textBytes(bucket.key.length) +
        textBytes(line.method.length) +
        textBytes(line.origin.length) +
        textBytes(route.template.length) * 2 +
        textBytes(route.rules.length);
      // Every part, literal or templated, is one object in the group and in each rule, template and name pass.
      bytes += ENTRY_BYTES * 4 * routePartCount(route.template);
      for (const segment of route.segments)
        bytes += SEGMENT_BYTES + ENTRY_BYTES + textBytes(segment.value.length);
      if (bodyShape !== null)
        for (const name of Object.keys(bodyShape.properties))
          bytes += ENTRY_BYTES + textBytes(name.length);
    } else {
      for (const segment of route.segments) {
        const part = group.parts[segment.segment_index];
        if (part?.kind === "param" && !part.values.has(segment.value))
          bytes += ENTRY_BYTES + textBytes(segment.value.length);
      }
    }
    if (line.status !== null && !group?.statuses.has(line.status))
      bytes += ENTRY_BYTES;
    if (mime !== null && !group?.contentTypes.has(mime))
      bytes += ENTRY_BYTES + textBytes(mime.length);
    for (const name of queryNames ?? [])
      if (!group?.queryNames.has(name))
        bytes += ENTRY_BYTES + textBytes(name.length);
    // Reserve the whole mutation while route scratch is still live.
    if (!this.retain(bytes)) return this.overBudget();
    const position = this.#records.length;
    this.#records.push({
      ordinal,
      method: line.method,
      path: line.path,
      status: line.status,
      contentType: mime,
    });
    const jsonLike = isJsonLike(mime);
    const stats = origin ?? { records: 0, jsonLike: 0 };
    stats.records += 1;
    if (jsonLike) stats.jsonLike += 1;
    this.#origins.set(line.origin, stats);
    if (group === undefined) {
      group = {
        method: line.method,
        origin: line.origin,
        api_surface_template: route.template,
        parts: routeParts(route),
        positions: [],
        statuses: new Set(),
        contentTypes: new Set(),
        queryNames: new Set(),
        bodyShape,
        apiPath: isApiPath(route.template),
        jsonLike: false,
        bodiless: BODILESS_WRITE_METHODS.has(line.method),
      };
      this.#groups.set(bucket.key, group);
    } else {
      for (const segment of route.segments) {
        const part = group.parts[segment.segment_index];
        if (part?.kind === "param") part.values.add(segment.value);
      }
    }
    group.positions.push(position);
    group.jsonLike ||= jsonLike;
    group.bodiless &&= exchange.empty;
    if (line.status !== null) group.statuses.add(line.status);
    if (mime !== null) group.contentTypes.add(mime);
    for (const name of queryNames ?? []) group.queryNames.add(name);
    return ok(undefined);
  }

  #tools(groups: readonly ToolGroup[]): Tool[] {
    const registry = toolNameRegistry();
    return applyTemplateRules(groups, this.#rules)
      .map((group) => {
        const names = parameterNames(group.parts);
        return { group, names, template: pathTemplate(group.parts, names) };
      })
      .sort((left, right) =>
        compareTools(
          { template: left.template, method: left.group.method },
          { template: right.template, method: right.group.method },
        ),
      )
      .map(({ group, names, template }): Tool => {
        let next = 0;
        return {
          name: toolName(group.method, group.parts, names, registry),
          method: group.method,
          path_template: template,
          api_surface_template: group.api_surface_template,
          params: group.parts.flatMap((part, index) => {
            if (part.kind !== "param") return [];
            const name = names[next] ?? "";
            next += 1;
            return [
              {
                index,
                name,
                rule: part.rule,
                basis: "inferred" as const,
                observed_values: [...part.values].sort(compare),
              },
            ];
          }),
          query_parameter_names: [...group.queryNames].sort(compare),
          body_shape: group.bodyShape,
          statuses: [...group.statuses].sort((left, right) => left - right),
          content_types: [...group.contentTypes].sort(compare),
          record_ordinals: group.positions.map(
            (position) => this.#records[position]!.ordinal,
          ),
        };
      });
  }

  /** One limitation per selected sibling rule when a route is too long for it to compare. */
  #segmentLimitLimitations(toolGroups: readonly ToolGroup[]): string[] {
    if (
      !toolGroups.some(
        (group) => group.parts.length > SIBLING_VARIANCE_SEGMENT_LIMIT,
      )
    )
      return [];
    return SEGMENT_LIMITED_RULES.filter((rule) => this.#rules.has(rule)).map(
      (rule) =>
        `Routes with more than ${String(SIBLING_VARIANCE_SEGMENT_LIMIT)} segments are not compared by ${rule}; they keep their grouped templates.`,
    );
  }

  /** Project the complete manifest; tools, records and exclusions are sorted deterministically. */
  finish(): Result<ManifestFields, AnalysisError> {
    const primary = primaryOrigin(this.#origins);
    const toolGroups: GroupState[] = [];
    const exclusions = new Map<string, Exclusion & { templated: boolean }>();
    for (const group of this.#groups.values()) {
      const reason: Exclusion["reason"] | null =
        group.origin !== primary
          ? "other-origin"
          : group.apiPath || group.jsonLike || answeredEmpty(group)
            ? null
            : isAsset(group)
              ? "ui-page-or-asset"
              : "non-json";
      if (reason === null) {
        toolGroups.push(group);
        continue;
      }
      const exclusion = {
        method: group.method,
        origin: group.origin,
        path_template: group.api_surface_template,
        reason,
      };
      // Distinct groups can print the same exclusion row; a templated one keeps the row inferred.
      const key = compositeKey(Object.values(exclusion));
      exclusions.set(key, {
        ...exclusion,
        templated:
          (exclusions.get(key)?.templated ?? false) || hasParam(group.parts),
      });
    }
    const tools = this.#tools(toolGroups);
    const cited = [...new Set(tools.flatMap((tool) => tool.record_ordinals))];
    const positions = new Map(
      this.#records.map((record, position) => [record.ordinal, position]),
    );
    const excluded = [...exclusions.values()].sort(
      (left, right) =>
        compare(left.origin, right.origin) ||
        compare(left.path_template, right.path_template) ||
        compare(left.method, right.method) ||
        compare(left.reason, right.reason),
    );
    const limitations = this.addLimitations([
      ...(this.reportsBodyShapes &&
      tools.some((tool) => tool.body_shape !== null)
        ? [
            "Body shapes list top-level JSON key types from one recorded request each; they do not establish required fields or a complete schema.",
          ]
        : []),
      ...(this.#unrepresentableBodyKey
        ? [
            "A top-level __proto__ key in a recorded JSON body is omitted from body_shape; it cannot be an own property of the published object.",
          ]
        : []),
      ...this.#segmentLimitLimitations(toolGroups),
      ...(this.#unrepresented > 0
        ? [
            `${String(this.#unrepresented)} retained records without a usable request line are not represented in the manifest.`,
          ]
        : []),
      ...(primary === null
        ? [
            "No retained record had a parseable origin; the manifest names no primary origin.",
          ]
        : []),
    ]);
    if (!limitations.ok) return limitations;
    return ok({
      origin: primary ?? "",
      name: primary === null ? "" : environmentName(primary),
      tools,
      records: cited
        .sort((left, right) => positions.get(left)! - positions.get(right)!)
        .map((ordinal) => {
          const record = this.#records[positions.get(ordinal)!]!;
          return {
            ordinal: record.ordinal,
            method: record.method,
            path: record.path,
            status: record.status,
            content_type: record.contentType,
          };
        }),
      excluded: excluded.map(
        ({ templated: _templated, ...exclusion }) => exclusion,
      ),
      unknowns: UNKNOWNS.map((unknown) => ({ ...unknown })),
      templated_groups:
        tools.filter((tool) => tool.params.length > 0).length +
        excluded.filter((exclusion) => exclusion.templated).length,
      coverage: completeWithinViewCoverage(tools.length, tools.length),
      limitations: this.sortedLimitations(),
    });
  }
}