Skip to content

armgit / rea

Public Imported from GitHub
0

Reverse engineer anything with agents, from app behavior down to native binaries.

fix(schema): compose portable stack-safe base64 constraints

morluto0949313d8fa2
diff --git a/src/browser/history/HarCapture.ts b/src/browser/history/HarCapture.ts
index 9e164e92d..ec55644d0 100644
--- a/src/browser/history/HarCapture.ts
+++ b/src/browser/history/HarCapture.ts
@@ -2,7 +2,7 @@ import { createHash } from "node:crypto";
import { LosslessNumber } from "lossless-json";
import { parseCaptureJson } from "./CaptureJson.js";
import type { JsonValue } from "../../domain/jsonValue.js";
-import { CANONICAL_BASE64_PATTERN } from "../../domain/stringPatterns.js";
+import { isCanonicalBase64 } from "../../domain/stringPatterns.js";
import {
WEB_NETWORK_CAPTURE_LIMITS,
type WebNetworkCapture,
@@ -265,7 +265,7 @@ const integerLexeme = (literal: string): boolean => {
const decodeHarBase64 = (value: string, pointer: string): Buffer => {
const compact = value.replace(/[\t\n\r ]/g, "");
- if (compact.length % 4 !== 0 || !CANONICAL_BASE64_PATTERN.test(compact))
+ if (!isCanonicalBase64(compact))
throw new CaptureFormatError(
"format",
"HAR content declares base64 but its text is not valid canonical base64 (ASCII whitespace is allowed).",
diff --git a/src/domain/go/goBinary.ts b/src/domain/go/goBinary.ts
index 8714159c3..550d7a834 100644
--- a/src/domain/go/goBinary.ts
+++ b/src/domain/go/goBinary.ts
@@ -1,7 +1,7 @@
import { z } from "zod";
import { digestSchema } from "../digests.js";
import { localPathStringSchema } from "../localPath.js";
-import { CANONICAL_BASE64_PATTERN } from "../stringPatterns.js";
+import { canonicalBase64Schema } from "../stringPatterns.js";
const sourceRangeSchema = z.strictObject({
offset: z.number().int().nonnegative(),
@@ -25,9 +25,7 @@ export const goModuleMetadataSchema = z.strictObject({
dependencies: z.array(goModuleSchema),
settings: z.array(z.strictObject({ key: z.string(), value: z.string() })),
unparsed_lines: z.array(z.string()),
- unparsed_line_bytes_base64: z.array(
- z.string().regex(CANONICAL_BASE64_PATTERN),
- ),
+ unparsed_line_bytes_base64: z.array(canonicalBase64Schema),
complete: z.boolean(),
});
@@ -43,9 +41,9 @@ export const goBuildInfoSchema = z.strictObject({
header_offset: z.number().int().nonnegative(),
encoding: z.enum(["inline", "pointer"]),
go_version: z.string().nullable(),
- go_version_bytes_base64: z.string().regex(CANONICAL_BASE64_PATTERN),
+ go_version_bytes_base64: canonicalBase64Schema,
module_text: z.string().nullable(),
- module_bytes_base64: z.string().regex(CANONICAL_BASE64_PATTERN),
+ module_bytes_base64: canonicalBase64Schema,
version_location: sourceRangeSchema,
module_location: sourceRangeSchema,
module: goModuleMetadataSchema,
diff --git a/src/domain/native/binaryLayout.ts b/src/domain/native/binaryLayout.ts
index cc37376dd..c1a694cc2 100644
--- a/src/domain/native/binaryLayout.ts
+++ b/src/domain/native/binaryLayout.ts
@@ -2,13 +2,13 @@ import { z } from "zod";
import { canBorrowParsedJsonSnapshot } from "../immutableJson.js";
import { localPathStringSchema } from "../localPath.js";
-import { CANONICAL_BASE64_PATTERN } from "../stringPatterns.js";
+import { canonicalBase64Schema } from "../stringPatterns.js";
const unsignedHex = z.string().regex(/^0x(?:0|[1-9a-f][0-9a-f]{0,15})$/);
const index = z.number().int().nonnegative();
const scalar = z.union([z.string(), z.number().int()]);
const range = z.strictObject({ offset: unsignedHex, bytes: unsignedHex });
-const canonicalBase64 = z.string().regex(CANONICAL_BASE64_PATTERN);
+const canonicalBase64 = canonicalBase64Schema;
const name = z.strictObject({
display: z.string(),
bytes_base64: canonicalBase64.nullable(),
diff --git a/src/domain/native/recordedCrash.ts b/src/domain/native/recordedCrash.ts
index 8bd04cac1..0163a8532 100644
--- a/src/domain/native/recordedCrash.ts
+++ b/src/domain/native/recordedCrash.ts
@@ -2,13 +2,13 @@ import { z } from "zod";
import type { BinaryArchitecture } from "../binaryTargetTypes.js";
import { localPathStringSchema } from "../localPath.js";
-import { CANONICAL_BASE64_PATTERN } from "../stringPatterns.js";
+import { canonicalBase64Schema } from "../stringPatterns.js";
import { recordedNoteCoverageIssues } from "./recordedCrashNoteCoverage.js";
const hex = z.string().regex(/^0x(?:0|[1-9a-f][0-9a-f]{0,15})$/);
const index = z.number().int().nonnegative();
const signed32 = z.number().int().min(-2147483648).max(2147483647);
-const base64 = z.string().regex(CANONICAL_BASE64_PATTERN);
+const base64 = canonicalBase64Schema;
const range = z.strictObject({ offset: hex, bytes: hex });
const diagnostics = z.strictObject({
stdout: z.string(),
diff --git a/src/domain/paddedBase64.ts b/src/domain/paddedBase64.ts
index 395bdf5b8..b4402f6b5 100644
--- a/src/domain/paddedBase64.ts
+++ b/src/domain/paddedBase64.ts
@@ -1,9 +1,8 @@
-import { PADDED_BASE64_PATTERN } from "./stringPatterns.js";
+import { isPaddedBase64 } from "./stringPatterns.js";
/** Validate padded syntax without imposing canonical zero pad bits. */
export const paddedBase64ByteLength = (value: string): number | undefined => {
- if (value.length % 4 !== 0 || !PADDED_BASE64_PATTERN.test(value))
- return undefined;
+ if (!isPaddedBase64(value)) return undefined;
const padding = value.endsWith("==") ? 2 : value.endsWith("=") ? 1 : 0;
return (value.length / 4) * 3 - padding;
};
diff --git a/src/domain/stringPatterns.test.ts b/src/domain/stringPatterns.test.ts
index 7537742bd..4410ad7b9 100644
--- a/src/domain/stringPatterns.test.ts
+++ b/src/domain/stringPatterns.test.ts
@@ -1,10 +1,16 @@
import { expect, it } from "vitest";
+import { z } from "zod";
import {
- CANONICAL_BASE64_PATTERN,
+ BASE64_QUARTET_LENGTH_PATTERN,
+ INVALID_BASE64_CHARACTER_PATTERN,
+ INVALID_BASE64_PADDING_PATTERN,
+ INVALID_CANONICAL_BASE64_PAD_BITS_PATTERN,
IDENTIFIER_PATTERN,
- PADDED_BASE64_PATTERN,
STABLE_IDENTIFIER_PATTERN,
+ canonicalBase64Schema,
+ isCanonicalBase64,
+ isPaddedBase64,
} from "./stringPatterns.js";
it("preserves identifier grammars under Unicode and Unicode-set compilation", () => {
@@ -43,7 +49,7 @@ it("preserves identifier grammars under Unicode and Unicode-set compilation", ()
}
});
-it("preserves canonical base64 padding and pad-bit validation", () => {
+it("preserves canonical and permissive padded base64 validation", () => {
const previous =
/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/][AQgw]==|[A-Za-z0-9+/]{2}[AEIMQUYcgkosw048]=)?$/;
const samples = [
@@ -71,28 +77,62 @@ it("preserves canonical base64 padding and pad-bit validation", () => {
for (const first of alphabet)
for (const second of alphabet)
samples.push(`${first}${second}==`, `A${first}${second}=`);
- for (const flag of ["u", "v"]) {
- const projected = new RegExp(CANONICAL_BASE64_PATTERN.source, flag);
- for (const sample of samples)
- expect(projected.test(sample), JSON.stringify({ flag, sample })).toBe(
- previous.test(sample),
- );
+ for (const sample of samples) {
+ expect(isCanonicalBase64(sample), JSON.stringify({ sample })).toBe(
+ previous.test(sample),
+ );
+ expect(isPaddedBase64(sample), JSON.stringify({ sample })).toBe(
+ sample.length % 4 === 0 && /^[A-Za-z0-9+/]*(?:==|=)?$/.test(sample),
+ );
}
});
-it("validates large canonical base64 under JSON Schema Unicode modes", () => {
- // Near the 32 MiB capture input budget: a quartet expansion alone still
- // overflows at this size, especially in Unicode-set client validators.
+it("advertises every portable base64 constraint and keeps large inputs stack-safe", () => {
+ const jsonSchema = z.toJSONSchema(canonicalBase64Schema, {
+ target: "draft-2020-12",
+ });
+ expect(jsonSchema).toMatchObject({
+ allOf: expect.arrayContaining([
+ { pattern: BASE64_QUARTET_LENGTH_PATTERN.source },
+ { not: { pattern: INVALID_BASE64_CHARACTER_PATTERN.source } },
+ { not: { pattern: INVALID_BASE64_PADDING_PATTERN.source } },
+ { not: { pattern: INVALID_CANONICAL_BASE64_PAD_BITS_PATTERN.source } },
+ ]),
+ });
+
const body = Buffer.alloc(24 * 1024 * 1024 - 3, 0x81);
const encoded = body.toString("base64");
for (const flag of ["", "u", "v"]) {
- const pattern = new RegExp(CANONICAL_BASE64_PATTERN.source, flag);
- expect(pattern.test(encoded), flag).toBe(true);
- expect(pattern.test(`${encoded.slice(0, -1)}!`), flag).toBe(false);
- expect(pattern.test(`${encoded.slice(0, -3)}R==`), flag).toBe(false);
- expect(pattern.test(`${encoded}A`), flag).toBe(false);
- const padded = new RegExp(PADDED_BASE64_PATTERN.source, flag);
- expect(padded.test(encoded), flag).toBe(true);
- expect(padded.test(`${encoded.slice(0, -1)}!`), flag).toBe(false);
+ expect(
+ new RegExp(BASE64_QUARTET_LENGTH_PATTERN.source, flag).test(encoded),
+ flag,
+ ).toBe(true);
+ expect(
+ new RegExp(INVALID_BASE64_CHARACTER_PATTERN.source, flag).test(encoded),
+ flag,
+ ).toBe(false);
+ expect(
+ new RegExp(INVALID_BASE64_PADDING_PATTERN.source, flag).test(encoded),
+ flag,
+ ).toBe(false);
+ expect(
+ new RegExp(INVALID_CANONICAL_BASE64_PAD_BITS_PATTERN.source, flag).test(
+ encoded,
+ ),
+ flag,
+ ).toBe(false);
+ expect(
+ new RegExp(INVALID_BASE64_CHARACTER_PATTERN.source, flag).test(
+ `${encoded.slice(0, -1)}!`,
+ ),
+ flag,
+ ).toBe(true);
+ expect(
+ new RegExp(INVALID_CANONICAL_BASE64_PAD_BITS_PATTERN.source, flag).test(
+ `${encoded.slice(0, -3)}R==`,
+ ),
+ flag,
+ ).toBe(true);
}
+ expect(isCanonicalBase64(encoded)).toBe(true);
});
diff --git a/src/domain/stringPatterns.ts b/src/domain/stringPatterns.ts
index ab8994f17..718cc1d1d 100644
--- a/src/domain/stringPatterns.ts
+++ b/src/domain/stringPatterns.ts
@@ -1,20 +1,44 @@
+import { z } from "zod";
+
/** Portable ASCII identifier starting with a letter. */
export const IDENTIFIER_PATTERN = /^[A-Za-z][A-Za-z0-9._\x2d]*$/u;
/** Stable evidence identifier, also admitting namespace and path separators. */
export const STABLE_IDENTIFIER_PATTERN = /^[A-Za-z][A-Za-z0-9._:\x2f\x2d]*$/u;
-// Repeated alphabet quartets exhaust V8's regexp stack on large bodies,
-// including under JSON Schema's Unicode-set mode. Check alphabet and terminal
-// padding separately, leaving an unconstrained fixed-width loop for length.
-const PADDED_BASE64_PREFIX = String.raw`^(?!.*[^A-Za-z0-9+\x2f=])(?!.*=[^=])(?!.*===)`;
+// These rules are separate JSON Schema constraints. A single RE2-compatible
+// regex for alphabet and quartet validity requires a repeated alphabet group;
+// V8's Unicode-set engine overflows its regexp stack on that group near the
+// provider's 32 MiB encoded-input budget. Wildcard quartets and literal bad
+// token probes stay stack-safe at that size and compose exactly with `allOf`.
+export const BASE64_QUARTET_LENGTH_PATTERN = /^(?:....)*$/;
+export const INVALID_BASE64_CHARACTER_PATTERN = /[^A-Za-z0-9+\x2f=]/;
+export const INVALID_BASE64_PADDING_PATTERN = /(?:=[^=]|===)/;
+export const INVALID_CANONICAL_BASE64_PAD_BITS_PATTERN =
+ /(?:[^AQgw]==$|[BCDFGHJKLNOPRSTVWXZabdefhijlmnpqrtuvxyz1235679+\x2f]=$)/;
+
+export const isPaddedBase64 = (value: string): boolean =>
+ value.length % 4 === 0 &&
+ !INVALID_BASE64_CHARACTER_PATTERN.test(value) &&
+ !INVALID_BASE64_PADDING_PATTERN.test(value);
+
+export const isCanonicalBase64 = (value: string): boolean =>
+ isPaddedBase64(value) &&
+ !INVALID_CANONICAL_BASE64_PAD_BITS_PATTERN.test(value);
-/** Padded base64 syntax, allowing nonzero pad bits and the empty encoding. */
-export const PADDED_BASE64_PATTERN = new RegExp(
- `${PADDED_BASE64_PREFIX}(?:....)*$`,
-);
+const paddedBase64SchemaRules = [
+ { pattern: BASE64_QUARTET_LENGTH_PATTERN.source },
+ { not: { pattern: INVALID_BASE64_CHARACTER_PATTERN.source } },
+ { not: { pattern: INVALID_BASE64_PADDING_PATTERN.source } },
+];
-/** Canonical padded base64, including zero pad bits and the empty encoding. */
-export const CANONICAL_BASE64_PATTERN = new RegExp(
- `${PADDED_BASE64_PREFIX}(?!.*[^AQgw]==$)(?!.*[^AEIMQUYcgkosw048=]=$)(?:....)*$`,
-);
+/** Canonical padded base64 with zero pad bits, including the empty encoding. */
+export const canonicalBase64Schema = z
+ .string()
+ .refine(isCanonicalBase64, "Expected canonical padded base64")
+ .meta({
+ allOf: [
+ ...paddedBase64SchemaRules,
+ { not: { pattern: INVALID_CANONICAL_BASE64_PAD_BITS_PATTERN.source } },
+ ],
+ });
diff --git a/tests/boundary/mcp/advertisedPatternsVMode.test.ts b/tests/boundary/mcp/advertisedPatternsVMode.test.ts
index b48fe27aa..1291d79d9 100644
--- a/tests/boundary/mcp/advertisedPatternsVMode.test.ts
+++ b/tests/boundary/mcp/advertisedPatternsVMode.test.ts
@@ -1,7 +1,14 @@
import { Client, InMemoryTransport } from "@modelcontextprotocol/client";
+import { AjvJsonSchemaValidator } from "@modelcontextprotocol/client/validators/ajv";
import { expect, it, onTestFinished } from "vitest";
import { createServer } from "../../../src/server/createServer.js";
+import {
+ BASE64_QUARTET_LENGTH_PATTERN,
+ INVALID_BASE64_CHARACTER_PATTERN,
+ INVALID_BASE64_PADDING_PATTERN,
+ INVALID_CANONICAL_BASE64_PAD_BITS_PATTERN,
+} from "../../../src/domain/stringPatterns.js";
import { createTestBinarySession } from "../../fixtures/binarySession.js";
interface AdvertisedPattern {
@@ -10,6 +17,33 @@ interface AdvertisedPattern {
readonly pattern: string;
}
+const collectCanonicalBase64Leaves = (
+ value: unknown,
+ leaves: Record<string, unknown>[],
+): void => {
+ if (value === null || typeof value !== "object") return;
+ if (Array.isArray(value)) {
+ value.forEach((entry) => collectCanonicalBase64Leaves(entry, leaves));
+ return;
+ }
+ const record = value as Record<string, unknown>;
+ const allOf = record.allOf;
+ if (
+ Array.isArray(allOf) &&
+ allOf.some(
+ (entry) =>
+ entry !== null &&
+ typeof entry === "object" &&
+ "pattern" in entry &&
+ entry.pattern === BASE64_QUARTET_LENGTH_PATTERN.source,
+ )
+ )
+ leaves.push(record);
+ Object.values(record).forEach((entry) =>
+ collectCanonicalBase64Leaves(entry, leaves),
+ );
+};
+
const collectPatterns = (
value: unknown,
where: string,
@@ -48,12 +82,29 @@ it("advertises portable NUL escapes and patterns that compile in all JS modes",
const advertised = (await client.listTools()).tools;
const patterns: AdvertisedPattern[] = [];
+ const canonicalBase64Leaves: Record<string, unknown>[] = [];
for (const tool of advertised) {
collectPatterns(tool.inputSchema, "inputSchema", tool.name, patterns);
+ collectCanonicalBase64Leaves(tool.inputSchema, canonicalBase64Leaves);
if (tool.outputSchema !== undefined)
collectPatterns(tool.outputSchema, "outputSchema", tool.name, patterns);
+ if (tool.outputSchema !== undefined)
+ collectCanonicalBase64Leaves(tool.outputSchema, canonicalBase64Leaves);
}
+ for (const pattern of [
+ BASE64_QUARTET_LENGTH_PATTERN,
+ INVALID_BASE64_CHARACTER_PATTERN,
+ INVALID_BASE64_PADDING_PATTERN,
+ INVALID_CANONICAL_BASE64_PAD_BITS_PATTERN,
+ ])
+ expect(
+ patterns.some(({ pattern: advertisedPattern }) =>
+ advertisedPattern.includes(pattern.source),
+ ),
+ pattern.source,
+ ).toBe(true);
+
// Annex B and `u` mode accept an unescaped `-`, `/` or `[` inside a character
// class, so REA's own schema validation cannot observe the defect. `v` mode
// rejects them, and a client that compiles advertised patterns rejects the
@@ -87,4 +138,19 @@ it("advertises portable NUL escapes and patterns that compile in all JS modes",
),
).toEqual([]);
expect(patterns.some(({ pattern }) => pattern.includes("\\x00"))).toBe(true);
+
+ expect(canonicalBase64Leaves.length).toBeGreaterThan(0);
+ const sdkValidator = new AjvJsonSchemaValidator();
+ const validators = canonicalBase64Leaves.map((schema) =>
+ sdkValidator.getValidator(schema),
+ );
+ for (const value of ["", "AA==", "AQID", "AB==", "AAA", "AA$=", "AA\n=="])
+ expect(
+ validators.map((validate) => validate(value).valid),
+ JSON.stringify(value),
+ ).toEqual(
+ canonicalBase64Leaves.map(
+ () => value === "" || value === "AA==" || value === "AQID",
+ ),
+ );
});