Skip to content

armgit / rea

Public Imported from GitHub
0

Reverse engineer anything with agents, from app behavior down to native binaries.

fix(json): name a leading byte-order mark in rejected JSON input (#1510)

Kaoru0822a93630372393
diff --git a/src/application/Utf8JsonInput.ts b/src/application/Utf8JsonInput.ts
index 86204e3c9..0e8ac8bb6 100644
--- a/src/application/Utf8JsonInput.ts
+++ b/src/application/Utf8JsonInput.ts
@@ -7,6 +7,13 @@ import { safeParseJson, type SafeJsonParseResult } from "../domain/safeJson.js";
export const JSON_INPUT_RESOURCE_REMEDIATION =
"Provide a smaller valid JSON value. For Evidence workflows, re-analyze a smaller selection of the original target and use its complete Evidence; splitting JSON text or trimming Evidence fields does not produce valid workflow input.";
+/**
+ * Strict JSON input keeps rejecting a leading byte-order mark, but names it:
+ * the parser's own message quotes the invisible U+FEFF as the bad token.
+ */
+export const JSON_BYTE_ORDER_MARK_MESSAGE =
+ "JSON input begins with a UTF-8 byte-order mark (EF BB BF), which strict JSON input does not accept. Save the file as UTF-8 without a byte-order mark.";
+
/** Decode JSON bytes without confusing a runtime string limit with malformed input. */
export const parseUtf8Json = (
bytes: Uint8Array,
@@ -77,5 +84,7 @@ export const parseUtf8Json = (
cause,
};
}
+ if (text.startsWith("\uFEFF"))
+ return { ok: false, error: JSON_BYTE_ORDER_MARK_MESSAGE, cause: undefined };
return safeParseJson(text);
};
diff --git a/src/application/process/ProcessCli.test.ts b/src/application/process/ProcessCli.test.ts
index 88ab756ad..34e777d31 100644
--- a/src/application/process/ProcessCli.test.ts
+++ b/src/application/process/ProcessCli.test.ts
@@ -6,6 +6,7 @@ import {
captureProcessScenarioFile,
projectProcessCliError,
} from "./ProcessCli.js";
+import { JSON_BYTE_ORDER_MARK_MESSAGE } from "../Utf8JsonInput.js";
const roots: string[] = [];
afterEach(async () => {
@@ -87,4 +88,14 @@ describe("process CLI input diagnostics", () => {
}),
});
});
+
+ it("names a leading byte-order mark instead of quoting it as a token", async () => {
+ const path = join(await fixture(), "scenario.json");
+ await writeFile(path, '\uFEFF{"executable":"/bin/echo"}');
+ expect(await captureProcessScenarioFile(path)).toEqual({
+ error: "Process command failed",
+ category: "invalid_input",
+ message: `Process input file is not valid UTF-8 JSON: ${path} (${JSON_BYTE_ORDER_MARK_MESSAGE}). Repair the file, then try again.`,
+ });
+ });
});
diff --git a/src/application/process/ProcessCli.ts b/src/application/process/ProcessCli.ts
index 135953073..c7c4d479b 100644
--- a/src/application/process/ProcessCli.ts
+++ b/src/application/process/ProcessCli.ts
@@ -18,6 +18,7 @@ import { parseProcessCapture } from "../../domain/process/processCaptureParsing.
import { captureProcessScenario } from "../../process/capture/ProcessHarness.js";
import { PROCESS_PROVIDER } from "../../domain/process/processEvidenceProvider.js";
import { createProcessCaptureEvidence } from "./ProcessEvidence.js";
+import { JSON_BYTE_ORDER_MARK_MESSAGE } from "../Utf8JsonInput.js";
/** Safe process-command failure returned to the CLI adapter. */
export interface ProcessCliErrorOutput {
@@ -181,6 +182,8 @@ const readJson = async (path: string): Promise<unknown> => {
fatal: true,
ignoreBOM: true,
}).decode(bytes);
+ if (text.startsWith("\uFEFF"))
+ throw new Error(JSON_BYTE_ORDER_MARK_MESSAGE);
const parsed: unknown = JSON.parse(text);
return parsed;
} catch (cause: unknown) {
diff --git a/src/cliJsonFile.ts b/src/cliJsonFile.ts
index ea75f7c29..c0d474f69 100644
--- a/src/cliJsonFile.ts
+++ b/src/cliJsonFile.ts
@@ -8,6 +8,7 @@ import streamValues from "stream-json/streamers/stream-values.js";
import { withRegularFile } from "./application/RegularFileRead.js";
import {
+ JSON_BYTE_ORDER_MARK_MESSAGE,
JSON_INPUT_RESOURCE_REMEDIATION,
parseUtf8Json,
} from "./application/Utf8JsonInput.js";
@@ -211,12 +212,22 @@ async function* decodedChunks(
prefix?: Uint8Array,
): AsyncGenerator<string> {
const decoder = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true });
+ let leading = true;
+ const checked = (text: string): string => {
+ if (leading && text.length > 0) {
+ leading = false;
+ if (text.startsWith("\uFEFF")) throw new JsonByteOrderMarkError();
+ }
+ return text;
+ };
if (prefix !== undefined)
for (let offset = 0; offset < prefix.length; offset += READ_CHUNK_BYTES) {
signal?.throwIfAborted();
- yield decoder.decode(prefix.subarray(offset, offset + READ_CHUNK_BYTES), {
- stream: true,
- });
+ yield checked(
+ decoder.decode(prefix.subarray(offset, offset + READ_CHUNK_BYTES), {
+ stream: true,
+ }),
+ );
}
const bytes = Buffer.allocUnsafe(READ_CHUNK_BYTES);
while (true) {
@@ -224,9 +235,18 @@ async function* decodedChunks(
const read = await handle.read(bytes, 0, bytes.length, null);
signal?.throwIfAborted();
if (read.bytesRead === 0) break;
- yield decoder.decode(bytes.subarray(0, read.bytesRead), { stream: true });
+ yield checked(
+ decoder.decode(bytes.subarray(0, read.bytesRead), { stream: true }),
+ );
+ }
+ yield checked(decoder.decode());
+}
+
+/** A streamed JSON file begins with a byte-order mark; reported as invalid JSON. */
+class JsonByteOrderMarkError extends SyntaxError {
+ constructor() {
+ super(JSON_BYTE_ORDER_MARK_MESSAGE);
}
- yield decoder.decode();
}
const readPrefix = async (
diff --git a/tests/boundary/filesystem/cliJsonInput.test.ts b/tests/boundary/filesystem/cliJsonInput.test.ts
index 099c95b25..dd4809ea8 100644
--- a/tests/boundary/filesystem/cliJsonInput.test.ts
+++ b/tests/boundary/filesystem/cliJsonInput.test.ts
@@ -9,6 +9,7 @@ import { createCli } from "../../../src/cli.js";
import { parseCliJsonInput } from "../../../src/cliJsonInput.js";
import { readCliJsonFile } from "../../../src/cliJsonFile.js";
import { analysisCliErrorEnvelopeSchema } from "../../../src/contracts/errorSchemas.js";
+import { JSON_BYTE_ORDER_MARK_MESSAGE } from "../../../src/application/Utf8JsonInput.js";
import { readWithoutFifoWriter } from "../../fixtures/fifoInput.js";
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
@@ -160,6 +161,21 @@ describe("CLI JSON streamed input", () => {
}
});
+ it("names a leading byte-order mark in small and streamed files", async () => {
+ const root = await createTestTempDirectory("rea-json-input-bom-");
+ const path = join(root, "input.json");
+ for (const body of ["{}", `${" ".repeat(9 * 1024 * 1024)}{}`]) {
+ await writeFile(path, `\uFEFF${body}`);
+ expect(await parseCliJsonInput(path, "test-input")).toMatchObject({
+ ok: false,
+ error: {
+ input_reason: "invalid-json",
+ details: { issues: [{ message: JSON_BYTE_ORDER_MARK_MESSAGE }] },
+ },
+ });
+ }
+ });
+
it.each([Buffer.from([0xf0, 0x28, 0x8c, 0xbc]), Buffer.from([0xe2, 0x82])])(
"rejects malformed and incomplete UTF-8 at a read boundary (%j)",
async (invalid) => {