Reverse engineer anything with agents, from app behavior down to native binaries.
TypeScript agent-skillsai-agentsbinary-analysisclaude-codeclicodexcordisdecompilerdeveloper-toolsdshdsh-pluginghidrahopperllmmcpmodel-context-protocolreareverse-engineeringstatic-analysistouhou
fix(sqlite): attribute generated-column evaluation failures to the selection (#5879)
Kaoru0822bcfcd8b6e2ee
diff --git a/docs/sqlite-databases.md b/docs/sqlite-databases.mdindex 4a9a43e9c..2d9ad6c76 100644--- a/docs/sqlite-databases.md+++ b/docs/sqlite-databases.md@@ -57,7 +57,12 @@ triggers or arbitrary SQL.Rows are optional. `table` selects one exact ordinary-table name. `columns`(CLI repeated `--column`) selects exact visible column names in the requestedorder; omission selects all visible columns, including generated columns.-Unselected cells are not evaluated or projected. Malformed column-name bytes+Unselected cells are not evaluated or projected. A selected virtual generated+column whose expression needs a function REA's connection lacks (extensions and+application functions are not loaded), or raises an SQL error for a sampled+row, is refused as `capability_unavailable`. The error names the table and+each failing selected column with SQLite's message, and recommends only columns+whose sampled values were read; stored and other generated cells stay readable. Malformed column-name bytescause a format refusal when all columns are requested. In inventory scope, an explicit valid subset canstill be sampled without decoding unrelated malformed column names.Column selection requires adiff --git a/src/sqlite/SqliteDatabaseInspection.ts b/src/sqlite/SqliteDatabaseInspection.tsindex a41ce6559..12e3030aa 100644--- a/src/sqlite/SqliteDatabaseInspection.ts+++ b/src/sqlite/SqliteDatabaseInspection.ts@@ -10,9 +10,14 @@ import {} from "../domain/sqlite/sqliteDatabase.js";import {SQLITE_DATABASE_LIMITS,+ SqliteColumnEvaluationFailure,SqliteInspectionFailure,+ sqlitePrimaryResultCode,} from "./SqliteDatabaseLimits.js";+/** SQLite's generic SQL error, raised while compiling or evaluating an expression. */+const SQLITE_ERROR = 1;+const natural = z.bigint().nonnegative()@@ -419,6 +424,107 @@ const readSchema = (return { schema, selectedTable };};+/** Byte length of one cell's projected representation, without copying its value. */+const cellByteLength = (name: string): string => {+ const column = quoteIdentifier(name);+ return `CASE typeof(${column}) WHEN 'text' THEN length(CAST(${column} AS BLOB)) WHEN 'blob' THEN length(${column}) ELSE 0 END`;+};++/** Ordinary and stored generated columns, read without evaluating an expression; skips undecodable names. */+const storedColumnNames = (+ db: DatabaseSync,+ table: string,+ encoding: DatabaseEncoding,+): string[] => {+ const names: string[] = [];+ for (const raw of db+ .prepare(+ "SELECT CAST(name AS BLOB) AS name FROM temp.pragma_table_xinfo(?, 'main') WHERE hidden IN (0, 3) ORDER BY cid",+ )+ .iterate(table)) {+ const { name } = z.object({ name: z.instanceof(Uint8Array) }).parse(raw);+ try {+ names.push(decodeSchemaText(name, encoding, "column name"));+ } catch (cause: unknown) {+ // An unrelated malformed name cannot be selected exactly, so it is not recommended.+ if (!(cause instanceof SqliteInspectionFailure)) throw cause;+ }+ }+ return names;+};++/**+ * Attribute an SQL error from selected cells to the virtual generated columns+ * that raise it on this connection. Each column is probed alone over the same+ * sampled rows, so only columns whose probe succeeds are recommended; any+ * other cause keeps its original classification.+ */+const columnEvaluationFailure = (+ db: DatabaseSync,+ cause: unknown,+ selection: {+ readonly table: SelectedTableFacts;+ readonly columns: readonly string[];+ readonly source: string;+ readonly limit: number;+ readonly encoding: DatabaseEncoding;+ },+): unknown => {+ if (sqlitePrimaryResultCode(cause) !== SQLITE_ERROR) return cause;+ const { table, columns, source, limit, encoding } = selection;+ // The aggregate evaluates exactly the sampled cells, as the length pass does.+ const probe = (name: string): { readonly failure: unknown } | undefined => {+ try {+ db.prepare(+ `SELECT count(${cellByteLength(name)}) FROM (SELECT ${quoteIdentifier(name)} FROM ${source} LIMIT ?)`,+ ).get(limit);+ return undefined;+ } catch (failure: unknown) {+ return { failure };+ }+ };+ const failures = new Map<string, unknown>();+ for (const name of new Set(columns)) {+ const failed = probe(name);+ if (failed !== undefined) failures.set(name, failed.failure);+ }+ const generated = new Set(+ table.columns+ .filter((column) => column.hidden === 2)+ .map((column) => column.name),+ );+ if (+ ![...failures].some(+ ([name, failure]) =>+ generated.has(name) &&+ sqlitePrimaryResultCode(failure) === SQLITE_ERROR,+ )+ )+ return cause;+ const remaining = columns.filter((name) => !failures.has(name));+ const readableStoredColumns = (): string[] => {+ try {+ return storedColumnNames(db, table.name, encoding).filter(+ (name) => !failures.has(name) && probe(name) === undefined,+ );+ } catch {+ // Discovery only refines recovery; it must not replace the attributed failure.+ return [];+ }+ };+ return new SqliteColumnEvaluationFailure(+ `SQLite cannot evaluate selected column(s) of table ${JSON.stringify(table.name)} on REA's read-only connection: ${[+ ...failures,+ ]+ .map(+ ([name, failure]) =>+ `${JSON.stringify(name)}: ${failure instanceof Error ? failure.message : String(failure)}`,+ )+ .join("; ")}`,+ remaining.length > 0 ? remaining : readableStoredColumns(),+ );+};+const readRows = (db: DatabaseSync,input: InspectSqliteDatabaseInput,@@ -460,19 +566,6 @@ const readRows = (}).join(", ");const source = `main.${quoteIdentifier(input.table)} NOT INDEXED`;- const lengths = db.prepare(- `SELECT ${columns- .map((name) => {- const column = quoteIdentifier(name);- return `CASE typeof(${column}) WHEN 'text' THEN length(CAST(${column} AS BLOB)) WHEN 'blob' THEN length(${column}) ELSE 0 END`;- })- .join(", ")} FROM ${source} LIMIT ?`,- );- lengths.setReturnArrays(true);- for (const raw of lengths.iterate(limit))- budget.beforeProjection(cellByteLengths.parse(raw));- const statement = db.prepare(`SELECT ${projection} FROM ${source} LIMIT ?`);- statement.setReturnArrays(true);const rows: NonNullable<SqliteDatabase["rows"]> = {table: input.table,columns,@@ -482,13 +575,33 @@ const readRows = (truncated: false,order: "unspecified",};- for (const raw of statement.iterate(limit)) {- const rawRow = arrayRow.parse(raw);- // Hex expansion already belongs to the retained-output budget, before decoding/copying cells.- budget.beforeDecode(rawRow);- const values = rawRow.map((value) => cell(value, encoding));- budget.retain(values);- rows.values.push(values);+ // SQLite compiles and evaluates only selected cells, so an SQL error here is+ // attributed to the selected generated expressions rather than the file.+ try {+ const lengths = db.prepare(+ `SELECT ${columns.map(cellByteLength).join(", ")} FROM ${source} LIMIT ?`,+ );+ lengths.setReturnArrays(true);+ for (const raw of lengths.iterate(limit))+ budget.beforeProjection(cellByteLengths.parse(raw));+ const statement = db.prepare(`SELECT ${projection} FROM ${source} LIMIT ?`);+ statement.setReturnArrays(true);+ for (const raw of statement.iterate(limit)) {+ const rawRow = arrayRow.parse(raw);+ // Hex expansion already belongs to the retained-output budget, before decoding/copying cells.+ budget.beforeDecode(rawRow);+ const values = rawRow.map((value) => cell(value, encoding));+ budget.retain(values);+ rows.values.push(values);+ }+ } catch (cause: unknown) {+ throw columnEvaluationFailure(db, cause, {+ table,+ columns,+ source,+ limit,+ encoding,+ });}rows.returned_rows = rows.values.length;rows.truncated =diff --git a/src/sqlite/SqliteDatabaseLimits.ts b/src/sqlite/SqliteDatabaseLimits.tsindex 50ab3978f..3b6fbd6c2 100644--- a/src/sqlite/SqliteDatabaseLimits.ts+++ b/src/sqlite/SqliteDatabaseLimits.ts@@ -12,6 +12,7 @@ export class SqliteInspectionFailure extends Error {| "format"| "selection"| "column-selection"+ | "column-evaluation"| "unsupported"| "unavailable"| "output-limit"@@ -21,3 +22,20 @@ export class SqliteInspectionFailure extends Error {super(message);}}+/** Selected generated cells the private connection cannot compute; the database stays readable. */+export class SqliteColumnEvaluationFailure extends SqliteInspectionFailure {+ constructor(+ message: string,+ /** Selected or stored columns that remain readable without the failed expressions. */+ readonly readableColumns: readonly string[],+ ) {+ super("column-evaluation", message);+ }+}+/** SQLite's primary result code for a native node:sqlite failure, if it carries one. */+export const sqlitePrimaryResultCode = (cause: unknown): number | undefined =>+ cause instanceof Error &&+ "errcode" in cause &&+ typeof cause.errcode === "number"+ ? cause.errcode & 0xff+ : undefined;diff --git a/src/sqlite/SqliteDatabaseWorker.ts b/src/sqlite/SqliteDatabaseWorker.tsindex ebd2d5bac..c55528a5d 100644--- a/src/sqlite/SqliteDatabaseWorker.ts+++ b/src/sqlite/SqliteDatabaseWorker.ts@@ -4,7 +4,9 @@ import { inspectSqliteDatabaseInputSchema } from "../domain/sqlite/sqliteDatabasimport { inspectSqliteDatabaseSnapshot } from "./SqliteDatabaseInspection.js";import {SQLITE_DATABASE_LIMITS,+ SqliteColumnEvaluationFailure,SqliteInspectionFailure,+ sqlitePrimaryResultCode,} from "./SqliteDatabaseLimits.js";const requestSchema = z.strictObject({@@ -29,20 +31,28 @@ const main = async (): Promise<void> => {),};} catch (cause: unknown) {- const code =- cause instanceof Error && "errcode" in cause ? cause.errcode : undefined;- reply = {- ok: false,- reason:- cause instanceof SqliteInspectionFailure- ? cause.reason- : code === 7- ? "memory"- : code === 18- ? "output-limit"- : "format",- message: cause instanceof Error ? cause.message : String(cause),- };+ const code = sqlitePrimaryResultCode(cause);+ const message = cause instanceof Error ? cause.message : String(cause);+ reply =+ cause instanceof SqliteColumnEvaluationFailure+ ? {+ ok: false,+ reason: cause.reason,+ message,+ readable_columns: cause.readableColumns,+ }+ : {+ ok: false,+ reason:+ cause instanceof SqliteInspectionFailure+ ? cause.reason+ : code === 7+ ? "memory"+ : code === 18+ ? "output-limit"+ : "format",+ message,+ };}let encoded = JSON.stringify(reply);if (Buffer.byteLength(encoded) > SQLITE_DATABASE_LIMITS.outputBytes)diff --git a/src/sqlite/SqliteDatabaseWorkerExecution.ts b/src/sqlite/SqliteDatabaseWorkerExecution.tsindex bdd82ad1e..acabd5666 100644--- a/src/sqlite/SqliteDatabaseWorkerExecution.ts+++ b/src/sqlite/SqliteDatabaseWorkerExecution.ts@@ -39,7 +39,7 @@ const {...inspectionShape} = sqliteDatabaseSchema.shape;const inspectionSchema = z.strictObject(inspectionShape);-const replySchema = z.discriminatedUnion("ok", [+const replySchema = z.union([z.strictObject({ ok: z.literal(true), inspection: inspectionSchema }),z.strictObject({ok: z.literal(false),@@ -54,6 +54,12 @@ const replySchema = z.discriminatedUnion("ok", []),message: z.string(),}),+ z.strictObject({+ ok: z.literal(false),+ reason: z.literal("column-evaluation"),+ message: z.string(),+ readable_columns: z.array(z.string()),+ }),]);/** Replaceable launcher retains the shared owned-process supervision boundary. */export type SqliteWorkerLauncher = NonNullable<@@ -80,6 +86,23 @@ const rejectedReply = (reply.message,{ capturedOutput, userMessage: reply.message },);+ if (reply.reason === "column-evaluation") {+ const readable = reply.readable_columns;+ return new AnalysisCapabilityUnavailableError(+ "sqlite",+ operation,+ reply.message,+ {+ capturedOutput,+ userMessage: reply.message,+ remediationAction: `The database is readable; only the listed column values failed. REA's SQLite connection loads no extensions or application functions and disables schema trust. ${+ readable.length === 0+ ? "No other readable column was found"+ : `Select columns whose values were read: columns: ${JSON.stringify(readable)} (CLI: one --column per name)`+ }; omit table and use schema_scope: database to read the stored declarations without evaluating rows. Obtaining the failed values needs a SQLite connection able to evaluate them. Do not repeat the unchanged request.`,+ },+ );+ }if (reply.reason === "memory")return new AnalysisResourceConstraintError(operation,diff --git a/tests/acceptance/sqliteDatabaseCli.test.ts b/tests/acceptance/sqliteDatabaseCli.test.tsindex d871fe31a..5ce76ea82 100644--- a/tests/acceptance/sqliteDatabaseCli.test.ts+++ b/tests/acceptance/sqliteDatabaseCli.test.ts@@ -796,3 +796,98 @@ cliTest.runIf(SQLITE_NATIVE_LIMITS_AVAILABLE)(}},);++cliTest.runIf(SQLITE_NATIVE_LIMITS_AVAILABLE)(+ "attributes an unevaluable generated column to its selection, not the readable database path",+ async ({ cli }) => {+ const root = await createTestTempDirectory("rea-sqlite-generated-");+ const path = join(root, "hashes.db");+ const producer = new DatabaseSync(path);+ try {+ // The producing application supplies a function REA's connection lacks.+ producer.function(+ "application_digest",+ { deterministic: true },+ (value) => (typeof value === "string" ? `digest:${value}` : null),+ );+ producer.exec(`+ CREATE TABLE t(a TEXT, b TEXT GENERATED ALWAYS AS (application_digest(a)) VIRTUAL);+ INSERT INTO t(a) VALUES ('hello'), ('world');+ `);+ } finally {+ producer.close();+ }+ const before = await readFile(path);+ const { call } = await connectLocalToolsMcp();+ const message = `SQLite cannot evaluate selected column(s) of table "t" on REA's read-only connection: "b": unknown function: application_digest()`;+ const expectedError = {+ code: "capability_unavailable",+ retryable: false,+ message,+ details: { provider_id: "sqlite", reason: message },+ remediation: {+ action: expect.stringContaining(+ 'Select columns whose values were read: columns: ["a"] (CLI: one --column per name)',+ ),+ },+ };+ for (const columns of [["b", "a"], undefined]) {+ const result = await cli.run({+ arguments: [+ "inspect-sqlite-database",+ path,+ "--table",+ "t",+ ...(columns ?? []).flatMap((column) => ["--column", column]),+ "--json",+ ],+ environment: { REA_LOG_LEVEL: "silent" },+ });+ expect(result.exitCode, result.stdout + result.stderr).toBe(1);+ expect(result.json).toMatchObject(expectedError);+ expect(result.stdout).not.toContain("invalid_format");+ const response = await call("inspect_sqlite_database", {+ path,+ table: "t",+ ...(columns === undefined ? {} : { columns }),+ });+ expect(parseMcpToolError(response).error).toMatchObject(expectedError);+ }+ const base = await cli.run({+ arguments: [+ "inspect-sqlite-database",+ path,+ "--table",+ "t",+ "--column",+ "a",+ "--json",+ ],+ environment: { REA_LOG_LEVEL: "silent" },+ });+ expect(base.exitCode, base.stdout + base.stderr).toBe(0);+ expect(base.json).toMatchObject({+ normalized_result: { rows: { columns: ["a"], returned_rows: 2 } },+ });+ const schema = await call("inspect_sqlite_database", {+ path,+ schema_scope: "database",+ });+ expect(schema.isError).not.toBe(true);+ expect(schema.structuredContent).toMatchObject({+ normalized_result: {+ rows: null,+ schema: {+ tables: [+ {+ name: "t",+ sql: expect.stringContaining("application_digest(a)"),+ columns: [{ name: "a" }, { name: "b", hidden: 2 }],+ },+ ],+ },+ },+ });+ expect(await readFile(path)).toEqual(before);+ },+);diff --git a/tests/boundary/sqlite/sqliteDatabaseInspection.test.ts b/tests/boundary/sqlite/sqliteDatabaseInspection.test.tsindex 26fab7d6a..17af7d551 100644--- a/tests/boundary/sqlite/sqliteDatabaseInspection.test.ts+++ b/tests/boundary/sqlite/sqliteDatabaseInspection.test.ts@@ -364,6 +364,179 @@ it("rejects the combined representation of individually small generated cells",}});+it("attributes compile and row evaluation errors to each failing generated column", async () => {+ const workspace = await createTestWorkspace("rea-sqlite-evaluation-");+ onTestFinished(() => removeTestWorkspace(workspace.root));+ const path = workspace.path("private.db");+ // Writable schema plants an expression that fails only for the second+ // stored row; virtual cells are not stored, so record layout is unchanged.+ const producer = new DatabaseSync(path, { defensive: false });+ try {+ producer.function("application_digest", { deterministic: true }, (value) =>+ String(value),+ );+ producer.exec(`+ CREATE TABLE t(+ id INTEGER,+ payload TEXT,+ digest TEXT GENERATED ALWAYS AS (application_digest(payload)) VIRTUAL,+ stored_digest TEXT GENERATED ALWAYS AS (application_digest(payload)) STORED,+ field GENERATED ALWAYS AS (payload) VIRTUAL,+ upper_payload GENERATED ALWAYS AS (upper(payload)) VIRTUAL+ );+ INSERT INTO t(id, payload) VALUES (1, '{"k":1}'), (2, 'not json');+ PRAGMA writable_schema = ON;+ UPDATE sqlite_schema+ SET sql = replace(sql, 'field GENERATED ALWAYS AS (payload)', 'field GENERATED ALWAYS AS (json_extract(payload, ''$.k''))')+ WHERE name = 't';+ PRAGMA writable_schema = OFF;+ `);+ } finally {+ producer.close();+ }+ const failure = (input: { columns?: string[]; row_limit?: number }) => {+ try {+ inspectSqliteDatabaseSnapshot(path, { path, table: "t", ...input });+ } catch (cause: unknown) {+ return cause;+ }+ throw new Error("Expected a generated-column evaluation failure");+ };+ const prefix = `SQLite cannot evaluate selected column(s) of table "t" on REA's read-only connection: `;+ expect(failure({})).toMatchObject({+ reason: "column-evaluation",+ message: `${prefix}"digest": unknown function: application_digest(); "field": malformed JSON`,+ readableColumns: ["id", "payload", "stored_digest", "upper_payload"],+ });+ expect(failure({ columns: ["field", "id", "field"] })).toMatchObject({+ message: `${prefix}"field": malformed JSON`,+ readableColumns: ["id"],+ });+ // With no readable selection left, stored columns remain available without evaluation.+ expect(failure({ columns: ["digest"] })).toMatchObject({+ message: `${prefix}"digest": unknown function: application_digest()`,+ readableColumns: ["id", "payload", "stored_digest"],+ });+ // Only sampled rows are evaluated, and stored or built-in generated cells are read.+ expect(+ inspectSqliteDatabaseSnapshot(path, {+ path,+ table: "t",+ columns: ["field", "stored_digest", "upper_payload"],+ row_limit: 1,+ }).rows,+ ).toMatchObject({+ values: [+ [+ { type: "integer", value: "1" },+ { type: "text", value: '{"k":1}' },+ { type: "text", value: '{"K":1}' },+ ],+ ],+ truncated: true,+ });+});++const createApplicationDatabase = async (+ sql: string,+ edit?: (bytes: Buffer) => void,+): Promise<string> => {+ const workspace = await createTestWorkspace("rea-sqlite-evaluation-");+ onTestFinished(() => removeTestWorkspace(workspace.root));+ const path = workspace.path("private.db");+ const producer = new DatabaseSync(path);+ try {+ // The producing application supplies a function REA's connection lacks.+ producer.function("application_digest", { deterministic: true }, (value) =>+ String(value),+ );+ producer.exec(sql);+ } finally {+ producer.close();+ }+ if (edit !== undefined) {+ const bytes = await readFile(path);+ edit(bytes);+ await writeFile(path, bytes);+ }+ return path;+};++it("never recommends a selected column whose own probe fails for another reason", async () => {+ const path = await createApplicationDatabase(`+ CREATE TABLE t(+ id INTEGER,+ digest TEXT GENERATED ALWAYS AS (application_digest(id)) VIRTUAL,+ payload BLOB GENERATED ALWAYS AS (zeroblob(72 * 1024 * 1024)) VIRTUAL+ );+ INSERT INTO t(id) VALUES (1);+ `);+ expect(() =>+ inspectSqliteDatabaseSnapshot(path, {+ path,+ table: "t",+ columns: ["payload", "digest", "id"],+ }),+ ).toThrow(+ expect.objectContaining({+ reason: "column-evaluation",+ message: `SQLite cannot evaluate selected column(s) of table "t" on REA's read-only connection: "payload": string or blob too big; "digest": unknown function: application_digest()`,+ readableColumns: ["id"],+ }),+ );+});++it("recommends stored columns outside an inventory-scope selection without decoding malformed names", async () => {+ const path = await createApplicationDatabase(+ `+ CREATE TABLE t(+ id INTEGER,+ "col" TEXT,+ payload TEXT,+ digest TEXT GENERATED ALWAYS AS (application_digest(payload)) VIRTUAL+ );+ INSERT INTO t(id, "col", payload) VALUES (1, 'kept', 'value');+ `,+ (bytes) => {+ const marker = Buffer.from('"col"');+ const offset = bytes.indexOf(marker);+ if (offset < 0 || bytes.indexOf(marker, offset + 1) !== -1)+ throw new Error("Expected unique authored column marker");+ bytes[offset + 1] = 0xff;+ },+ );+ expect(() =>+ inspectSqliteDatabaseSnapshot(path, {+ path,+ table: "t",+ schema_scope: "inventory",+ columns: ["digest"],+ }),+ ).toThrow(+ expect.objectContaining({+ reason: "column-evaluation",+ message: `SQLite cannot evaluate selected column(s) of table "t" on REA's read-only connection: "digest": unknown function: application_digest()`,+ readableColumns: ["id", "payload"],+ }),+ );+ // The recommended stored columns are themselves selectable in the same scope.+ expect(+ inspectSqliteDatabaseSnapshot(path, {+ path,+ table: "t",+ schema_scope: "inventory",+ columns: ["id", "payload"],+ }).rows,+ ).toMatchObject({+ values: [+ [+ { type: "integer", value: "1" },+ { type: "text", value: "value" },+ ],+ ],+ });+});+it("samples above 1000 rows and safe integer limits within the byte budget", async () => {const path = await createDatabase(`CREATE TABLE items(value INTEGER);WITH RECURSIVE numbers(value) AS (SELECT 1 UNION ALL SELECT value + 1 FROM numbers WHERE value < 1002)