Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
feat(frontend): handle protobuf bytes in page agent api tools (#19941)
p0ny43c130ff2499
diff --git a/frontend/scripts/generate_openapi_index.js b/frontend/scripts/generate_openapi_index.jsindex 047bd11ff8..8db57046dd 100644--- a/frontend/scripts/generate_openapi_index.js+++ b/frontend/scripts/generate_openapi_index.js@@ -26,34 +26,123 @@ function refName(ref) {return parts[parts.length - 1];}-// Extract property type and description from a schema property object.-function extractPropertyTypeAndDesc(prop) {- if (!prop) return { type: "object", description: "" };+function isObjectSchema(schemaDef) {+ return (+ !!schemaDef &&+ typeof schemaDef === "object" &&+ (schemaDef.type === "object" ||+ !!schemaDef.properties ||+ Array.isArray(schemaDef.allOf) ||+ Array.isArray(schemaDef.oneOf))+ );+}++function mergeObjectShape(schemaDef, mergedProperties, mergedRequired, options) {+ if (!isObjectSchema(schemaDef)) {+ return;+ }++ if (schemaDef.properties) {+ Object.assign(mergedProperties, schemaDef.properties);+ }- // If the property itself is a $ref- if (prop.$ref) {- return { type: refName(prop.$ref), description: prop.description || "" };+ if (options.includeRequired && Array.isArray(schemaDef.required)) {+ for (const requiredName of schemaDef.required) {+ mergedRequired.add(requiredName);+ }}- let propType = "object";- const rawType = Array.isArray(prop.type)- ? prop.type.find((t) => t !== "null")- : prop.type;- if (rawType) propType = rawType;-- // For arrays, get the item type- if (propType === "array" && prop.items) {- if (prop.items.$ref) {- propType = `array<${refName(prop.items.$ref)}>`;- } else if (prop.items.type) {- const itemType = Array.isArray(prop.items.type)- ? prop.items.type.find((t) => t !== "null")- : prop.items.type;- if (itemType) propType = `array<${itemType}>`;+ if (Array.isArray(schemaDef.allOf)) {+ for (const part of schemaDef.allOf) {+ mergeObjectShape(part, mergedProperties, mergedRequired, options);}}- return { type: propType, description: prop.description || "" };+ if (Array.isArray(schemaDef.oneOf)) {+ for (const branch of schemaDef.oneOf) {+ mergeObjectShape(branch, mergedProperties, mergedRequired, {+ includeRequired: false,+ });+ }+ }+}++function collectObjectShape(schemaDef) {+ const mergedProperties = {};+ const mergedRequired = new Set();++ mergeObjectShape(schemaDef, mergedProperties, mergedRequired, {+ includeRequired: true,+ });++ return {+ properties: mergedProperties,+ required: mergedRequired,+ };+}++// Extract property metadata from a schema property object.+function extractPropertyInfo(propName, propDef, requiredSet) {+ const info = {+ name: propName,+ type: "object",+ };++ if (requiredSet.has(propName)) {+ info.required = true;+ }++ if (!propDef) {+ return info;+ }++ if (Array.isArray(propDef.oneOf)) {+ const nonNullBranches = propDef.oneOf.filter(+ (branch) => branch && branch.type !== "null"+ );+ if (nonNullBranches.length === 1) {+ const unionDef = { ...nonNullBranches[0] };+ if (propDef.description && !unionDef.description) {+ unionDef.description = propDef.description;+ }+ return extractPropertyInfo(propName, unionDef, requiredSet);+ }+ }++ if (propDef.$ref) {+ info.type = refName(propDef.$ref);+ if (propDef.description) info.description = propDef.description;+ return info;+ }++ const rawType = Array.isArray(propDef.type)+ ? propDef.type.find((t) => t !== "null")+ : propDef.type;+ if (rawType) {+ info.type = rawType;+ }+ if (propDef.format) {+ info.format = propDef.format;+ }+ if (propDef.description) {+ info.description = propDef.description;+ }+ if (info.type === "array" && propDef.items) {+ info.items = extractPropertyInfo("item", propDef.items, new Set());+ info.type = `array<${info.items.type}>`;+ }+ if (+ info.type === "object" &&+ propDef.additionalProperties &&+ typeof propDef.additionalProperties === "object"+ ) {+ info.additionalProperties = extractPropertyInfo(+ "value",+ propDef.additionalProperties,+ new Set()+ );+ }+ return info;}// --- Main ---@@ -114,13 +203,12 @@ for (const [pathStr, pathItem] of Object.entries(paths)) {});}-// 2. Extract schemas (only bytebase.v1.* component schemas)+// 2. Extract schemasconst schemas = {};const componentSchemas =(spec.components && spec.components.schemas) || {};for (const [schemaName, schemaDef] of Object.entries(componentSchemas)) {- if (!schemaName.startsWith("bytebase.v1.")) continue;if (!schemaDef) continue;// Handle enum types@@ -133,33 +221,17 @@ for (const [schemaName, schemaDef] of Object.entries(componentSchemas)) {continue;}- // Merge allOf properties into a flat properties object- let mergedProperties = schemaDef.properties;- let mergedRequired = schemaDef.required || [];- if (schemaDef.allOf && Array.isArray(schemaDef.allOf)) {- mergedProperties = {};- for (const part of schemaDef.allOf) {- if (part.properties) {- Object.assign(mergedProperties, part.properties);- }- if (part.required) {- mergedRequired = mergedRequired.concat(part.required);- }- }- }+ const { properties: mergedProperties, required: mergedRequired } =+ collectObjectShape(schemaDef);// Handle object types with propertiesif (!mergedProperties || Object.keys(mergedProperties).length === 0) continue;- const requiredSet = new Set(mergedRequired);+ const requiredSet = mergedRequired;const properties = [];for (const [propName, propDef] of Object.entries(mergedProperties)) {- const { type, description } = extractPropertyTypeAndDesc(propDef);- const prop = { name: propName, type };- if (description) prop.description = description;- if (requiredSet.has(propName)) prop.required = true;- properties.push(prop);+ properties.push(extractPropertyInfo(propName, propDef, requiredSet));}// Sort properties by name for consistent output@@ -198,13 +270,16 @@ lines.push(" type: string;"," description?: string;"," required?: boolean;",+ " format?: string;",+ " items?: PropertyInfo;",+ " additionalProperties?: PropertyInfo;","}","","export interface SchemaInfo {"," type: \"object\" | \"enum\";"," description: string;"," properties?: PropertyInfo[];",- " values?: string[];",+ " values?: Array<string | number>;","}","");diff --git a/frontend/src/react/plugins/agent/logic/tools/callApi.test.ts b/frontend/src/react/plugins/agent/logic/tools/callApi.test.tsindex 4e9cdfc194..bfd4d74805 100644--- a/frontend/src/react/plugins/agent/logic/tools/callApi.test.ts+++ b/frontend/src/react/plugins/agent/logic/tools/callApi.test.ts@@ -1,8 +1,11 @@import { beforeEach, describe, expect, test, vi } from "vitest";+import { type SchemaInfo, schemas } from "./gen/openapi-index";const mocks = vi.hoisted(() => ({fetch: vi.fn(),getEndpointPath: vi.fn(),+ getRequestSchema: vi.fn(),+ getSchema: vi.fn(),refreshTokens: vi.fn(),}));@@ -12,9 +15,11 @@ vi.mock("@/connect/refreshToken", () => ({vi.mock("./searchApi", () => ({getEndpointPath: mocks.getEndpointPath,+ getRequestSchema: mocks.getRequestSchema,+ getSchema: mocks.getSchema,}));-import { callApi } from "./callApi";+import { __testOnly, callApi } from "./callApi";const createResponse = ({status,@@ -32,13 +37,23 @@ const createResponse = ({: vi.fn().mockResolvedValue(body),}) as never;+const getRequestBody = (): unknown => {+ const request = mocks.fetch.mock.calls.at(-1)?.[1];+ const body = request && "body" in request ? request.body : undefined;+ return JSON.parse(typeof body === "string" ? body : "{}");+};+describe("callApi", () => {beforeEach(() => {vi.stubGlobal("fetch", mocks.fetch);mocks.fetch.mockReset();mocks.getEndpointPath.mockReset();+ mocks.getRequestSchema.mockReset();+ mocks.getSchema.mockReset();mocks.refreshTokens.mockReset();mocks.getEndpointPath.mockReturnValue("/bytebase.v1.SQLService/Query");+ mocks.getRequestSchema.mockReturnValue(undefined);+ mocks.getSchema.mockReturnValue(undefined);});test("refreshes once and retries unauthenticated requests", async () => {@@ -154,4 +169,243 @@ describe("callApi", () => {});expect(mocks.refreshTokens).not.toHaveBeenCalled();});++ test("coerces nested byte fields to base64 before sending the request", async () => {+ mocks.getEndpointPath.mockReturnValue(+ "/bytebase.v1.WorksheetService/CreateWorksheet"+ );+ mocks.getRequestSchema.mockReturnValue(+ schemas["bytebase.v1.CreateWorksheetRequest"]+ );+ mocks.getSchema.mockImplementation(+ (schemaName: string) => schemas[schemaName]+ );+ mocks.fetch.mockResolvedValue(+ createResponse({+ status: 200,+ body: { name: "projects/demo/worksheets/1" },+ })+ );++ await callApi({+ operationId: "WorksheetService/CreateWorksheet",+ body: {+ parent: "projects/demo",+ worksheet: {+ content: "select 1;",+ title: "demo",+ },+ },+ });++ expect(getRequestBody()).toEqual({+ parent: "projects/demo",+ worksheet: {+ content: "c2VsZWN0IDE7",+ title: "demo",+ },+ });+ });++ test("coerces byte fields inside arrays using the real batch create sheets schema", async () => {+ mocks.getEndpointPath.mockReturnValue(+ "/bytebase.v1.SheetService/BatchCreateSheets"+ );+ mocks.getRequestSchema.mockReturnValue(+ schemas["bytebase.v1.BatchCreateSheetsRequest"]+ );+ mocks.getSchema.mockImplementation(+ (schemaName: string) => schemas[schemaName]+ );+ mocks.fetch.mockResolvedValue(+ createResponse({+ status: 200,+ body: { sheets: [] },+ })+ );++ await callApi({+ operationId: "SheetService/BatchCreateSheets",+ body: {+ parent: "projects/demo",+ requests: [+ {+ parent: "projects/demo",+ sheet: {+ content: "select 1;",+ title: "sheet one",+ },+ },+ {+ parent: "projects/demo",+ sheet: {+ content: "select 2;",+ title: "sheet two",+ },+ },+ ],+ },+ });++ expect(getRequestBody()).toEqual({+ parent: "projects/demo",+ requests: [+ {+ parent: "projects/demo",+ sheet: {+ content: "c2VsZWN0IDE7",+ title: "sheet one",+ },+ },+ {+ parent: "projects/demo",+ sheet: {+ content: "c2VsZWN0IDI7",+ title: "sheet two",+ },+ },+ ],+ });+ });++ test("coerces byte fields through external oneof schemas in batch deparse requests", async () => {+ mocks.getEndpointPath.mockReturnValue(+ "/bytebase.v1.CelService/BatchDeparse"+ );+ mocks.getRequestSchema.mockReturnValue(+ schemas["bytebase.v1.BatchDeparseRequest"]+ );+ mocks.getSchema.mockImplementation(+ (schemaName: string) => schemas[schemaName]+ );+ mocks.fetch.mockResolvedValue(+ createResponse({+ status: 200,+ body: { expressions: [] },+ })+ );++ await callApi({+ operationId: "CelService/BatchDeparse",+ body: {+ expressions: [+ {+ id: "1",+ constExpr: {+ bytesValue: "hello",+ },+ },+ ],+ },+ });++ expect(getRequestBody()).toEqual({+ expressions: [+ {+ id: "1",+ constExpr: {+ bytesValue: "aGVsbG8=",+ },+ },+ ],+ });+ });++ test("coerces byte fields through top-level oneof sasl config branches", async () => {+ mocks.getEndpointPath.mockReturnValue(+ "/bytebase.v1.InstanceService/AddDataSource"+ );+ mocks.getRequestSchema.mockReturnValue(+ schemas["bytebase.v1.AddDataSourceRequest"]+ );+ mocks.getSchema.mockImplementation(+ (schemaName: string) => schemas[schemaName]+ );+ mocks.fetch.mockResolvedValue(+ createResponse({+ status: 200,+ body: { name: "instances/demo/dataSources/readonly" },+ })+ );++ await callApi({+ operationId: "InstanceService/AddDataSource",+ body: {+ name: "instances/demo",+ dataSource: {+ saslConfig: {+ krbConfig: {+ primary: "postgres",+ realm: "EXAMPLE.COM",+ keytab: "keytab-bytes",+ },+ },+ },+ },+ });++ expect(getRequestBody()).toEqual({+ name: "instances/demo",+ dataSource: {+ saslConfig: {+ krbConfig: {+ primary: "postgres",+ realm: "EXAMPLE.COM",+ keytab: "a2V5dGFiLWJ5dGVz",+ },+ },+ },+ });+ });++ test("coerces byte fields inside map values through the helper path", () => {+ const schema: SchemaInfo = {+ type: "object",+ description: "test schema for map coercion",+ properties: [+ {+ name: "entries",+ type: "object",+ additionalProperties: {+ name: "value",+ type: "test.Entry",+ },+ },+ ],+ };++ mocks.getSchema.mockImplementation((schemaName: string) => {+ if (schemaName === "test.Entry") {+ return {+ type: "object",+ description: "test entry schema",+ properties: [+ {+ name: "content",+ type: "string",+ format: "byte",+ },+ ],+ };+ }+ return schemas[schemaName];+ });++ expect(+ __testOnly.coerceRequestBody(+ {+ entries: {+ alpha: { content: "select 1;" },+ beta: { content: "select 2;" },+ },+ },+ schema+ )+ ).toEqual({+ entries: {+ alpha: { content: "c2VsZWN0IDE7" },+ beta: { content: "c2VsZWN0IDI7" },+ },+ });+ });});diff --git a/frontend/src/react/plugins/agent/logic/tools/callApi.ts b/frontend/src/react/plugins/agent/logic/tools/callApi.tsindex 5479ebf6a4..b8cf65486d 100644--- a/frontend/src/react/plugins/agent/logic/tools/callApi.ts+++ b/frontend/src/react/plugins/agent/logic/tools/callApi.ts@@ -1,5 +1,6 @@import { refreshTokens } from "@/connect/refreshToken";-import { getEndpointPath } from "./searchApi";+import type { PropertyInfo, SchemaInfo } from "./gen/openapi-index";+import { getEndpointPath, getRequestSchema, getSchema } from "./searchApi";const baseAddress = import.meta.env.BB_GRPC_LOCAL || window.location.origin;const REFRESH_PATH = "/bytebase.v1.AuthService/Refresh";@@ -15,6 +16,97 @@ interface ParsedApiResponse {error?: string;}+const textEncoder = new TextEncoder();++const isPlainObject = (value: unknown): value is Record<string, unknown> =>+ typeof value === "object" && value !== null && !Array.isArray(value);++const encodeUtf8ToBase64 = (value: string): string => {+ const bytes = textEncoder.encode(value);+ let binary = "";+ for (const byte of bytes) {+ binary += String.fromCharCode(byte);+ }+ return btoa(binary);+};++const getArrayItemSchema = (+ property: PropertyInfo+): PropertyInfo | undefined => {+ if (property.items) {+ return property.items;+ }++ if (property.type.startsWith("array<") && property.type.endsWith(">")) {+ return {+ name: "item",+ type: property.type.slice(6, -1),+ };+ }++ return undefined;+};++const coerceRequestValue = (+ value: unknown,+ property?: PropertyInfo,+ schema?: SchemaInfo+): unknown => {+ if (property?.type === "string" && property.format === "byte") {+ return typeof value === "string" ? encodeUtf8ToBase64(value) : value;+ }++ const resolvedSchema =+ schema ??+ (property && !property.type.startsWith("array<")+ ? getSchema(property.type)+ : undefined);++ if (Array.isArray(value)) {+ const itemSchema = property ? getArrayItemSchema(property) : undefined;+ return itemSchema+ ? value.map((item) => coerceRequestValue(item, itemSchema))+ : value;+ }++ if (!isPlainObject(value)) {+ return value;+ }++ if (property?.additionalProperties) {+ return Object.fromEntries(+ Object.entries(value).map(([key, itemValue]) => [+ key,+ coerceRequestValue(itemValue, property.additionalProperties),+ ])+ );+ }++ if (resolvedSchema?.type !== "object" || !resolvedSchema.properties) {+ return value;+ }++ const propertiesByName = new Map(+ resolvedSchema.properties.map((prop) => [prop.name, prop])+ );++ return Object.fromEntries(+ Object.entries(value).map(([key, itemValue]) => {+ const childProperty = propertiesByName.get(key);+ return [key, coerceRequestValue(itemValue, childProperty)];+ })+ );+};++const coerceRequestBody = (+ body: Record<string, unknown>,+ schema?: SchemaInfo+): unknown => coerceRequestValue(body, undefined, schema);++export const __testOnly = {+ coerceRequestBody,+};+const parseResponse = async (response: Response): Promise<ParsedApiResponse> => {@@ -87,7 +179,9 @@ export async function callApi(args: CallApiArgs): Promise<string> {const controller = new AbortController();const timeoutId = setTimeout(() => controller.abort(), 30_000);- const body = JSON.stringify(args.body ?? {});+ const body = JSON.stringify(+ coerceRequestBody(args.body ?? {}, getRequestSchema(args.operationId))+ );try {let result = await fetchApi({diff --git a/frontend/src/react/plugins/agent/logic/tools/index.test.ts b/frontend/src/react/plugins/agent/logic/tools/index.test.tsindex 8a03a74968..a494ed7d18 100644--- a/frontend/src/react/plugins/agent/logic/tools/index.test.ts+++ b/frontend/src/react/plugins/agent/logic/tools/index.test.ts@@ -197,6 +197,54 @@ describe("agent tools ask_user", () => {});});+describe("agent tools search_api", () => {+ test("renders byte fields as bytes with the UTF-8 call_api note", async () => {+ const { searchApi: actualSearchApi } =+ await vi.importActual<typeof import("./searchApi")>("./searchApi");++ const result = await actualSearchApi({+ operationId: "SheetService/CreateSheet",+ });++ const requestBodyStart = result.indexOf("### Request Body");+ const responseBodyStart = result.indexOf("### Response Body");+ const requestBody =+ requestBodyStart >= 0+ ? result.slice(+ requestBodyStart,+ responseBodyStart >= 0 ? responseBodyStart : undefined+ )+ : "";+ const responseBody =+ responseBodyStart >= 0 ? result.slice(responseBodyStart) : "";+ const requestNote =+ "request body includes protobuf bytes fields; plain strings passed to call_api are UTF-8 encoded automatically.";+ const requestNoteCount = requestBody.split(requestNote).length - 1;++ expect(requestBody).toContain('"sheet": bytebase.v1.Sheet');+ expect(requestBody).toContain(requestNote);+ expect(requestNoteCount).toBe(1);+ expect(responseBody).toContain('"content": bytes');+ expect(responseBody).not.toContain(requestNote);+ });++ test("detects transitive byte fields through external oneof request schemas", async () => {+ const { searchApi: actualSearchApi } =+ await vi.importActual<typeof import("./searchApi")>("./searchApi");++ const result = await actualSearchApi({+ operationId: "CelService/BatchDeparse",+ });++ expect(result).toContain(+ '"expressions": array<google.api.expr.v1alpha1.Expr>'+ );+ expect(result).toContain(+ "Note: request body includes protobuf bytes fields; plain strings passed to call_api are UTF-8 encoded automatically."+ );+ });+});+describe("agent tools concurrency guard", () => {test("blocks concurrent page-changing tools across threads", async () => {let releaseNavigation: (() => void) | undefined;diff --git a/frontend/src/react/plugins/agent/logic/tools/searchApi.ts b/frontend/src/react/plugins/agent/logic/tools/searchApi.tsindex 0db2718051..72f33aeaf5 100644--- a/frontend/src/react/plugins/agent/logic/tools/searchApi.ts+++ b/frontend/src/react/plugins/agent/logic/tools/searchApi.ts@@ -73,11 +73,31 @@ class OpenAPIIndex {const index = new OpenAPIIndex();+function getSchemaNameFromRef(schemaRef: string): string {+ const parts = schemaRef.split("/");+ return parts[parts.length - 1];+}+// Resolve operationId to HTTP path (used by callApi).export function getEndpointPath(operationId: string): string | undefined {return index.getEndpoint(operationId)?.path;}+export function getSchema(schemaNameOrRef: string): SchemaInfo | undefined {+ const schemaName = schemaNameOrRef.startsWith("#/")+ ? getSchemaNameFromRef(schemaNameOrRef)+ : schemaNameOrRef;+ return schemas[schemaName];+}++export function getRequestSchema(operationId: string): SchemaInfo | undefined {+ const schemaRef = index.getEndpoint(operationId)?.requestSchemaRef;+ if (!schemaRef) {+ return undefined;+ }+ return getSchema(schemaRef);+}+// --- Formatting helpers (matching Go tool_search.go) ---function truncate(s: string, max: number): string {@@ -85,19 +105,77 @@ function truncate(s: string, max: number): string {return s.slice(0, max) + "...";}+function formatPropertyType(prop: PropertyInfo): string {+ if (prop.format === "byte") {+ return "bytes";+ }+ return prop.type;+}+function formatProperty(prop: PropertyInfo): string {const required = prop.required ? " (required)" : "";+ const type = formatPropertyType(prop);let desc = "";+const shortDesc = TYPE_DESCRIPTIONS[prop.type];if (shortDesc) {- desc = ` // ${shortDesc}`;+ desc = desc ? `${desc}; ${shortDesc}` : ` // ${shortDesc}`;} else if (prop.description) {const clean = prop.description.replace(/[\n\r]/g, " ");- desc = ` // ${truncate(clean, 97)}`;+ desc = desc+ ? `${desc}; ${truncate(clean, 97)}`+ : ` // ${truncate(clean, 97)}`;+ }++ return ` "${prop.name}": ${type}${required}${desc}`;+}++function hasBytesFieldInProperty(+ prop: PropertyInfo,+ seen: Set<string>+): boolean {+ if (prop.format === "byte") {+ return true;+ }++ if (prop.items && hasBytesFieldInProperty(prop.items, seen)) {+ return true;+ }++ if (+ prop.additionalProperties &&+ hasBytesFieldInProperty(prop.additionalProperties, seen)+ ) {+ return true;+ }++ const schema = getSchema(prop.type);+ if (!schema || schema.type !== "object") {+ return false;+ }++ if (seen.has(prop.type)) {+ return false;+ }+ const nextSeen = new Set(seen);+ nextSeen.add(prop.type);++ return (+ schema.properties?.some((child) =>+ hasBytesFieldInProperty(child, nextSeen)+ ) ?? false+ );+}++function schemaContainsBytesField(schemaRef: string): boolean {+ const schema = getSchema(schemaRef);+ if (!schema || schema.type !== "object" || !schema.properties) {+ return false;}- return ` "${prop.name}": ${prop.type}${required}${desc}`;+ const seen = new Set<string>();+ return schema.properties.some((prop) => hasBytesFieldInProperty(prop, seen));}function formatServiceList(): string {@@ -137,10 +215,7 @@ function formatEndpoints(eps: EndpointInfo[], limit: number): string {}function getSchemaProps(schemaRef: string): PropertyInfo[] | undefined {- // Extract name from ref: "#/components/schemas/bytebase.v1.QueryRequest"- const parts = schemaRef.split("/");- const name = parts[parts.length - 1];- const info = schemas[name];+ const info = getSchema(schemaRef);if (!info) return undefined;return info.properties;}@@ -159,6 +234,7 @@ function formatEndpointDetail(operationId: string): string {if (ep.requestSchemaRef) {const props = getSchemaProps(ep.requestSchemaRef);if (props && props.length > 0) {+ const hasBytesField = schemaContainsBytesField(ep.requestSchemaRef);lines.push("### Request Body");lines.push("```json");lines.push("{");@@ -167,6 +243,11 @@ function formatEndpointDetail(operationId: string): string {}lines.push("}");lines.push("```\n");+ if (hasBytesField) {+ lines.push(+ "Note: request body includes protobuf bytes fields; plain strings passed to call_api are UTF-8 encoded automatically.\n"+ );+ }}}