Skip to content

The Postgres development platform. Supabase gives you a dedicated Postgres database to build your web, mobile, and AI applications.

Fix handle truncation for multi dim array for table-row-query (#51834)

Joshen Lim7a9565530c1c
diff --git a/apps/studio/components/interfaces/TableGridEditor/SidePanelEditor/RowEditor/RowEditor.utils.test.ts b/apps/studio/components/interfaces/TableGridEditor/SidePanelEditor/RowEditor/RowEditor.utils.test.ts
index 64a66b311c1..531f1db4959 100644
--- a/apps/studio/components/interfaces/TableGridEditor/SidePanelEditor/RowEditor/RowEditor.utils.test.ts
+++ b/apps/studio/components/interfaces/TableGridEditor/SidePanelEditor/RowEditor/RowEditor.utils.test.ts
@@ -1,3 +1,4 @@
+import { MAX_ARRAY_SIZE } from '@supabase/pg-meta/src/query/table-row-query'
import { describe, expect, it, vi } from 'vitest'
import { RowField } from '@/components/interfaces/TableGridEditor/SidePanelEditor/RowEditor/RowEditor.types'
@@ -335,14 +336,26 @@ describe('isValueTruncated', () => {
const jsonArrayTruncated = `[${Array(5).fill('{"key":"value"}').join(',')},{"truncated":true}]`
expect(isValueTruncated(jsonArrayTruncated)).toBe(true)
- // Pattern 4: Multi-dimensional array (lines 211-212)
- // column[1:50]::type[] - no special marker, just detect by [[ pattern
- expect(isValueTruncated('[["item"]]', '_text')).toBe(true)
- expect(isValueTruncated('[["item1","item2"]]', '_text')).toBe(true)
+ // Pattern 4: Multi-dimensional array
+ // column[1:50]::type[] + an extra entry filled with '...'
})
- it('should detect multidimensional arrays', () => {
- expect(isValueTruncated('[["item1", "item2"]]', '_text')).toBe(true)
+ it('should detect multidimensional arrays with the truncation marker row', () => {
+ expect(isValueTruncated('[["a","b"],["c","d"],["...","..."]]', '_text')).toBe(true)
+ expect(isValueTruncated('[[["a","b"]],[["...","..."]]]', '_text')).toBe(true)
+ expect(isValueTruncated('[[{"k":1}],[{"truncated":true}]]', '_jsonb')).toBe(true)
+ })
+
+ it('should not flag multidimensional arrays or 1D arrays of JSON arrays that are not truncated', () => {
+ expect(isValueTruncated('[["item1","item2"],["item3","item4"]]', '_text')).toBe(false)
+ expect(isValueTruncated('[["item"]]', '_jsonb')).toBe(false)
+ expect(isValueTruncated('[["item"', '_jsonb')).toBe(false)
+ // empty trailing entries have no marker
+ expect(isValueTruncated('[["a"],[]]', '_text')).toBe(false)
+ expect(isValueTruncated('[["a"],[[]]]', '_text')).toBe(false)
+ // exactly MAX_ARRAY_SIZE entries, but complete
+ const rows = Array.from({ length: MAX_ARRAY_SIZE }, (_, i) => [`a${i}`, 'b'])
+ expect(isValueTruncated(JSON.stringify(rows), '_text')).toBe(false)
})
it('should detect truncated JSON arrays with truncated flag', () => {
diff --git a/apps/studio/components/interfaces/TableGridEditor/SidePanelEditor/RowEditor/RowEditor.utils.ts b/apps/studio/components/interfaces/TableGridEditor/SidePanelEditor/RowEditor/RowEditor.utils.ts
index 90466bf8eb2..e1a808b48ea 100644
--- a/apps/studio/components/interfaces/TableGridEditor/SidePanelEditor/RowEditor/RowEditor.utils.ts
+++ b/apps/studio/components/interfaces/TableGridEditor/SidePanelEditor/RowEditor/RowEditor.utils.ts
@@ -277,6 +277,29 @@ export const generateUpdateRowPayload = (originalRow: any, fields: RowField[]) =
return payload
}
+/**
+ * Truncated multi-dimensional arrays are cut to their first MAX_ARRAY_SIZE entries, with an extra last entry
+ * filled with the "..." (or {"truncated":true} for json) marker, see getTableRowsSql
+ */
+const isTruncatedMultiDimArray = (value: string) => {
+ if (!value.startsWith('[[')) return false
+ try {
+ const parsed: unknown = JSON.parse(value)
+ if (!Array.isArray(parsed) || !Array.isArray(parsed[0])) return false
+
+ const lastEntry: unknown = parsed[parsed.length - 1]
+ if (!Array.isArray(lastEntry)) return false
+
+ // `every` is vacuously true for empty arrays (e.g. `[[1],[]]`), so require at least one marker
+ const flattened = lastEntry.flat(Infinity)
+ return (
+ flattened.length > 0 && flattened.every((x) => x === '...' || isEqual(x, { truncated: true }))
+ )
+ } catch {
+ return false
+ }
+}
+
/**
* Checks if the value is truncated. The JSON types are usually truncated if they're too big to show in the editor.
*/
@@ -294,7 +317,7 @@ export const isValueTruncated = (value: string | null | undefined, format?: stri
// If the array have MAX_ARRAY_SIZE elements in it
// its a large truncated array
(value.match(/","/g) || []).length === MAX_ARRAY_SIZE) ||
- (typeof value === 'string' && isArrayColumn && value.startsWith('[["')) ||
+ (typeof value === 'string' && isArrayColumn && isTruncatedMultiDimArray(value)) ||
// [Joshen] For json arrays, refer to getTableRowsSql from table-row-query
// for array types, we're adding {"truncated": true} as the last item of the JSON to
// maintain the JSON array structure
diff --git a/packages/pg-meta/src/query/table-row-query.ts b/packages/pg-meta/src/query/table-row-query.ts
index f9e101a2e2d..42c3b0745b1 100644
--- a/packages/pg-meta/src/query/table-row-query.ts
+++ b/packages/pg-meta/src/query/table-row-query.ts
@@ -241,18 +241,18 @@ export const getTableRowsSql = ({
const isJson = (JSON_SET as Set<string>).has(format)
// format comes from pg_attribute (e.g. 'text', 'json') — ident() ensures safe quoting
const arrayTypeCast = isJson ? safeSql`::${keyword(format)}[]` : safeSql`::text[]`
- const lastElement: SafeSqlFragment = isJson
- ? safeSql`array['{"truncated": true}'::json]`
- : safeSql`array['...']`
+ const marker: SafeSqlFragment = isJson
+ ? safeSql`'{"truncated": true}'::json`
+ : safeSql`'...'::text`
+ const lastElement: SafeSqlFragment = safeSql`array[${marker}]`
const col = ident(columnName)
if (index >= 0) {
// We cast to text[] but limit the array size if the total size of the array is too large (same logic than for text fields)
// This returns the first MAX_ARRAY_SIZE elements of the array (adjustable) and adds '...' if truncated
// NOTE: this is not optimal, as the first element in the array could still be very large (more than 10Kb) and in such case
// the trimming might fail.
- // Also handle multi-dimentionals array truncation, but won't happen the extra `...` element to it as we can't determine what's
- // the right number of items to generate within the array. Studio side, we'll consider any multi-dimentional array as possibly
- // truncated.
+ // Multi-dimensional arrays are sliced the same way, but the marker has to be a whole extra entry of the first
+ // dimension (so that the array stays rectangular). It's filled with the marker, using the lengths of the other dimensions.
selectExpressions[index] = safeSql`
case
when octet_length(${col}::text) > ${literal(maxCharacters)}
@@ -262,7 +262,13 @@ export const getTableRowsSql = ({
then
(select array_cat(${col}[1:${literal(maxArraySize)}]${arrayTypeCast}, ${lastElement}${arrayTypeCast}))${arrayTypeCast}
else
- ${col}[1:${literal(maxArraySize)}]${arrayTypeCast}
+ array_cat(
+ ${col}[1:${literal(maxArraySize)}]${arrayTypeCast},
+ array_fill(
+ ${marker},
+ array(select array_length(${col}, d) from generate_series(2, array_ndims(${col})) as d order by d)
+ )${arrayTypeCast}
+ )${arrayTypeCast}
end
else ${col}${arrayTypeCast}
end
diff --git a/packages/pg-meta/test/query/table-row-query.test.ts b/packages/pg-meta/test/query/table-row-query.test.ts
index 27d62a906f4..2dc88c4f9e4 100644
--- a/packages/pg-meta/test/query/table-row-query.test.ts
+++ b/packages/pg-meta/test/query/table-row-query.test.ts
@@ -2,7 +2,11 @@ import { afterAll, beforeAll, describe, expect, test } from 'vitest'
import pgMeta from '../../src/index'
import { Filter, Sort } from '../../src/query'
-import { getDefaultOrderByColumns, getTableRowsSql } from '../../src/query/table-row-query'
+import {
+ getDefaultOrderByColumns,
+ getTableRowsSql,
+ MAX_ARRAY_SIZE,
+} from '../../src/query/table-row-query'
import { cleanupRoot, createTestDatabase } from '../db/utils'
beforeAll(async () => {
@@ -111,9 +115,15 @@ describe('Table Row Query', () => {
case
when array_ndims(history) = 1
then
- (select array_cat(history[1:50]::text[], array['...']::text[]))::text[]
+ (select array_cat(history[1:50]::text[], array['...'::text]::text[]))::text[]
else
- history[1:50]::text[]
+ array_cat(
+ history[1:50]::text[],
+ array_fill(
+ '...'::text,
+ array(select array_length(history, d) from generate_series(2, array_ndims(history)) as d order by d)
+ )::text[]
+ )::text[]
end
else history::text[]
end
@@ -184,9 +194,15 @@ describe('Table Row Query', () => {
case
when array_ndims(history) = 1
then
- (select array_cat(history[1:50]::text[], array['...']::text[]))::text[]
+ (select array_cat(history[1:50]::text[], array['...'::text]::text[]))::text[]
else
- history[1:50]::text[]
+ array_cat(
+ history[1:50]::text[],
+ array_fill(
+ '...'::text,
+ array(select array_length(history, d) from generate_series(2, array_ndims(history)) as d order by d)
+ )::text[]
+ )::text[]
end
else history::text[]
end
@@ -230,9 +246,15 @@ describe('Table Row Query', () => {
case
when array_ndims(history) = 1
then
- (select array_cat(history[1:50]::text[], array['...']::text[]))::text[]
+ (select array_cat(history[1:50]::text[], array['...'::text]::text[]))::text[]
else
- history[1:50]::text[]
+ array_cat(
+ history[1:50]::text[],
+ array_fill(
+ '...'::text,
+ array(select array_length(history, d) from generate_series(2, array_ndims(history)) as d order by d)
+ )::text[]
+ )::text[]
end
else history::text[]
end
@@ -283,9 +305,15 @@ describe('Table Row Query', () => {
case
when array_ndims(tags) = 1
then
- (select array_cat(tags[1:50]::text[], array['...']::text[]))::text[]
+ (select array_cat(tags[1:50]::text[], array['...'::text]::text[]))::text[]
else
- tags[1:50]::text[]
+ array_cat(
+ tags[1:50]::text[],
+ array_fill(
+ '...'::text,
+ array(select array_length(tags, d) from generate_series(2, array_ndims(tags)) as d order by d)
+ )::text[]
+ )::text[]
end
else tags::text[]
end
@@ -323,6 +351,52 @@ describe('Table Row Query', () => {
`)
})
+ withTestDatabase('should add a marker to truncated multi-dimensional arrays', async (db) => {
+ await db.executeQuery(`
+ CREATE TABLE test_multi_dim_table (
+ id SERIAL PRIMARY KEY,
+ grid TEXT[][],
+ cube TEXT[][][],
+ docs JSONB[][]
+ );
+
+ INSERT INTO test_multi_dim_table (grid, cube, docs)
+ SELECT
+ (SELECT array_agg(ARRAY[repeat('a', 300), 'b']) FROM generate_series(1, 120)),
+ (SELECT array_agg(ARRAY[ARRAY[repeat('a', 300), 'b'], ARRAY['c', 'd']]) FROM generate_series(1, 120)),
+ (SELECT array_agg(ARRAY['{"k": 1}'::jsonb, to_jsonb(repeat('a', 300))]) FROM generate_series(1, 120));
+
+ INSERT INTO test_multi_dim_table (grid, cube, docs)
+ VALUES (ARRAY[ARRAY['a', 'b'], ARRAY['c', 'd']], ARRAY[ARRAY[ARRAY['a']]], ARRAY[ARRAY['{"k": 1}'::jsonb]]);
+ `)
+
+ const { sql: tablesSql, zod: tablesZod } = pgMeta.tables.list()
+ const tables = tablesZod.parse(await db.executeQuery(tablesSql))
+ const testTable = tables.find((table) => table.name === 'test_multi_dim_table')
+ expect(testTable).toBeDefined()
+
+ const sql = getTableRowsSql({ table: testTable!, page: 1, limit: 10 })
+ const [truncated, small] = await db.executeQuery(sql)
+
+ expect(truncated.grid).toHaveLength(MAX_ARRAY_SIZE + 1)
+ expect(truncated.grid.at(-1)).toEqual(['...', '...'])
+ expect(truncated.cube).toHaveLength(MAX_ARRAY_SIZE + 1)
+ expect(truncated.cube.at(-1)).toEqual([
+ ['...', '...'],
+ ['...', '...'],
+ ])
+ expect(truncated.docs).toHaveLength(MAX_ARRAY_SIZE + 1)
+ expect(truncated.docs.at(-1)).toEqual([{ truncated: true }, { truncated: true }])
+
+ // arrays under the limit are returned as they are
+ expect(small.grid).toEqual([
+ ['a', 'b'],
+ ['c', 'd'],
+ ])
+ expect(small.cube).toEqual([[['a']]])
+ expect(small.docs).toEqual([[{ k: 1 }]])
+ })
+
withTestDatabase('should generate basic SELECT SQL for a table', async (db) => {
// Create test table and insert data
await db.executeQuery(`
@@ -430,9 +504,15 @@ describe('Table Row Query', () => {
case
when array_ndims(large_array) = 1
then
- (select array_cat(large_array[1:10]::text[], array['...']::text[]))::text[]
+ (select array_cat(large_array[1:10]::text[], array['...'::text]::text[]))::text[]
else
- large_array[1:10]::text[]
+ array_cat(
+ large_array[1:10]::text[],
+ array_fill(
+ '...'::text,
+ array(select array_length(large_array, d) from generate_series(2, array_ndims(large_array)) as d order by d)
+ )::text[]
+ )::text[]
end
else large_array::text[]
end
@@ -666,7 +746,13 @@ describe('Table Row Query', () => {
then
(select array_cat(large_array_jsonb[1:10]::jsonb[], array['{"truncated": true}'::json]::jsonb[]))::jsonb[]
else
- large_array_jsonb[1:10]::jsonb[]
+ array_cat(
+ large_array_jsonb[1:10]::jsonb[],
+ array_fill(
+ '{"truncated": true}'::json,
+ array(select array_length(large_array_jsonb, d) from generate_series(2, array_ndims(large_array_jsonb)) as d order by d)
+ )::jsonb[]
+ )::jsonb[]
end
else large_array_jsonb::jsonb[]
end
@@ -679,7 +765,13 @@ describe('Table Row Query', () => {
then
(select array_cat(large_array_json[1:10]::json[], array['{"truncated": true}'::json]::json[]))::json[]
else
- large_array_json[1:10]::json[]
+ array_cat(
+ large_array_json[1:10]::json[],
+ array_fill(
+ '{"truncated": true}'::json,
+ array(select array_length(large_array_json, d) from generate_series(2, array_ndims(large_array_json)) as d order by d)
+ )::json[]
+ )::json[]
end
else large_array_json::json[]
end
@@ -1041,9 +1133,15 @@ describe('Table Row Query', () => {
case
when array_ndims(large_array) = 1
then
- (select array_cat(large_array[1:50]::text[], array['...']::text[]))::text[]
+ (select array_cat(large_array[1:50]::text[], array['...'::text]::text[]))::text[]
else
- large_array[1:50]::text[]
+ array_cat(
+ large_array[1:50]::text[],
+ array_fill(
+ '...'::text,
+ array(select array_length(large_array, d) from generate_series(2, array_ndims(large_array)) as d order by d)
+ )::text[]
+ )::text[]
end
else large_array::text[]
end
@@ -1722,9 +1820,15 @@ describe('Table Row Query', () => {
case
when array_ndims("timestamp") = 1
then
- (select array_cat("timestamp"[1:50]::text[], array['...']::text[]))::text[]
+ (select array_cat("timestamp"[1:50]::text[], array['...'::text]::text[]))::text[]
else
- "timestamp"[1:50]::text[]
+ array_cat(
+ "timestamp"[1:50]::text[],
+ array_fill(
+ '...'::text,
+ array(select array_length("timestamp", d) from generate_series(2, array_ndims("timestamp")) as d order by d)
+ )::text[]
+ )::text[]
end
else "timestamp"::text[]
end
@@ -1735,9 +1839,15 @@ describe('Table Row Query', () => {
case
when array_ndims("PPG") = 1
then
- (select array_cat("PPG"[1:50]::text[], array['...']::text[]))::text[]
+ (select array_cat("PPG"[1:50]::text[], array['...'::text]::text[]))::text[]
else
- "PPG"[1:50]::text[]
+ array_cat(
+ "PPG"[1:50]::text[],
+ array_fill(
+ '...'::text,
+ array(select array_length("PPG", d) from generate_series(2, array_ndims("PPG")) as d order by d)
+ )::text[]
+ )::text[]
end
else "PPG"::text[]
end
@@ -1748,9 +1858,15 @@ describe('Table Row Query', () => {
case
when array_ndims("ACC") = 1
then
- (select array_cat("ACC"[1:50]::text[], array['...']::text[]))::text[]
+ (select array_cat("ACC"[1:50]::text[], array['...'::text]::text[]))::text[]
else
- "ACC"[1:50]::text[]
+ array_cat(
+ "ACC"[1:50]::text[],
+ array_fill(
+ '...'::text,
+ array(select array_length("ACC", d) from generate_series(2, array_ndims("ACC")) as d order by d)
+ )::text[]
+ )::text[]
end
else "ACC"::text[]
end
@@ -1774,10 +1890,12 @@ describe('Table Row Query', () => {
const largeRow = queryResult.find((row: any) => row.subject_id === 'subject-large')
expect(largeRow).toBeDefined()
expect(largeRow.timestamp).toHaveLength(51) // Has the extra '...' element
- expect(largeRow.PPG).toHaveLength(50)
+ expect(largeRow.PPG).toHaveLength(51) // Has the extra '...' row
expect(largeRow.PPG[0]).toHaveLength(6)
- expect(largeRow.ACC).toHaveLength(50)
+ expect(largeRow.PPG[50]).toEqual(Array(6).fill('...'))
+ expect(largeRow.ACC).toHaveLength(51)
expect(largeRow.ACC[0]).toHaveLength(6)
+ expect(largeRow.ACC[50]).toEqual(Array(6).fill('...'))
// Test with custom maxArraySize
const sqlWithCustomSize = getTableRowsSql({
@@ -1790,8 +1908,8 @@ describe('Table Row Query', () => {
const customSizeResult = await db.executeQuery(sqlWithCustomSize)
const largeRowCustom = customSizeResult.find((row: any) => row.subject_id === 'subject-large')
expect(largeRowCustom.timestamp).toHaveLength(11) // Has the extra '...' element
- expect(largeRowCustom.PPG).toHaveLength(10) // multi-dimentional array are truncated
- expect(largeRowCustom.ACC).toHaveLength(10)
+ expect(largeRowCustom.PPG).toHaveLength(11) // multi-dimentional arrays get an extra '...' row too
+ expect(largeRowCustom.ACC).toHaveLength(11)
})
withTestDatabase('should handle reserved keyword "collation" as column name', async (db) => {