The Postgres development platform. Supabase gives you a dedicated Postgres database to build your web, mobile, and AI applications.
TypeScript aialternativeauthdatabasedenoembeddingsexamplefirebasenextjsoauth2pgvectorpostgispostgrespostgresqlpostgrestrealtimesupabasevectorswebsockets
Fix handle truncation for multi dim array for table-row-query (#51834)
Joshen Lim7a9565530c1c
diff --git a/apps/studio/components/interfaces/TableGridEditor/SidePanelEditor/RowEditor/RowEditor.utils.test.ts b/apps/studio/components/interfaces/TableGridEditor/SidePanelEditor/RowEditor/RowEditor.utils.test.tsindex 64a66b311c1..531f1db4959 100644--- a/apps/studio/components/interfaces/TableGridEditor/SidePanelEditor/RowEditor/RowEditor.utils.test.ts+++ b/apps/studio/components/interfaces/TableGridEditor/SidePanelEditor/RowEditor/RowEditor.utils.test.ts@@ -1,3 +1,4 @@+import { MAX_ARRAY_SIZE } from '@supabase/pg-meta/src/query/table-row-query'import { describe, expect, it, vi } from 'vitest'import { RowField } from '@/components/interfaces/TableGridEditor/SidePanelEditor/RowEditor/RowEditor.types'@@ -335,14 +336,26 @@ describe('isValueTruncated', () => {const jsonArrayTruncated = `[${Array(5).fill('{"key":"value"}').join(',')},{"truncated":true}]`expect(isValueTruncated(jsonArrayTruncated)).toBe(true)- // Pattern 4: Multi-dimensional array (lines 211-212)- // column[1:50]::type[] - no special marker, just detect by [[ pattern- expect(isValueTruncated('[["item"]]', '_text')).toBe(true)- expect(isValueTruncated('[["item1","item2"]]', '_text')).toBe(true)+ // Pattern 4: Multi-dimensional array+ // column[1:50]::type[] + an extra entry filled with '...'})- it('should detect multidimensional arrays', () => {- expect(isValueTruncated('[["item1", "item2"]]', '_text')).toBe(true)+ it('should detect multidimensional arrays with the truncation marker row', () => {+ expect(isValueTruncated('[["a","b"],["c","d"],["...","..."]]', '_text')).toBe(true)+ expect(isValueTruncated('[[["a","b"]],[["...","..."]]]', '_text')).toBe(true)+ expect(isValueTruncated('[[{"k":1}],[{"truncated":true}]]', '_jsonb')).toBe(true)+ })++ it('should not flag multidimensional arrays or 1D arrays of JSON arrays that are not truncated', () => {+ expect(isValueTruncated('[["item1","item2"],["item3","item4"]]', '_text')).toBe(false)+ expect(isValueTruncated('[["item"]]', '_jsonb')).toBe(false)+ expect(isValueTruncated('[["item"', '_jsonb')).toBe(false)+ // empty trailing entries have no marker+ expect(isValueTruncated('[["a"],[]]', '_text')).toBe(false)+ expect(isValueTruncated('[["a"],[[]]]', '_text')).toBe(false)+ // exactly MAX_ARRAY_SIZE entries, but complete+ const rows = Array.from({ length: MAX_ARRAY_SIZE }, (_, i) => [`a${i}`, 'b'])+ expect(isValueTruncated(JSON.stringify(rows), '_text')).toBe(false)})it('should detect truncated JSON arrays with truncated flag', () => {diff --git a/apps/studio/components/interfaces/TableGridEditor/SidePanelEditor/RowEditor/RowEditor.utils.ts b/apps/studio/components/interfaces/TableGridEditor/SidePanelEditor/RowEditor/RowEditor.utils.tsindex 90466bf8eb2..e1a808b48ea 100644--- a/apps/studio/components/interfaces/TableGridEditor/SidePanelEditor/RowEditor/RowEditor.utils.ts+++ b/apps/studio/components/interfaces/TableGridEditor/SidePanelEditor/RowEditor/RowEditor.utils.ts@@ -277,6 +277,29 @@ export const generateUpdateRowPayload = (originalRow: any, fields: RowField[]) =return payload}+/**+ * Truncated multi-dimensional arrays are cut to their first MAX_ARRAY_SIZE entries, with an extra last entry+ * filled with the "..." (or {"truncated":true} for json) marker, see getTableRowsSql+ */+const isTruncatedMultiDimArray = (value: string) => {+ if (!value.startsWith('[[')) return false+ try {+ const parsed: unknown = JSON.parse(value)+ if (!Array.isArray(parsed) || !Array.isArray(parsed[0])) return false++ const lastEntry: unknown = parsed[parsed.length - 1]+ if (!Array.isArray(lastEntry)) return false++ // `every` is vacuously true for empty arrays (e.g. `[[1],[]]`), so require at least one marker+ const flattened = lastEntry.flat(Infinity)+ return (+ flattened.length > 0 && flattened.every((x) => x === '...' || isEqual(x, { truncated: true }))+ )+ } catch {+ return false+ }+}+/*** Checks if the value is truncated. The JSON types are usually truncated if they're too big to show in the editor.*/@@ -294,7 +317,7 @@ export const isValueTruncated = (value: string | null | undefined, format?: stri// If the array have MAX_ARRAY_SIZE elements in it// its a large truncated array(value.match(/","/g) || []).length === MAX_ARRAY_SIZE) ||- (typeof value === 'string' && isArrayColumn && value.startsWith('[["')) ||+ (typeof value === 'string' && isArrayColumn && isTruncatedMultiDimArray(value)) ||// [Joshen] For json arrays, refer to getTableRowsSql from table-row-query// for array types, we're adding {"truncated": true} as the last item of the JSON to// maintain the JSON array structurediff --git a/packages/pg-meta/src/query/table-row-query.ts b/packages/pg-meta/src/query/table-row-query.tsindex f9e101a2e2d..42c3b0745b1 100644--- a/packages/pg-meta/src/query/table-row-query.ts+++ b/packages/pg-meta/src/query/table-row-query.ts@@ -241,18 +241,18 @@ export const getTableRowsSql = ({const isJson = (JSON_SET as Set<string>).has(format)// format comes from pg_attribute (e.g. 'text', 'json') — ident() ensures safe quotingconst arrayTypeCast = isJson ? safeSql`::${keyword(format)}[]` : safeSql`::text[]`- const lastElement: SafeSqlFragment = isJson- ? safeSql`array['{"truncated": true}'::json]`- : safeSql`array['...']`+ const marker: SafeSqlFragment = isJson+ ? safeSql`'{"truncated": true}'::json`+ : safeSql`'...'::text`+ const lastElement: SafeSqlFragment = safeSql`array[${marker}]`const col = ident(columnName)if (index >= 0) {// We cast to text[] but limit the array size if the total size of the array is too large (same logic than for text fields)// This returns the first MAX_ARRAY_SIZE elements of the array (adjustable) and adds '...' if truncated// NOTE: this is not optimal, as the first element in the array could still be very large (more than 10Kb) and in such case// the trimming might fail.- // Also handle multi-dimentionals array truncation, but won't happen the extra `...` element to it as we can't determine what's- // the right number of items to generate within the array. Studio side, we'll consider any multi-dimentional array as possibly- // truncated.+ // Multi-dimensional arrays are sliced the same way, but the marker has to be a whole extra entry of the first+ // dimension (so that the array stays rectangular). It's filled with the marker, using the lengths of the other dimensions.selectExpressions[index] = safeSql`casewhen octet_length(${col}::text) > ${literal(maxCharacters)}@@ -262,7 +262,13 @@ export const getTableRowsSql = ({then(select array_cat(${col}[1:${literal(maxArraySize)}]${arrayTypeCast}, ${lastElement}${arrayTypeCast}))${arrayTypeCast}else- ${col}[1:${literal(maxArraySize)}]${arrayTypeCast}+ array_cat(+ ${col}[1:${literal(maxArraySize)}]${arrayTypeCast},+ array_fill(+ ${marker},+ array(select array_length(${col}, d) from generate_series(2, array_ndims(${col})) as d order by d)+ )${arrayTypeCast}+ )${arrayTypeCast}endelse ${col}${arrayTypeCast}enddiff --git a/packages/pg-meta/test/query/table-row-query.test.ts b/packages/pg-meta/test/query/table-row-query.test.tsindex 27d62a906f4..2dc88c4f9e4 100644--- a/packages/pg-meta/test/query/table-row-query.test.ts+++ b/packages/pg-meta/test/query/table-row-query.test.ts@@ -2,7 +2,11 @@ import { afterAll, beforeAll, describe, expect, test } from 'vitest'import pgMeta from '../../src/index'import { Filter, Sort } from '../../src/query'-import { getDefaultOrderByColumns, getTableRowsSql } from '../../src/query/table-row-query'+import {+ getDefaultOrderByColumns,+ getTableRowsSql,+ MAX_ARRAY_SIZE,+} from '../../src/query/table-row-query'import { cleanupRoot, createTestDatabase } from '../db/utils'beforeAll(async () => {@@ -111,9 +115,15 @@ describe('Table Row Query', () => {casewhen array_ndims(history) = 1then- (select array_cat(history[1:50]::text[], array['...']::text[]))::text[]+ (select array_cat(history[1:50]::text[], array['...'::text]::text[]))::text[]else- history[1:50]::text[]+ array_cat(+ history[1:50]::text[],+ array_fill(+ '...'::text,+ array(select array_length(history, d) from generate_series(2, array_ndims(history)) as d order by d)+ )::text[]+ )::text[]endelse history::text[]end@@ -184,9 +194,15 @@ describe('Table Row Query', () => {casewhen array_ndims(history) = 1then- (select array_cat(history[1:50]::text[], array['...']::text[]))::text[]+ (select array_cat(history[1:50]::text[], array['...'::text]::text[]))::text[]else- history[1:50]::text[]+ array_cat(+ history[1:50]::text[],+ array_fill(+ '...'::text,+ array(select array_length(history, d) from generate_series(2, array_ndims(history)) as d order by d)+ )::text[]+ )::text[]endelse history::text[]end@@ -230,9 +246,15 @@ describe('Table Row Query', () => {casewhen array_ndims(history) = 1then- (select array_cat(history[1:50]::text[], array['...']::text[]))::text[]+ (select array_cat(history[1:50]::text[], array['...'::text]::text[]))::text[]else- history[1:50]::text[]+ array_cat(+ history[1:50]::text[],+ array_fill(+ '...'::text,+ array(select array_length(history, d) from generate_series(2, array_ndims(history)) as d order by d)+ )::text[]+ )::text[]endelse history::text[]end@@ -283,9 +305,15 @@ describe('Table Row Query', () => {casewhen array_ndims(tags) = 1then- (select array_cat(tags[1:50]::text[], array['...']::text[]))::text[]+ (select array_cat(tags[1:50]::text[], array['...'::text]::text[]))::text[]else- tags[1:50]::text[]+ array_cat(+ tags[1:50]::text[],+ array_fill(+ '...'::text,+ array(select array_length(tags, d) from generate_series(2, array_ndims(tags)) as d order by d)+ )::text[]+ )::text[]endelse tags::text[]end@@ -323,6 +351,52 @@ describe('Table Row Query', () => {`)})+ withTestDatabase('should add a marker to truncated multi-dimensional arrays', async (db) => {+ await db.executeQuery(`+ CREATE TABLE test_multi_dim_table (+ id SERIAL PRIMARY KEY,+ grid TEXT[][],+ cube TEXT[][][],+ docs JSONB[][]+ );++ INSERT INTO test_multi_dim_table (grid, cube, docs)+ SELECT+ (SELECT array_agg(ARRAY[repeat('a', 300), 'b']) FROM generate_series(1, 120)),+ (SELECT array_agg(ARRAY[ARRAY[repeat('a', 300), 'b'], ARRAY['c', 'd']]) FROM generate_series(1, 120)),+ (SELECT array_agg(ARRAY['{"k": 1}'::jsonb, to_jsonb(repeat('a', 300))]) FROM generate_series(1, 120));++ INSERT INTO test_multi_dim_table (grid, cube, docs)+ VALUES (ARRAY[ARRAY['a', 'b'], ARRAY['c', 'd']], ARRAY[ARRAY[ARRAY['a']]], ARRAY[ARRAY['{"k": 1}'::jsonb]]);+ `)++ const { sql: tablesSql, zod: tablesZod } = pgMeta.tables.list()+ const tables = tablesZod.parse(await db.executeQuery(tablesSql))+ const testTable = tables.find((table) => table.name === 'test_multi_dim_table')+ expect(testTable).toBeDefined()++ const sql = getTableRowsSql({ table: testTable!, page: 1, limit: 10 })+ const [truncated, small] = await db.executeQuery(sql)++ expect(truncated.grid).toHaveLength(MAX_ARRAY_SIZE + 1)+ expect(truncated.grid.at(-1)).toEqual(['...', '...'])+ expect(truncated.cube).toHaveLength(MAX_ARRAY_SIZE + 1)+ expect(truncated.cube.at(-1)).toEqual([+ ['...', '...'],+ ['...', '...'],+ ])+ expect(truncated.docs).toHaveLength(MAX_ARRAY_SIZE + 1)+ expect(truncated.docs.at(-1)).toEqual([{ truncated: true }, { truncated: true }])++ // arrays under the limit are returned as they are+ expect(small.grid).toEqual([+ ['a', 'b'],+ ['c', 'd'],+ ])+ expect(small.cube).toEqual([[['a']]])+ expect(small.docs).toEqual([[{ k: 1 }]])+ })+withTestDatabase('should generate basic SELECT SQL for a table', async (db) => {// Create test table and insert dataawait db.executeQuery(`@@ -430,9 +504,15 @@ describe('Table Row Query', () => {casewhen array_ndims(large_array) = 1then- (select array_cat(large_array[1:10]::text[], array['...']::text[]))::text[]+ (select array_cat(large_array[1:10]::text[], array['...'::text]::text[]))::text[]else- large_array[1:10]::text[]+ array_cat(+ large_array[1:10]::text[],+ array_fill(+ '...'::text,+ array(select array_length(large_array, d) from generate_series(2, array_ndims(large_array)) as d order by d)+ )::text[]+ )::text[]endelse large_array::text[]end@@ -666,7 +746,13 @@ describe('Table Row Query', () => {then(select array_cat(large_array_jsonb[1:10]::jsonb[], array['{"truncated": true}'::json]::jsonb[]))::jsonb[]else- large_array_jsonb[1:10]::jsonb[]+ array_cat(+ large_array_jsonb[1:10]::jsonb[],+ array_fill(+ '{"truncated": true}'::json,+ array(select array_length(large_array_jsonb, d) from generate_series(2, array_ndims(large_array_jsonb)) as d order by d)+ )::jsonb[]+ )::jsonb[]endelse large_array_jsonb::jsonb[]end@@ -679,7 +765,13 @@ describe('Table Row Query', () => {then(select array_cat(large_array_json[1:10]::json[], array['{"truncated": true}'::json]::json[]))::json[]else- large_array_json[1:10]::json[]+ array_cat(+ large_array_json[1:10]::json[],+ array_fill(+ '{"truncated": true}'::json,+ array(select array_length(large_array_json, d) from generate_series(2, array_ndims(large_array_json)) as d order by d)+ )::json[]+ )::json[]endelse large_array_json::json[]end@@ -1041,9 +1133,15 @@ describe('Table Row Query', () => {casewhen array_ndims(large_array) = 1then- (select array_cat(large_array[1:50]::text[], array['...']::text[]))::text[]+ (select array_cat(large_array[1:50]::text[], array['...'::text]::text[]))::text[]else- large_array[1:50]::text[]+ array_cat(+ large_array[1:50]::text[],+ array_fill(+ '...'::text,+ array(select array_length(large_array, d) from generate_series(2, array_ndims(large_array)) as d order by d)+ )::text[]+ )::text[]endelse large_array::text[]end@@ -1722,9 +1820,15 @@ describe('Table Row Query', () => {casewhen array_ndims("timestamp") = 1then- (select array_cat("timestamp"[1:50]::text[], array['...']::text[]))::text[]+ (select array_cat("timestamp"[1:50]::text[], array['...'::text]::text[]))::text[]else- "timestamp"[1:50]::text[]+ array_cat(+ "timestamp"[1:50]::text[],+ array_fill(+ '...'::text,+ array(select array_length("timestamp", d) from generate_series(2, array_ndims("timestamp")) as d order by d)+ )::text[]+ )::text[]endelse "timestamp"::text[]end@@ -1735,9 +1839,15 @@ describe('Table Row Query', () => {casewhen array_ndims("PPG") = 1then- (select array_cat("PPG"[1:50]::text[], array['...']::text[]))::text[]+ (select array_cat("PPG"[1:50]::text[], array['...'::text]::text[]))::text[]else- "PPG"[1:50]::text[]+ array_cat(+ "PPG"[1:50]::text[],+ array_fill(+ '...'::text,+ array(select array_length("PPG", d) from generate_series(2, array_ndims("PPG")) as d order by d)+ )::text[]+ )::text[]endelse "PPG"::text[]end@@ -1748,9 +1858,15 @@ describe('Table Row Query', () => {casewhen array_ndims("ACC") = 1then- (select array_cat("ACC"[1:50]::text[], array['...']::text[]))::text[]+ (select array_cat("ACC"[1:50]::text[], array['...'::text]::text[]))::text[]else- "ACC"[1:50]::text[]+ array_cat(+ "ACC"[1:50]::text[],+ array_fill(+ '...'::text,+ array(select array_length("ACC", d) from generate_series(2, array_ndims("ACC")) as d order by d)+ )::text[]+ )::text[]endelse "ACC"::text[]end@@ -1774,10 +1890,12 @@ describe('Table Row Query', () => {const largeRow = queryResult.find((row: any) => row.subject_id === 'subject-large')expect(largeRow).toBeDefined()expect(largeRow.timestamp).toHaveLength(51) // Has the extra '...' element- expect(largeRow.PPG).toHaveLength(50)+ expect(largeRow.PPG).toHaveLength(51) // Has the extra '...' rowexpect(largeRow.PPG[0]).toHaveLength(6)- expect(largeRow.ACC).toHaveLength(50)+ expect(largeRow.PPG[50]).toEqual(Array(6).fill('...'))+ expect(largeRow.ACC).toHaveLength(51)expect(largeRow.ACC[0]).toHaveLength(6)+ expect(largeRow.ACC[50]).toEqual(Array(6).fill('...'))// Test with custom maxArraySizeconst sqlWithCustomSize = getTableRowsSql({@@ -1790,8 +1908,8 @@ describe('Table Row Query', () => {const customSizeResult = await db.executeQuery(sqlWithCustomSize)const largeRowCustom = customSizeResult.find((row: any) => row.subject_id === 'subject-large')expect(largeRowCustom.timestamp).toHaveLength(11) // Has the extra '...' element- expect(largeRowCustom.PPG).toHaveLength(10) // multi-dimentional array are truncated- expect(largeRowCustom.ACC).toHaveLength(10)+ expect(largeRowCustom.PPG).toHaveLength(11) // multi-dimentional arrays get an extra '...' row too+ expect(largeRowCustom.ACC).toHaveLength(11)})withTestDatabase('should handle reserved keyword "collation" as column name', async (db) => {