Skip to content

AI Agents & MCPs & AI Workflow Automation • (~400 MCP servers for AI agents) • AI Automation / AI Agent with MCPs • AI Workflows & AI Agents • MCPs for AI Agents

feat(duckdb): Usability enhancements (#15584)

Daniel Poonde6c7d12324e
diff --git a/bun.lock b/bun.lock
index e31cb132040..5e6d3e85e63 100644
--- a/bun.lock
+++ b/bun.lock
@@ -2786,16 +2786,18 @@
},
"packages/pieces/community/duckdb": {
"name": "@activepieces/piece-duckdb",
- "version": "0.0.8",
+ "version": "0.0.9",
"dependencies": {
"@activepieces/core-piece-types": "workspace:*",
"@activepieces/core-utils": "workspace:*",
"@activepieces/pieces-common": "workspace:*",
"@activepieces/pieces-framework": "workspace:*",
"@duckdb/node-api": "1.4.3-r.2",
+ "csv-parse": "5.6.0",
},
"devDependencies": {
"tslib": "^2.3.0",
+ "vitest": "3.2.6",
},
},
"packages/pieces/community/dumpling-ai": {
diff --git a/packages/pieces/community/duckdb/package.json b/packages/pieces/community/duckdb/package.json
index a96832fb42f..352cf80c7bc 100644
--- a/packages/pieces/community/duckdb/package.json
+++ b/packages/pieces/community/duckdb/package.json
@@ -1,6 +1,6 @@
{
"name": "@activepieces/piece-duckdb",
- "version": "0.0.8",
+ "version": "0.0.9",
"type": "commonjs",
"main": "./dist/src/index.js",
"types": "./dist/src/index.d.ts",
@@ -8,15 +8,18 @@
"@activepieces/pieces-common": "workspace:*",
"@activepieces/pieces-framework": "workspace:*",
"@duckdb/node-api": "1.4.3-r.2",
+ "csv-parse": "5.6.0",
"@activepieces/core-piece-types": "workspace:*",
"@activepieces/core-utils": "workspace:*"
},
"devDependencies": {
- "tslib": "^2.3.0"
+ "tslib": "^2.3.0",
+ "vitest": "3.2.6"
},
"scripts": {
"build": "tsc -p tsconfig.lib.json && cp package.json dist/",
"bundle": "node ../../../../dist/packages/cli/src/index.js pieces bundle",
- "lint": "eslint 'src/**/*.ts'"
+ "lint": "eslint 'src/**/*.ts'",
+ "test": "vitest run"
}
}
diff --git a/packages/pieces/community/duckdb/src/lib/actions/create-and-query-db.ts b/packages/pieces/community/duckdb/src/lib/actions/create-and-query-db.ts
index 3be14a2abfd..66b627c3046 100644
--- a/packages/pieces/community/duckdb/src/lib/actions/create-and-query-db.ts
+++ b/packages/pieces/community/duckdb/src/lib/actions/create-and-query-db.ts
@@ -1,6 +1,8 @@
-import { createAction, Property } from '@activepieces/pieces-framework';
+import { ApFile, createAction, Property } from '@activepieces/pieces-framework';
import { DuckDBInstance } from '@duckdb/node-api';
+import { quotedIdentifier } from '@duckdb/node-api/lib/sql';
+import { parse as parseCsv } from 'csv-parse/sync';
export const createAndQueryDB = createAction({
name: 'createAndQueryDB',
@@ -29,16 +31,24 @@ export const createAndQueryDB = createAction({
data: Property.Json({
displayName: 'JSON Data',
description:
- 'Table data in the form of a JSON array of objects. Object keys would be taken as column names.',
- required: true,
+ 'Table data as a JSON array of objects. Object keys are taken as column names. Takes priority over the file below if set.',
+ required: false,
defaultValue: [],
}),
+ file: Property.File({
+ displayName: 'CSV/JSON File',
+ description:
+ 'Upload a CSV or JSON file as the table source, used only if JSON Data above is empty. Format is detected from the file extension.',
+ required: false,
+ }),
schema: Property.Json({
displayName: 'Schema in JSON format',
description: `
Provide the schema as a JSON object. Note that only the specified columns would be loaded.
Leave empty to autodetect schema, although it is recommended to specify the schema for consistency and avoid unexpected behaviour.
+A column type that doesn't match the actual shape of the row data (e.g. a STRUCT/array type for a column that's a plain string, which is always the case for CSV-sourced tables) silently loads as null instead of erroring - double check this against autodetected schema if a column is unexpectedly empty.
+
More information on data types and accepted values:
- https://duckdb.org/2023/03/03/json
- https://duckdb.org/docs/stable/sql/data_types/overview
@@ -46,6 +56,13 @@ More information on data types and accepted values:
required: false,
defaultValue: {},
}),
+ keepNestedFields: Property.Checkbox({
+ displayName: 'Keep Nested Fields',
+ description:
+ 'When off, a nested object/struct field is split into individual top-level columns (e.g. "meta": {"a":1} becomes column "a"). Turn on to keep it as a single nested column instead, queryable with dot notation (e.g. "meta.a").',
+ required: false,
+ defaultValue: false,
+ }),
},
}),
query: Property.ShortText({
@@ -72,10 +89,12 @@ More information on data types and accepted values:
const dbTables: any[] = context.propsValue.tables ?? [];
for (const dbTable of dbTables) {
- const dbData = JSON.stringify(dbTable.data);
+ const tableName = quotedIdentifier(dbTable.name);
+ const dbData = JSON.stringify(resolveTableRows(dbTable));
let dbSchema = null;
- if (dbTable.schema) {
+ const hasSchema = dbTable.schema && Object.keys(dbTable.schema).length > 0;
+ if (hasSchema) {
dbSchema = JSON.stringify([dbTable.schema]);
} else {
const schemaResult = await connection.run(
@@ -87,16 +106,21 @@ More information on data types and accepted values:
dbSchema = detectedSchema[0][0];
}
- await connection.run(
- `
- CREATE TABLE ${dbTable.name} AS
+ const shouldFlatten = !dbTable.keepNestedFields;
+ const createTableQuery = shouldFlatten
+ ? `
+ CREATE TABLE ${tableName} AS
SELECT UNNEST(JSON_TRANSFORM($sourceData, $sourceSchema), recursive := true);
- `,
- {
- sourceData: dbData,
- sourceSchema: dbSchema,
- }
- );
+ `
+ : `
+ CREATE TABLE ${tableName} AS
+ SELECT row.* FROM (SELECT UNNEST(JSON_TRANSFORM($sourceData, $sourceSchema)) AS row) t;
+ `;
+
+ await connection.run(createTableQuery, {
+ sourceData: dbData,
+ sourceSchema: dbSchema,
+ });
}
const queryArgs = context.propsValue.args ?? [];
@@ -118,3 +142,23 @@ More information on data types and accepted values:
return results;
},
});
+
+function resolveTableRows(dbTable: { data?: unknown; file?: ApFile }): unknown {
+ if (Array.isArray(dbTable.data) ? dbTable.data.length > 0 : dbTable.data) {
+ return dbTable.data;
+ }
+
+ if (!dbTable.file) {
+ return dbTable.data;
+ }
+
+ const text = dbTable.file.data.toString('utf-8');
+ return dbTable.file.extension?.toLowerCase() === 'csv'
+ ? parseCsv(text, {
+ columns: true,
+ skip_empty_lines: true,
+ bom: true,
+ relax_column_count: true,
+ })
+ : JSON.parse(text.replace(/^/, ''));
+}
diff --git a/packages/pieces/community/duckdb/test/create-and-query-db.test.ts b/packages/pieces/community/duckdb/test/create-and-query-db.test.ts
new file mode 100644
index 00000000000..1537c703aae
--- /dev/null
+++ b/packages/pieces/community/duckdb/test/create-and-query-db.test.ts
@@ -0,0 +1,215 @@
+import { ApFile } from '@activepieces/pieces-framework';
+import { describe, expect, it } from 'vitest';
+import { createAndQueryDB } from '../src/lib/actions/create-and-query-db';
+
+function runAction(propsValue: Record<string, unknown>) {
+ return createAndQueryDB.run({ propsValue } as never);
+}
+
+function table(overrides: Record<string, unknown>) {
+ return { name: 't', data: [], schema: {}, ...overrides };
+}
+
+describe('JSON Data prop', () => {
+ it('loads rows straight from the JSON Data array', async () => {
+ const rows = await runAction({
+ tables: [
+ table({
+ data: [{ id: 1, name: 'a' }, { id: 2, name: 'b' }],
+ schema: { id: 'INTEGER', name: 'VARCHAR' },
+ }),
+ ],
+ query: 'SELECT * FROM t ORDER BY id',
+ });
+
+ expect(rows).toEqual([{ id: 1, name: 'a' }, { id: 2, name: 'b' }]);
+ });
+
+ it('an explicit schema restricts the table to only the listed columns', async () => {
+ const rows = await runAction({
+ tables: [
+ table({
+ data: [{ id: 1, name: 'a', extra: 'dropped' }],
+ schema: { id: 'INTEGER', name: 'VARCHAR' },
+ }),
+ ],
+ query: 'SELECT * FROM t',
+ });
+
+ expect(rows).toEqual([{ id: 1, name: 'a' }]);
+ });
+});
+
+describe('File prop (CSV/JSON)', () => {
+ it('loads rows from an uploaded JSON file when JSON Data is empty', async () => {
+ const file = new ApFile('rows.json', Buffer.from(JSON.stringify([{ id: 1, name: 'a' }])), 'json');
+
+ const rows = await runAction({
+ tables: [table({ data: [], file, schema: { id: 'INTEGER', name: 'VARCHAR' } })],
+ query: 'SELECT * FROM t',
+ });
+
+ expect(rows).toEqual([{ id: 1, name: 'a' }]);
+ });
+
+ it('loads rows from an uploaded CSV file, format detected from the extension', async () => {
+ const file = new ApFile('rows.csv', Buffer.from('id,name\n1,a\n2,b\n'), 'csv');
+
+ const rows = await runAction({
+ tables: [table({ data: [], file, schema: { id: 'INTEGER', name: 'VARCHAR' } })],
+ query: 'SELECT * FROM t ORDER BY id',
+ });
+
+ expect(rows).toEqual([{ id: 1, name: 'a' }, { id: 2, name: 'b' }]);
+ });
+
+ it('strips a UTF-8 BOM from a CSV header instead of corrupting the first column name', async () => {
+ const file = new ApFile('rows.csv', Buffer.from('id,name\n1,a\n'), 'csv');
+
+ const rows = await runAction({
+ tables: [table({ data: [], file, schema: { id: 'INTEGER', name: 'VARCHAR' } })],
+ query: 'SELECT * FROM t',
+ });
+
+ expect(rows).toEqual([{ id: 1, name: 'a' }]);
+ });
+
+ it('strips a UTF-8 BOM from a JSON file instead of failing to parse', async () => {
+ const file = new ApFile(
+ 'rows.json',
+ Buffer.from('' + JSON.stringify([{ id: 1, name: 'a' }])),
+ 'json'
+ );
+
+ const rows = await runAction({
+ tables: [table({ data: [], file, schema: { id: 'INTEGER', name: 'VARCHAR' } })],
+ query: 'SELECT * FROM t',
+ });
+
+ expect(rows).toEqual([{ id: 1, name: 'a' }]);
+ });
+
+ it('a ragged CSV row does not abort the whole file', async () => {
+ const file = new ApFile('rows.csv', Buffer.from('id,name\n1,a,extra\n2,b\n'), 'csv');
+
+ const rows = await runAction({
+ tables: [table({ data: [], file, schema: { id: 'INTEGER', name: 'VARCHAR' } })],
+ query: 'SELECT * FROM t ORDER BY id',
+ });
+
+ expect(rows).toEqual([{ id: 1, name: 'a' }, { id: 2, name: 'b' }]);
+ });
+
+ it('JSON Data takes priority over the file when both are set', async () => {
+ const file = new ApFile('rows.csv', Buffer.from('id,name\n99,ignored\n'), 'csv');
+
+ const rows = await runAction({
+ tables: [table({ data: [{ id: 1, name: 'from-data' }], file, schema: { id: 'INTEGER', name: 'VARCHAR' } })],
+ query: 'SELECT * FROM t',
+ });
+
+ expect(rows).toEqual([{ id: 1, name: 'from-data' }]);
+ });
+});
+
+describe('Nested schema and Keep Nested Fields', () => {
+ const NESTED_DATA = [{ id: 1, meta: { a: 'x', b: 2 }, tags: ['a', 'b'] }];
+ const NESTED_SCHEMA = { id: 'INTEGER', meta: 'STRUCT(a VARCHAR, b INTEGER)', tags: 'VARCHAR[]' };
+
+ it('flattens nested struct fields into top-level columns by default', async () => {
+ const rows = await runAction({
+ tables: [table({ data: NESTED_DATA, schema: NESTED_SCHEMA })],
+ query: 'SELECT * FROM t',
+ });
+
+ expect(rows).toEqual([{ id: 1, a: 'x', b: 2, tags: ['a', 'b'] }]);
+ });
+
+ it('a fresh array row seeds Keep Nested Fields to false (builder default for a checkbox sub-prop), so it still flattens', async () => {
+ const rows = await runAction({
+ tables: [table({ data: NESTED_DATA, schema: NESTED_SCHEMA, keepNestedFields: false })],
+ query: 'SELECT * FROM t',
+ });
+
+ expect(rows).toEqual([{ id: 1, a: 'x', b: 2, tags: ['a', 'b'] }]);
+ });
+
+ it('keepNestedFields: true keeps the nested object as a single struct column', async () => {
+ const rows = await runAction({
+ tables: [table({ data: NESTED_DATA, schema: NESTED_SCHEMA, keepNestedFields: true })],
+ query: 'SELECT * FROM t',
+ });
+
+ expect(rows).toEqual([{ id: 1, meta: { a: 'x', b: 2 }, tags: ['a', 'b'] }]);
+ });
+
+ it('a schema/data shape mismatch silently nulls the column rather than erroring (known DuckDB behavior)', async () => {
+ const rows = await runAction({
+ tables: [
+ table({
+ data: [{ id: 1, meta: 'not-a-struct' }],
+ schema: { id: 'INTEGER', meta: 'STRUCT(a VARCHAR, b INTEGER)' },
+ }),
+ ],
+ query: 'SELECT * FROM t',
+ });
+
+ expect(rows).toEqual([{ id: 1, a: null, b: null }]);
+ });
+});
+
+describe('Table Name is a SQL identifier, not a value (cannot be parameterized)', () => {
+ it('quotes the table name so multi-statement SQL injection cannot execute', async () => {
+ const maliciousName =
+ "z(v INT); CREATE TABLE injected_evil AS SELECT 'pwned' AS marker; CREATE TABLE legit_final";
+
+ const rows = await runAction({
+ tables: [table({ name: maliciousName, data: [{ id: 1 }] })],
+ query: 'SELECT table_name FROM information_schema.tables',
+ });
+
+ expect(rows).toEqual([{ table_name: maliciousName }]);
+ });
+
+ it('accepts a plain alphanumeric/underscore table name', async () => {
+ const rows = await runAction({
+ tables: [table({ name: 'valid_table_1', data: [{ id: 1 }] })],
+ query: 'SELECT * FROM valid_table_1',
+ });
+
+ expect(rows).toEqual([{ id: '1' }]);
+ });
+});
+
+describe('Multiple tables', () => {
+ it('loads more than one table and can join across them in a single query', async () => {
+ const rows = await runAction({
+ tables: [
+ table({
+ name: 'users',
+ data: [{ id: 1, name: 'a' }],
+ schema: { id: 'INTEGER', name: 'VARCHAR' },
+ }),
+ table({
+ name: 'orders',
+ data: [{ id: 10, user_id: 1, total: 5 }],
+ schema: { id: 'INTEGER', user_id: 'INTEGER', total: 'INTEGER' },
+ }),
+ ],
+ query: 'SELECT u.name, o.total FROM users u JOIN orders o ON o.user_id = u.id',
+ });
+
+ expect(rows).toEqual([{ name: 'a', total: 5 }]);
+ });
+});
+
+describe('Schema autodetection (schema left at its default {})', () => {
+ it('does not error on a fresh table row whose schema is still the default empty object, and autodetects types (note: integer columns come back as strings, since json_structure infers UBIGINT and getRowObjectsJson stringifies 64-bit ints to avoid precision loss)', async () => {
+ const rows = await runAction({
+ tables: [table({ data: [{ id: 1, name: 'a' }] })],
+ query: 'SELECT * FROM t',
+ });
+
+ expect(rows).toEqual([{ id: '1', name: 'a' }]);
+ });
+});
diff --git a/packages/pieces/community/duckdb/vitest.config.ts b/packages/pieces/community/duckdb/vitest.config.ts
new file mode 100644
index 00000000000..ba8ade4a178
--- /dev/null
+++ b/packages/pieces/community/duckdb/vitest.config.ts
@@ -0,0 +1,17 @@
+import path from 'path'
+import { defineConfig } from 'vitest/config'
+
+const repoRoot = path.resolve(__dirname, '../../../..')
+
+export default defineConfig({
+ test: {
+ globals: true,
+ environment: 'node',
+ },
+ resolve: {
+ alias: {
+ '@activepieces/pieces-framework': path.resolve(repoRoot, 'packages/pieces/framework/src/index.ts'),
+ '@activepieces/pieces-common': path.resolve(repoRoot, 'packages/pieces/common/src/index.ts'),
+ },
+ },
+})