AI Agents & MCPs & AI Workflow Automation • (~400 MCP servers for AI agents) • AI Automation / AI Agent with MCPs • AI Workflows & AI Agents • MCPs for AI Agents
TypeScript ai-agentai-agent-toolsai-agentsai-agents-frameworkmcpmcp-servermcp-toolsmcpsn8n-alternativeno-code-automationworkflowworkflow-automationworkflows
feat(duckdb): Usability enhancements (#15584)
Daniel Poonde6c7d12324e
diff --git a/bun.lock b/bun.lockindex e31cb132040..5e6d3e85e63 100644--- a/bun.lock+++ b/bun.lock@@ -2786,16 +2786,18 @@},"packages/pieces/community/duckdb": {"name": "@activepieces/piece-duckdb",- "version": "0.0.8",+ "version": "0.0.9","dependencies": {"@activepieces/core-piece-types": "workspace:*","@activepieces/core-utils": "workspace:*","@activepieces/pieces-common": "workspace:*","@activepieces/pieces-framework": "workspace:*","@duckdb/node-api": "1.4.3-r.2",+ "csv-parse": "5.6.0",},"devDependencies": {"tslib": "^2.3.0",+ "vitest": "3.2.6",},},"packages/pieces/community/dumpling-ai": {diff --git a/packages/pieces/community/duckdb/package.json b/packages/pieces/community/duckdb/package.jsonindex a96832fb42f..352cf80c7bc 100644--- a/packages/pieces/community/duckdb/package.json+++ b/packages/pieces/community/duckdb/package.json@@ -1,6 +1,6 @@{"name": "@activepieces/piece-duckdb",- "version": "0.0.8",+ "version": "0.0.9","type": "commonjs","main": "./dist/src/index.js","types": "./dist/src/index.d.ts",@@ -8,15 +8,18 @@"@activepieces/pieces-common": "workspace:*","@activepieces/pieces-framework": "workspace:*","@duckdb/node-api": "1.4.3-r.2",+ "csv-parse": "5.6.0","@activepieces/core-piece-types": "workspace:*","@activepieces/core-utils": "workspace:*"},"devDependencies": {- "tslib": "^2.3.0"+ "tslib": "^2.3.0",+ "vitest": "3.2.6"},"scripts": {"build": "tsc -p tsconfig.lib.json && cp package.json dist/","bundle": "node ../../../../dist/packages/cli/src/index.js pieces bundle",- "lint": "eslint 'src/**/*.ts'"+ "lint": "eslint 'src/**/*.ts'",+ "test": "vitest run"}}diff --git a/packages/pieces/community/duckdb/src/lib/actions/create-and-query-db.ts b/packages/pieces/community/duckdb/src/lib/actions/create-and-query-db.tsindex 3be14a2abfd..66b627c3046 100644--- a/packages/pieces/community/duckdb/src/lib/actions/create-and-query-db.ts+++ b/packages/pieces/community/duckdb/src/lib/actions/create-and-query-db.ts@@ -1,6 +1,8 @@-import { createAction, Property } from '@activepieces/pieces-framework';+import { ApFile, createAction, Property } from '@activepieces/pieces-framework';import { DuckDBInstance } from '@duckdb/node-api';+import { quotedIdentifier } from '@duckdb/node-api/lib/sql';+import { parse as parseCsv } from 'csv-parse/sync';export const createAndQueryDB = createAction({name: 'createAndQueryDB',@@ -29,16 +31,24 @@ export const createAndQueryDB = createAction({data: Property.Json({displayName: 'JSON Data',description:- 'Table data in the form of a JSON array of objects. Object keys would be taken as column names.',- required: true,+ 'Table data as a JSON array of objects. Object keys are taken as column names. Takes priority over the file below if set.',+ required: false,defaultValue: [],}),+ file: Property.File({+ displayName: 'CSV/JSON File',+ description:+ 'Upload a CSV or JSON file as the table source, used only if JSON Data above is empty. Format is detected from the file extension.',+ required: false,+ }),schema: Property.Json({displayName: 'Schema in JSON format',description: `Provide the schema as a JSON object. Note that only the specified columns would be loaded.Leave empty to autodetect schema, although it is recommended to specify the schema for consistency and avoid unexpected behaviour.+A column type that doesn't match the actual shape of the row data (e.g. a STRUCT/array type for a column that's a plain string, which is always the case for CSV-sourced tables) silently loads as null instead of erroring - double check this against autodetected schema if a column is unexpectedly empty.+More information on data types and accepted values:- https://duckdb.org/2023/03/03/json- https://duckdb.org/docs/stable/sql/data_types/overview@@ -46,6 +56,13 @@ More information on data types and accepted values:required: false,defaultValue: {},}),+ keepNestedFields: Property.Checkbox({+ displayName: 'Keep Nested Fields',+ description:+ 'When off, a nested object/struct field is split into individual top-level columns (e.g. "meta": {"a":1} becomes column "a"). Turn on to keep it as a single nested column instead, queryable with dot notation (e.g. "meta.a").',+ required: false,+ defaultValue: false,+ }),},}),query: Property.ShortText({@@ -72,10 +89,12 @@ More information on data types and accepted values:const dbTables: any[] = context.propsValue.tables ?? [];for (const dbTable of dbTables) {- const dbData = JSON.stringify(dbTable.data);+ const tableName = quotedIdentifier(dbTable.name);+ const dbData = JSON.stringify(resolveTableRows(dbTable));let dbSchema = null;- if (dbTable.schema) {+ const hasSchema = dbTable.schema && Object.keys(dbTable.schema).length > 0;+ if (hasSchema) {dbSchema = JSON.stringify([dbTable.schema]);} else {const schemaResult = await connection.run(@@ -87,16 +106,21 @@ More information on data types and accepted values:dbSchema = detectedSchema[0][0];}- await connection.run(- `- CREATE TABLE ${dbTable.name} AS+ const shouldFlatten = !dbTable.keepNestedFields;+ const createTableQuery = shouldFlatten+ ? `+ CREATE TABLE ${tableName} ASSELECT UNNEST(JSON_TRANSFORM($sourceData, $sourceSchema), recursive := true);- `,- {- sourceData: dbData,- sourceSchema: dbSchema,- }- );+ `+ : `+ CREATE TABLE ${tableName} AS+ SELECT row.* FROM (SELECT UNNEST(JSON_TRANSFORM($sourceData, $sourceSchema)) AS row) t;+ `;++ await connection.run(createTableQuery, {+ sourceData: dbData,+ sourceSchema: dbSchema,+ });}const queryArgs = context.propsValue.args ?? [];@@ -118,3 +142,23 @@ More information on data types and accepted values:return results;},});++function resolveTableRows(dbTable: { data?: unknown; file?: ApFile }): unknown {+ if (Array.isArray(dbTable.data) ? dbTable.data.length > 0 : dbTable.data) {+ return dbTable.data;+ }++ if (!dbTable.file) {+ return dbTable.data;+ }++ const text = dbTable.file.data.toString('utf-8');+ return dbTable.file.extension?.toLowerCase() === 'csv'+ ? parseCsv(text, {+ columns: true,+ skip_empty_lines: true,+ bom: true,+ relax_column_count: true,+ })+ : JSON.parse(text.replace(/^/, ''));+}diff --git a/packages/pieces/community/duckdb/test/create-and-query-db.test.ts b/packages/pieces/community/duckdb/test/create-and-query-db.test.tsnew file mode 100644index 00000000000..1537c703aae--- /dev/null+++ b/packages/pieces/community/duckdb/test/create-and-query-db.test.ts@@ -0,0 +1,215 @@+import { ApFile } from '@activepieces/pieces-framework';+import { describe, expect, it } from 'vitest';+import { createAndQueryDB } from '../src/lib/actions/create-and-query-db';++function runAction(propsValue: Record<string, unknown>) {+ return createAndQueryDB.run({ propsValue } as never);+}++function table(overrides: Record<string, unknown>) {+ return { name: 't', data: [], schema: {}, ...overrides };+}++describe('JSON Data prop', () => {+ it('loads rows straight from the JSON Data array', async () => {+ const rows = await runAction({+ tables: [+ table({+ data: [{ id: 1, name: 'a' }, { id: 2, name: 'b' }],+ schema: { id: 'INTEGER', name: 'VARCHAR' },+ }),+ ],+ query: 'SELECT * FROM t ORDER BY id',+ });++ expect(rows).toEqual([{ id: 1, name: 'a' }, { id: 2, name: 'b' }]);+ });++ it('an explicit schema restricts the table to only the listed columns', async () => {+ const rows = await runAction({+ tables: [+ table({+ data: [{ id: 1, name: 'a', extra: 'dropped' }],+ schema: { id: 'INTEGER', name: 'VARCHAR' },+ }),+ ],+ query: 'SELECT * FROM t',+ });++ expect(rows).toEqual([{ id: 1, name: 'a' }]);+ });+});++describe('File prop (CSV/JSON)', () => {+ it('loads rows from an uploaded JSON file when JSON Data is empty', async () => {+ const file = new ApFile('rows.json', Buffer.from(JSON.stringify([{ id: 1, name: 'a' }])), 'json');++ const rows = await runAction({+ tables: [table({ data: [], file, schema: { id: 'INTEGER', name: 'VARCHAR' } })],+ query: 'SELECT * FROM t',+ });++ expect(rows).toEqual([{ id: 1, name: 'a' }]);+ });++ it('loads rows from an uploaded CSV file, format detected from the extension', async () => {+ const file = new ApFile('rows.csv', Buffer.from('id,name\n1,a\n2,b\n'), 'csv');++ const rows = await runAction({+ tables: [table({ data: [], file, schema: { id: 'INTEGER', name: 'VARCHAR' } })],+ query: 'SELECT * FROM t ORDER BY id',+ });++ expect(rows).toEqual([{ id: 1, name: 'a' }, { id: 2, name: 'b' }]);+ });++ it('strips a UTF-8 BOM from a CSV header instead of corrupting the first column name', async () => {+ const file = new ApFile('rows.csv', Buffer.from('id,name\n1,a\n'), 'csv');++ const rows = await runAction({+ tables: [table({ data: [], file, schema: { id: 'INTEGER', name: 'VARCHAR' } })],+ query: 'SELECT * FROM t',+ });++ expect(rows).toEqual([{ id: 1, name: 'a' }]);+ });++ it('strips a UTF-8 BOM from a JSON file instead of failing to parse', async () => {+ const file = new ApFile(+ 'rows.json',+ Buffer.from('' + JSON.stringify([{ id: 1, name: 'a' }])),+ 'json'+ );++ const rows = await runAction({+ tables: [table({ data: [], file, schema: { id: 'INTEGER', name: 'VARCHAR' } })],+ query: 'SELECT * FROM t',+ });++ expect(rows).toEqual([{ id: 1, name: 'a' }]);+ });++ it('a ragged CSV row does not abort the whole file', async () => {+ const file = new ApFile('rows.csv', Buffer.from('id,name\n1,a,extra\n2,b\n'), 'csv');++ const rows = await runAction({+ tables: [table({ data: [], file, schema: { id: 'INTEGER', name: 'VARCHAR' } })],+ query: 'SELECT * FROM t ORDER BY id',+ });++ expect(rows).toEqual([{ id: 1, name: 'a' }, { id: 2, name: 'b' }]);+ });++ it('JSON Data takes priority over the file when both are set', async () => {+ const file = new ApFile('rows.csv', Buffer.from('id,name\n99,ignored\n'), 'csv');++ const rows = await runAction({+ tables: [table({ data: [{ id: 1, name: 'from-data' }], file, schema: { id: 'INTEGER', name: 'VARCHAR' } })],+ query: 'SELECT * FROM t',+ });++ expect(rows).toEqual([{ id: 1, name: 'from-data' }]);+ });+});++describe('Nested schema and Keep Nested Fields', () => {+ const NESTED_DATA = [{ id: 1, meta: { a: 'x', b: 2 }, tags: ['a', 'b'] }];+ const NESTED_SCHEMA = { id: 'INTEGER', meta: 'STRUCT(a VARCHAR, b INTEGER)', tags: 'VARCHAR[]' };++ it('flattens nested struct fields into top-level columns by default', async () => {+ const rows = await runAction({+ tables: [table({ data: NESTED_DATA, schema: NESTED_SCHEMA })],+ query: 'SELECT * FROM t',+ });++ expect(rows).toEqual([{ id: 1, a: 'x', b: 2, tags: ['a', 'b'] }]);+ });++ it('a fresh array row seeds Keep Nested Fields to false (builder default for a checkbox sub-prop), so it still flattens', async () => {+ const rows = await runAction({+ tables: [table({ data: NESTED_DATA, schema: NESTED_SCHEMA, keepNestedFields: false })],+ query: 'SELECT * FROM t',+ });++ expect(rows).toEqual([{ id: 1, a: 'x', b: 2, tags: ['a', 'b'] }]);+ });++ it('keepNestedFields: true keeps the nested object as a single struct column', async () => {+ const rows = await runAction({+ tables: [table({ data: NESTED_DATA, schema: NESTED_SCHEMA, keepNestedFields: true })],+ query: 'SELECT * FROM t',+ });++ expect(rows).toEqual([{ id: 1, meta: { a: 'x', b: 2 }, tags: ['a', 'b'] }]);+ });++ it('a schema/data shape mismatch silently nulls the column rather than erroring (known DuckDB behavior)', async () => {+ const rows = await runAction({+ tables: [+ table({+ data: [{ id: 1, meta: 'not-a-struct' }],+ schema: { id: 'INTEGER', meta: 'STRUCT(a VARCHAR, b INTEGER)' },+ }),+ ],+ query: 'SELECT * FROM t',+ });++ expect(rows).toEqual([{ id: 1, a: null, b: null }]);+ });+});++describe('Table Name is a SQL identifier, not a value (cannot be parameterized)', () => {+ it('quotes the table name so multi-statement SQL injection cannot execute', async () => {+ const maliciousName =+ "z(v INT); CREATE TABLE injected_evil AS SELECT 'pwned' AS marker; CREATE TABLE legit_final";++ const rows = await runAction({+ tables: [table({ name: maliciousName, data: [{ id: 1 }] })],+ query: 'SELECT table_name FROM information_schema.tables',+ });++ expect(rows).toEqual([{ table_name: maliciousName }]);+ });++ it('accepts a plain alphanumeric/underscore table name', async () => {+ const rows = await runAction({+ tables: [table({ name: 'valid_table_1', data: [{ id: 1 }] })],+ query: 'SELECT * FROM valid_table_1',+ });++ expect(rows).toEqual([{ id: '1' }]);+ });+});++describe('Multiple tables', () => {+ it('loads more than one table and can join across them in a single query', async () => {+ const rows = await runAction({+ tables: [+ table({+ name: 'users',+ data: [{ id: 1, name: 'a' }],+ schema: { id: 'INTEGER', name: 'VARCHAR' },+ }),+ table({+ name: 'orders',+ data: [{ id: 10, user_id: 1, total: 5 }],+ schema: { id: 'INTEGER', user_id: 'INTEGER', total: 'INTEGER' },+ }),+ ],+ query: 'SELECT u.name, o.total FROM users u JOIN orders o ON o.user_id = u.id',+ });++ expect(rows).toEqual([{ name: 'a', total: 5 }]);+ });+});++describe('Schema autodetection (schema left at its default {})', () => {+ it('does not error on a fresh table row whose schema is still the default empty object, and autodetects types (note: integer columns come back as strings, since json_structure infers UBIGINT and getRowObjectsJson stringifies 64-bit ints to avoid precision loss)', async () => {+ const rows = await runAction({+ tables: [table({ data: [{ id: 1, name: 'a' }] })],+ query: 'SELECT * FROM t',+ });++ expect(rows).toEqual([{ id: '1', name: 'a' }]);+ });+});diff --git a/packages/pieces/community/duckdb/vitest.config.ts b/packages/pieces/community/duckdb/vitest.config.tsnew file mode 100644index 00000000000..ba8ade4a178--- /dev/null+++ b/packages/pieces/community/duckdb/vitest.config.ts@@ -0,0 +1,17 @@+import path from 'path'+import { defineConfig } from 'vitest/config'++const repoRoot = path.resolve(__dirname, '../../../..')++export default defineConfig({+ test: {+ globals: true,+ environment: 'node',+ },+ resolve: {+ alias: {+ '@activepieces/pieces-framework': path.resolve(repoRoot, 'packages/pieces/framework/src/index.ts'),+ '@activepieces/pieces-common': path.resolve(repoRoot, 'packages/pieces/common/src/index.ts'),+ },+ },+})