AI Agents & MCPs & AI Workflow Automation • (~400 MCP servers for AI agents) • AI Automation / AI Agent with MCPs • AI Workflows & AI Agents • MCPs for AI Agents
TypeScript ai-agentai-agent-toolsai-agentsai-agents-frameworkmcpmcp-servermcp-toolsmcpsn8n-alternativeno-code-automationworkflowworkflow-automationworkflows
feat(duckdb): let AI agents query, profile and compare JSON datasets (#17074)
Kishan Parmarab7e40ee5305
diff --git a/bun.lock b/bun.lockindex e7a26f1af20..700d0fd7d83 100644--- a/bun.lock+++ b/bun.lock@@ -2880,7 +2880,7 @@},"packages/pieces/community/duckdb": {"name": "@activepieces/piece-duckdb",- "version": "0.0.9",+ "version": "0.1.0","dependencies": {"@activepieces/core-piece-types": "workspace:*","@activepieces/core-utils": "workspace:*",diff --git a/packages/pieces/community/duckdb/package.json b/packages/pieces/community/duckdb/package.jsonindex 352cf80c7bc..74dfecc031a 100644--- a/packages/pieces/community/duckdb/package.json+++ b/packages/pieces/community/duckdb/package.json@@ -1,6 +1,6 @@{"name": "@activepieces/piece-duckdb",- "version": "0.0.9",+ "version": "0.1.0","type": "commonjs","main": "./dist/src/index.js","types": "./dist/src/index.d.ts",diff --git a/packages/pieces/community/duckdb/src/index.ts b/packages/pieces/community/duckdb/src/index.tsindex 32e43183167..81007d0fc3e 100644--- a/packages/pieces/community/duckdb/src/index.ts+++ b/packages/pieces/community/duckdb/src/index.ts@@ -1,16 +1,17 @@import { createPiece, PieceAuth } from '@activepieces/pieces-framework';import { createAndQueryDB } from './lib/actions/create-and-query-db';+import { duckdbAiActions } from './lib/actions/ai';import { PieceCategory } from '@activepieces/pieces-framework';export const duckdb = createPiece({displayName: 'DuckDB',auth: PieceAuth.None(),- minimumSupportedRelease: '0.36.1',+ minimumSupportedRelease: '0.88.2',logoUrl: 'https://cdn.activepieces.com/pieces/duckdb.png',description: 'Run SQL queries on an in-memory DuckDB database.',categories: [PieceCategory.DEVELOPER_TOOLS],authors: ['danielpoonwj'],- actions: [createAndQueryDB],+ actions: [createAndQueryDB, ...duckdbAiActions],triggers: [],});diff --git a/packages/pieces/community/duckdb/src/lib/actions/ai/compare/compare-datasets.ts b/packages/pieces/community/duckdb/src/lib/actions/ai/compare/compare-datasets.tsnew file mode 100644index 00000000000..7c8df70cb31--- /dev/null+++ b/packages/pieces/community/duckdb/src/lib/actions/ai/compare/compare-datasets.ts@@ -0,0 +1,180 @@+import { createAction, Property } from '@activepieces/pieces-framework';+import { DuckDBConnection } from '@duckdb/node-api';+import { duckdbClient } from '../../../common/client';+import { duckdbCompareDatasetsOutputSchema } from '../../../output-schemas';++export const duckdbCompareDatasets = createAction({+ name: 'duckdb_compare_datasets',+ outputSchema: duckdbCompareDatasetsOutputSchema,+ displayName: 'Compare Datasets',+ description:+ 'Diff two JSON arrays of rows by key columns into added, removed and changed rows.',+ audience: 'ai',+ classification: 'READ',+ aiMetadata: {+ description:+ 'Matches rows of two JSON arrays (an older "before" and a newer "after" snapshot) on the key columns and returns rows only in after (added), rows only in before (removed), and matched rows whose other values differ (changed, with the changed column names). Use to find what changed between two exports or syncs without writing SQL. Fails if a key repeats within one side; run duckdb_deduplicate_rows on that side first.',+ idempotent: true,+ },+ props: {+ before: Property.Json({+ displayName: 'Before',+ description:+ 'The older dataset: a JSON array of row objects. May be empty.',+ required: true,+ }),+ after: Property.Json({+ displayName: 'After',+ description:+ 'The newer dataset: a JSON array of row objects. May be empty.',+ required: true,+ }),+ key_columns: Property.Array({+ displayName: 'Key Columns',+ description:+ 'Column names that identify a row on both sides, e.g. ["id"] or ["order_id", "line"]. Values must be unique within each side.',+ required: true,+ }),+ },+ async run(context) {+ const before = duckdbClient.parseRows({+ value: context.propsValue.before,+ propName: 'before',+ allowEmpty: true,+ });+ const after = duckdbClient.parseRows({+ value: context.propsValue.after,+ propName: 'after',+ allowEmpty: true,+ });+ const keyColumns = duckdbClient.parseKeyColumns({+ value: context.propsValue.key_columns,+ propName: 'key_columns',+ });+ if (keyColumns.length === 0) {+ throw new Error('key_columns needs at least one column name.');+ }+ if (before.length === 0 && after.length === 0) {+ return { added: [], removed: [], changed: [], unchanged_count: 0 };+ }+ return duckdbClient.withDatabase({+ run: async (connection) => {+ const structure = await duckdbClient.detectRowStructure({+ connection,+ rows: [...before, ...after],+ });+ await duckdbClient.loadIndexedRows({+ connection,+ name: 'before_rows',+ rows: before,+ structure,+ });+ await duckdbClient.loadIndexedRows({+ connection,+ name: 'after_rows',+ rows: after,+ structure,+ });+ await assertUniqueKeys({ connection, table: 'before_rows', side: 'before', keyColumns });+ await assertUniqueKeys({ connection, table: 'after_rows', side: 'after', keyColumns });++ const match = keyColumns+ .map(+ (column) =>+ `${duckdbClient.keyExpression({ alias: 'b', column })} IS NOT DISTINCT FROM ${duckdbClient.keyExpression({ alias: 'a', column })}`+ )+ .join(' AND ');++ const addedIndexes = duckdbClient.readIndexes({+ reader: await connection.runAndReadAll(+ `SELECT a.i FROM after_rows a WHERE NOT EXISTS (SELECT 1 FROM before_rows b WHERE ${match}) ORDER BY a.i`+ ),+ });+ const removedIndexes = duckdbClient.readIndexes({+ reader: await connection.runAndReadAll(+ `SELECT b.i FROM before_rows b WHERE NOT EXISTS (SELECT 1 FROM after_rows a WHERE ${match}) ORDER BY b.i`+ ),+ });+ const pairs = (+ await connection.runAndReadAll(+ `SELECT b.i, a.i, b.j IS DISTINCT FROM a.j FROM before_rows b JOIN after_rows a ON ${match} ORDER BY a.i`+ )+ ).getRows();++ const changed = pairs+ .filter((pair) => pair[2] === true)+ .map((pair) => {+ const beforeRow = before[Number(pair[0])];+ const afterRow = after[Number(pair[1])];+ return {+ key: Object.fromEntries(+ keyColumns.map((column) => [column, afterRow[column] ?? null])+ ),+ changed_columns: changedColumns({ beforeRow, afterRow }),+ before: beforeRow,+ after: afterRow,+ };+ });++ return {+ added: addedIndexes.map((i) => after[i]),+ removed: removedIndexes.map((i) => before[i]),+ changed,+ unchanged_count: pairs.length - changed.length,+ };+ },+ });+ },+});++async function assertUniqueKeys({+ connection,+ table,+ side,+ keyColumns,+}: {+ connection: DuckDBConnection;+ table: string;+ side: string;+ keyColumns: string[];+}): Promise<void> {+ const keys = keyColumns+ .map((column) => duckdbClient.keyExpression({ alias: 't', column }))+ .join(', ');+ const reader = await connection.runAndReadAll(+ `SELECT min(t.i) FROM ${table} t GROUP BY ${keys} HAVING count(*) > 1 LIMIT 1`+ );+ const duplicate = duckdbClient.readIndexes({ reader });+ if (duplicate.length > 0) {+ throw new Error(+ `${side} has more than one row with the key of ${side}[${duplicate[0]}]; key_columns must be unique within each side.`+ );+ }+}++function changedColumns({+ beforeRow,+ afterRow,+}: {+ beforeRow: Record<string, unknown>;+ afterRow: Record<string, unknown>;+}): string[] {+ const columns = [...new Set([...Object.keys(beforeRow), ...Object.keys(afterRow)])];+ return columns.filter(+ (column) =>+ columnState({ row: beforeRow, column }) !==+ columnState({ row: afterRow, column })+ );+}++function columnState({+ row,+ column,+}: {+ row: Record<string, unknown>;+ column: string;+}): string {+ return column in row+ ? duckdbClient.canonicalJson({ value: row[column] })+ : 'missing';+}diff --git a/packages/pieces/community/duckdb/src/lib/actions/ai/compare/deduplicate-rows.ts b/packages/pieces/community/duckdb/src/lib/actions/ai/compare/deduplicate-rows.tsnew file mode 100644index 00000000000..5764302c6a9--- /dev/null+++ b/packages/pieces/community/duckdb/src/lib/actions/ai/compare/deduplicate-rows.ts@@ -0,0 +1,94 @@+import { createAction, Property } from '@activepieces/pieces-framework';+import { duckdbClient } from '../../../common/client';+import { duckdbDeduplicateRowsOutputSchema } from '../../../output-schemas';++export const duckdbDeduplicateRows = createAction({+ name: 'duckdb_deduplicate_rows',+ outputSchema: duckdbDeduplicateRowsOutputSchema,+ displayName: 'Deduplicate Rows',+ description:+ 'Remove duplicate rows from a JSON array, matching on key columns or on the whole row.',+ audience: 'ai',+ classification: 'READ',+ aiMetadata: {+ description:+ 'Removes duplicates from a JSON array of row objects and returns the remaining rows unchanged, in their original order, plus how many were removed. Rows are duplicates when their key columns are equal, or when every value is equal if no key columns are given; keep chooses whether the first or last occurrence survives. Use before duckdb_compare_datasets when a key repeats.',+ idempotent: true,+ },+ props: {+ rows: Property.Json({+ displayName: 'Rows',+ description: 'A JSON array of row objects. May be empty.',+ required: true,+ }),+ key_columns: Property.Array({+ displayName: 'Key Columns',+ description:+ 'Column names that identify a duplicate, e.g. ["email"]. Leave empty to treat only fully identical rows as duplicates.',+ required: false,+ }),+ keep: Property.StaticDropdown({+ displayName: 'Keep',+ description:+ 'Which occurrence of each duplicate group to keep, by position in rows. Defaults to first.',+ required: false,+ defaultValue: 'first',+ options: {+ options: [+ { label: 'First', value: 'first' },+ { label: 'Last', value: 'last' },+ ],+ },+ }),+ },+ async run(context) {+ const rows = duckdbClient.parseRows({+ value: context.propsValue.rows,+ propName: 'rows',+ allowEmpty: true,+ });+ const keyColumns = duckdbClient.parseKeyColumns({+ value: context.propsValue.key_columns,+ propName: 'key_columns',+ });+ const keep = context.propsValue.keep ?? 'first';+ if (keep !== 'first' && keep !== 'last') {+ throw new Error('keep must be "first" or "last".');+ }+ if (rows.length === 0) {+ return { rows: [], removed_count: 0 };+ }+ return duckdbClient.withDatabase({+ run: async (connection) => {+ const structure = await duckdbClient.detectRowStructure({+ connection,+ rows,+ });+ await duckdbClient.loadIndexedRows({+ connection,+ name: 'data',+ rows,+ structure,+ });+ const partition =+ keyColumns.length > 0+ ? keyColumns+ .map((column) =>+ duckdbClient.keyExpression({ alias: 't', column })+ )+ .join(', ')+ : 't.j';+ const direction = keep === 'first' ? 'ASC' : 'DESC';+ const kept = duckdbClient.readIndexes({+ reader: await connection.runAndReadAll(+ `SELECT t.i FROM data t QUALIFY row_number() OVER (PARTITION BY ${partition} ORDER BY t.i ${direction}) = 1 ORDER BY t.i`+ ),+ });+ return {+ rows: kept.map((i) => rows[i]),+ removed_count: rows.length - kept.length,+ };+ },+ });+ },+});diff --git a/packages/pieces/community/duckdb/src/lib/actions/ai/index.ts b/packages/pieces/community/duckdb/src/lib/actions/ai/index.tsnew file mode 100644index 00000000000..5d653480b50--- /dev/null+++ b/packages/pieces/community/duckdb/src/lib/actions/ai/index.ts@@ -0,0 +1,13 @@+import { duckdbCompareDatasets } from './compare/compare-datasets';+import { duckdbDeduplicateRows } from './compare/deduplicate-rows';+import { duckdbDescribeData } from './profile/describe-data';+import { duckdbSummarizeData } from './profile/summarize-data';+import { duckdbRunQuery } from './query/run-query';++export const duckdbAiActions = [+ duckdbRunQuery,+ duckdbDescribeData,+ duckdbSummarizeData,+ duckdbCompareDatasets,+ duckdbDeduplicateRows,+];diff --git a/packages/pieces/community/duckdb/src/lib/actions/ai/profile/describe-data.ts b/packages/pieces/community/duckdb/src/lib/actions/ai/profile/describe-data.tsnew file mode 100644index 00000000000..ebfd1148323--- /dev/null+++ b/packages/pieces/community/duckdb/src/lib/actions/ai/profile/describe-data.ts@@ -0,0 +1,45 @@+import { createAction, Property } from '@activepieces/pieces-framework';+import { duckdbClient } from '../../../common/client';+import { duckdbDescribeDataOutputSchema } from '../../../output-schemas';++export const duckdbDescribeData = createAction({+ name: 'duckdb_describe_data',+ outputSchema: duckdbDescribeDataOutputSchema,+ displayName: 'Describe Data',+ description:+ 'Detect the columns and DuckDB types of a JSON array of rows.',+ audience: 'ai',+ classification: 'READ',+ aiMetadata: {+ description:+ 'Loads a JSON array of row objects into a throwaway in-memory DuckDB and returns each column with the type DuckDB detected, plus the row count. Use before writing SQL for duckdb_run_query to learn column names and types; use duckdb_summarize_data when you also need per-column statistics.',+ idempotent: true,+ },+ props: {+ rows: Property.Json({+ displayName: 'Rows',+ description:+ 'A JSON array of row objects, e.g. [{"id": 1, "email": "[email protected]"}]. At least one row.',+ required: true,+ }),+ },+ async run(context) {+ const rows = duckdbClient.parseRows({+ value: context.propsValue.rows,+ propName: 'rows',+ });+ return duckdbClient.withDatabase({+ run: async (connection) => {+ await duckdbClient.loadTable({ connection, name: 'data', rows });+ const reader = await connection.runAndReadAll('DESCRIBE data');+ return {+ columns: duckdbClient.readRows({ reader }).map((column) => ({+ name: column['column_name'],+ type: column['column_type'],+ })),+ row_count: rows.length,+ };+ },+ });+ },+});diff --git a/packages/pieces/community/duckdb/src/lib/actions/ai/profile/summarize-data.ts b/packages/pieces/community/duckdb/src/lib/actions/ai/profile/summarize-data.tsnew file mode 100644index 00000000000..c80d04f9016--- /dev/null+++ b/packages/pieces/community/duckdb/src/lib/actions/ai/profile/summarize-data.ts@@ -0,0 +1,42 @@+import { createAction, Property } from '@activepieces/pieces-framework';+import { duckdbClient } from '../../../common/client';+import { duckdbSummarizeDataOutputSchema } from '../../../output-schemas';++export const duckdbSummarizeData = createAction({+ name: 'duckdb_summarize_data',+ outputSchema: duckdbSummarizeDataOutputSchema,+ displayName: 'Summarize Data',+ description:+ 'Per-column statistics (min, max, distinct count, average, quartiles, null %) for a JSON array of rows.',+ audience: 'ai',+ classification: 'READ',+ aiMetadata: {+ description:+ 'Runs DuckDB SUMMARIZE over a JSON array of row objects and returns, for every column, its type, min, max, approximate distinct count, average, standard deviation, 25/50/75th percentiles, value count and null percentage. Use to profile or sanity-check a dataset; use duckdb_describe_data when column names and types are enough.',+ idempotent: true,+ },+ props: {+ rows: Property.Json({+ displayName: 'Rows',+ description:+ 'A JSON array of row objects, e.g. [{"id": 1, "amount": 12.5}]. At least one row.',+ required: true,+ }),+ },+ async run(context) {+ const rows = duckdbClient.parseRows({+ value: context.propsValue.rows,+ propName: 'rows',+ });+ return duckdbClient.withDatabase({+ run: async (connection) => {+ await duckdbClient.loadTable({ connection, name: 'data', rows });+ const reader = await connection.runAndReadAll('SUMMARIZE data');+ return {+ columns: duckdbClient.readRows({ reader }),+ row_count: rows.length,+ };+ },+ });+ },+});diff --git a/packages/pieces/community/duckdb/src/lib/actions/ai/query/run-query.ts b/packages/pieces/community/duckdb/src/lib/actions/ai/query/run-query.tsnew file mode 100644index 00000000000..9f4a36e69ce--- /dev/null+++ b/packages/pieces/community/duckdb/src/lib/actions/ai/query/run-query.ts@@ -0,0 +1,95 @@+import { createAction, Property } from '@activepieces/pieces-framework';+import { DuckDBValue } from '@duckdb/node-api';+import { duckdbClient } from '../../../common/client';+import { duckdbRunQueryOutputSchema } from '../../../output-schemas';++export const duckdbRunQuery = createAction({+ name: 'duckdb_run_query',+ outputSchema: duckdbRunQueryOutputSchema,+ displayName: 'Run SQL Query',+ description:+ 'Load JSON rows as tables in a temporary in-memory DuckDB and run one SQL query over them.',+ audience: 'ai',+ classification: 'SEARCH',+ aiMetadata: {+ description:+ 'Runs one SQL statement (DuckDB dialect) over JSON data you already have, loaded as named tables in a throwaway in-memory database with no file or network access. Use for joins, filters, aggregations, window functions and reshaping across one or more datasets; use duckdb_compare_datasets or duckdb_deduplicate_rows instead when you only need a diff or de-duplication. Pass user-supplied values as $1, $2, … in params rather than inside the SQL.',+ idempotent: true,+ },+ props: {+ tables: Property.Json({+ displayName: 'Tables',+ description:+ 'A JSON object mapping each table name to its rows, e.g. {"orders": [{"id": 1, "total": 9.5}], "customers": [{"id": 1, "name": "Ann"}]}. Columns come from the row keys and types are detected from the values; a nested object stays one STRUCT column, queried with dot notation (meta.city). Each table needs at least one row.',+ required: true,+ }),+ query: Property.LongText({+ displayName: 'SQL Query',+ description:+ 'One DuckDB SQL statement over the tables above, e.g. SELECT c.name, sum(o.total) AS spent FROM orders o JOIN customers c ON c.id = o.customer_id GROUP BY ALL. Quote table or column names that are not plain identifiers with double quotes.',+ required: true,+ }),+ params: Property.Json({+ displayName: 'Parameters',+ description:+ 'A JSON array of values bound to $1, $2, … in the query, in order, e.g. ["IN", 10, null]. Strings, numbers, booleans or null.',+ required: false,+ }),+ },+ async run(context) {+ const { tables, query, params } = context.propsValue;+ if (!duckdbClient.isPlainObject(tables) || Object.keys(tables).length === 0) {+ throw new Error(+ 'tables must be a JSON object mapping each table name to an array of row objects.'+ );+ }+ const values = parseParams({ params });+ return duckdbClient.withDatabase({+ run: async (connection) => {+ for (const [name, value] of Object.entries(tables)) {+ const rows = duckdbClient.parseRows({+ value,+ propName: `tables.${name}`,+ });+ await duckdbClient.loadTable({ connection, name, rows });+ }+ const reader = await connection.runAndReadAll(query, values);+ const rows = duckdbClient.readRows({ reader });+ return {+ rows,+ row_count: rows.length,+ columns: duckdbClient.readColumns({ reader }),+ };+ },+ });+ },+});++function parseParams({ params }: { params: unknown }): DuckDBValue[] {+ if (+ params === undefined ||+ params === null ||+ (duckdbClient.isPlainObject(params) && Object.keys(params).length === 0)+ ) {+ return [];+ }+ if (!Array.isArray(params)) {+ throw new Error('params must be a JSON array of values.');+ }+ const badIndex = params.findIndex((param) => !isScalar(param));+ if (badIndex !== -1) {+ throw new Error(+ `params[${badIndex}] must be a string, number, boolean or null.`+ );+ }+ return params.filter(isScalar);+}++function isScalar(value: unknown): value is string | number | boolean | null {+ return (+ value === null ||+ typeof value === 'string' ||+ typeof value === 'number' ||+ typeof value === 'boolean'+ );+}diff --git a/packages/pieces/community/duckdb/src/lib/actions/create-and-query-db.ts b/packages/pieces/community/duckdb/src/lib/actions/create-and-query-db.tsindex 66b627c3046..163ca309fc6 100644--- a/packages/pieces/community/duckdb/src/lib/actions/create-and-query-db.ts+++ b/packages/pieces/community/duckdb/src/lib/actions/create-and-query-db.ts@@ -8,11 +8,7 @@ export const createAndQueryDB = createAction({name: 'createAndQueryDB',displayName: 'Create and Query DB',description: 'Create DB from data and run query',- audience: 'both',- aiMetadata: {- description: 'Spins up a throwaway in-memory DuckDB, loads one or more tables from supplied JSON arrays (column names taken from object keys; schema can be given explicitly or autodetected), then runs a SQL query against them and returns the rows. Use to run SQL joins, aggregations, or transforms over JSON data you already have in the flow, without any external database. Use parameterized placeholders ($1, $2, ...) with the args list rather than interpolating values into the query string. Read-only with respect to external systems: the database exists only for this call, so repeating with the same input yields the same result.',- idempotent: true,- },+ audience: 'human',props: {markdown: Property.MarkDown({value: `diff --git a/packages/pieces/community/duckdb/src/lib/common/client.ts b/packages/pieces/community/duckdb/src/lib/common/client.tsnew file mode 100644index 00000000000..06a5d99feaa--- /dev/null+++ b/packages/pieces/community/duckdb/src/lib/common/client.ts@@ -0,0 +1,236 @@+import {+ DuckDBConnection,+ DuckDBInstance,+ DuckDBResultReader,+ DuckDBValue,+ DuckDBValueConverter,+ Json,+ JsonDuckDBValueConverter,+ quotedIdentifier,+} from '@duckdb/node-api';++async function withDatabase<T>({+ run,+}: {+ run: (connection: DuckDBConnection) => Promise<T>;+}): Promise<T> {+ const instance = await DuckDBInstance.create(':memory:');+ const connection = await instance.connect();+ try {+ await connection.run(`+ SET enable_logging = 0;+ SET enable_external_access = false;+ SET lock_configuration = true;+ `);+ return await run(connection);+ } finally {+ connection.closeSync();+ instance.closeSync();+ }+}++async function detectRowStructure({+ connection,+ rows,+}: {+ connection: DuckDBConnection;+ rows: Record<string, unknown>[];+}): Promise<Json> {+ const reader = await connection.runAndReadAll(+ 'SELECT json_structure($data)::VARCHAR AS structure',+ { data: JSON.stringify(rows) }+ );+ const structure: unknown = JSON.parse(String(reader.getRows()[0][0]));+ const rowStructure = Array.isArray(structure) ? structure[0] : undefined;+ if (!isJsonObject(rowStructure)) {+ throw new Error(+ 'Could not detect columns: every row must be a JSON object with at least one key.'+ );+ }+ return rowStructure;+}++async function loadTable({+ connection,+ name,+ rows,+}: {+ connection: DuckDBConnection;+ name: string;+ rows: Record<string, unknown>[];+}): Promise<void> {+ const structure = await detectRowStructure({ connection, rows });+ await connection.run(+ `CREATE TABLE ${quotedIdentifier(+ name+ )} AS SELECT r.* FROM (SELECT UNNEST(JSON_TRANSFORM($data, $schema)) AS r) t`,+ {+ data: JSON.stringify(rows),+ schema: JSON.stringify([structure]),+ }+ );+}++async function loadIndexedRows({+ connection,+ name,+ rows,+ structure,+}: {+ connection: DuckDBConnection;+ name: string;+ rows: Record<string, unknown>[];+ structure: Json;+}): Promise<void> {+ await connection.run(+ `CREATE TABLE ${quotedIdentifier(+ name+ )} AS SELECT x.i AS i, x.r AS r, x.j AS j FROM (SELECT UNNEST(JSON_TRANSFORM($data, $schema)) AS x) t`,+ {+ data: JSON.stringify(+ rows.map((row, i) => ({ i, r: row, j: canonicalJson({ value: row }) }))+ ),+ schema: JSON.stringify([{ i: 'UBIGINT', r: structure, j: 'VARCHAR' }]),+ }+ );+}++function canonicalJson({ value }: { value: unknown }): string {+ if (Array.isArray(value)) {+ return `[${value.map((item) => canonicalJson({ value: item })).join(',')}]`;+ }+ if (isPlainObject(value)) {+ return `{${Object.keys(value)+ .sort()+ .map(+ (key) => `${JSON.stringify(key)}:${canonicalJson({ value: value[key] })}`+ )+ .join(',')}}`;+ }+ return JSON.stringify(value);+}++function readRows({+ reader,+}: {+ reader: DuckDBResultReader;+}): Record<string, Json>[] {+ return reader+ .convertRowObjects(jsonConverter)+ .map((row) =>+ Object.fromEntries(+ Object.entries(row).map(([key, value]) => [key, value ?? null])+ )+ );+}++function readColumns({+ reader,+}: {+ reader: DuckDBResultReader;+}): { name: string; type: string }[] {+ const types = reader.columnTypes();+ return reader+ .columnNames()+ .map((name, i) => ({ name, type: types[i].toString() }));+}++function readIndexes({ reader }: { reader: DuckDBResultReader }): number[] {+ return reader.getRows().map((row) => Number(row[0]));+}++function parseRows({+ value,+ propName,+ allowEmpty = false,+}: {+ value: unknown;+ propName: string;+ allowEmpty?: boolean;+}): Record<string, unknown>[] {+ if (!Array.isArray(value)) {+ throw new Error(`${propName} must be a JSON array of row objects.`);+ }+ if (!allowEmpty && value.length === 0) {+ throw new Error(+ `${propName} has no rows; columns are detected from the rows, so give at least one.`+ );+ }+ const badIndex = value.findIndex((row) => !isPlainObject(row));+ if (badIndex !== -1) {+ throw new Error(+ `${propName}[${badIndex}] is not a JSON object; every row must be an object of column → value.`+ );+ }+ return value.filter(isPlainObject);+}++function parseKeyColumns({+ value,+ propName,+}: {+ value: unknown;+ propName: string;+}): string[] {+ const columns: unknown[] = Array.isArray(value) ? value : [];+ const badIndex = columns.findIndex(+ (column) => typeof column !== 'string' || column.length === 0+ );+ if (badIndex !== -1) {+ throw new Error(`${propName}[${badIndex}] must be a non-empty column name.`);+ }+ return columns.filter(+ (column): column is string => typeof column === 'string'+ );+}++function keyExpression({+ alias,+ column,+}: {+ alias: string;+ column: string;+}): string {+ return `${alias}.r[${quoteLiteral(column)}]`;+}++function quoteLiteral(value: string): string {+ return `'${value.replace(/'/g, "''")}'`;+}++function isPlainObject(value: unknown): value is Record<string, unknown> {+ return typeof value === 'object' && value !== null && !Array.isArray(value);+}++function isJsonObject(value: unknown): value is { [key: string]: Json } {+ return isPlainObject(value) && Object.keys(value).length > 0;+}++const jsonConverter: DuckDBValueConverter<Json> = (+ value: DuckDBValue,+ type,+ converter+) => {+ if (typeof value === 'bigint') {+ return value >= BigInt(Number.MIN_SAFE_INTEGER) &&+ value <= BigInt(Number.MAX_SAFE_INTEGER)+ ? Number(value)+ : value.toString();+ }+ return JsonDuckDBValueConverter(value, type, converter);+};++export const duckdbClient = {+ withDatabase,+ detectRowStructure,+ loadTable,+ loadIndexedRows,+ canonicalJson,+ readRows,+ readColumns,+ readIndexes,+ parseRows,+ parseKeyColumns,+ keyExpression,+ isPlainObject,+};diff --git a/packages/pieces/community/duckdb/src/lib/output-schemas.ts b/packages/pieces/community/duckdb/src/lib/output-schemas.tsnew file mode 100644index 00000000000..a4d45322eba--- /dev/null+++ b/packages/pieces/community/duckdb/src/lib/output-schemas.ts@@ -0,0 +1,102 @@+import { OutputSchema, OutputSchemaField } from '@activepieces/pieces-framework';++const columnFields: OutputSchemaField[] = [+ { key: 'name', label: 'Column Name' },+ { key: 'type', label: 'DuckDB Type' },+];++export const duckdbRunQueryOutputSchema: OutputSchema = {+ fields: [+ {+ key: 'rows',+ label: 'Rows',+ description:+ 'One object per result row, keyed by the column names the query selected.',+ },+ { key: 'row_count', label: 'Row Count', format: 'number' },+ {+ key: 'columns',+ label: 'Columns',+ labelKey: 'name',+ listItems: columnFields,+ },+ ],+};++export const duckdbDescribeDataOutputSchema: OutputSchema = {+ fields: [+ {+ key: 'columns',+ label: 'Columns',+ labelKey: 'name',+ listItems: columnFields,+ },+ { key: 'row_count', label: 'Row Count', format: 'number' },+ ],+};++export const duckdbSummarizeDataOutputSchema: OutputSchema = {+ fields: [+ {+ key: 'columns',+ label: 'Column Statistics',+ labelKey: 'column_name',+ listItems: [+ { key: 'column_name', label: 'Column Name' },+ { key: 'column_type', label: 'DuckDB Type' },+ { key: 'min', label: 'Minimum' },+ { key: 'max', label: 'Maximum' },+ { key: 'approx_unique', label: 'Approximate Distinct Values', format: 'number' },+ { key: 'avg', label: 'Average' },+ { key: 'std', label: 'Standard Deviation' },+ { key: 'q25', label: '25th Percentile' },+ { key: 'q50', label: 'Median' },+ { key: 'q75', label: '75th Percentile' },+ { key: 'count', label: 'Value Count', format: 'number' },+ { key: 'null_percentage', label: 'Null Percentage' },+ ],+ },+ { key: 'row_count', label: 'Row Count', format: 'number' },+ ],+};++export const duckdbCompareDatasetsOutputSchema: OutputSchema = {+ fields: [+ {+ key: 'added',+ label: 'Added Rows',+ description: 'Rows from after whose key is not in before, as given.',+ },+ {+ key: 'removed',+ label: 'Removed Rows',+ description: 'Rows from before whose key is not in after, as given.',+ },+ {+ key: 'changed',+ label: 'Changed Rows',+ listItems: [+ {+ key: 'key',+ label: 'Key',+ description: 'The key column values of the matched row.',+ },+ { key: 'changed_columns', label: 'Changed Columns' },+ { key: 'before', label: 'Before', description: 'The row as it was in before.' },+ { key: 'after', label: 'After', description: 'The row as it is in after.' },+ ],+ },+ { key: 'unchanged_count', label: 'Unchanged Rows', format: 'number' },+ ],+};++export const duckdbDeduplicateRowsOutputSchema: OutputSchema = {+ fields: [+ {+ key: 'rows',+ label: 'Rows',+ description: 'The rows that remain, unchanged and in their original order.',+ },+ { key: 'removed_count', label: 'Removed Rows', format: 'number' },+ ],+};diff --git a/packages/pieces/community/duckdb/test/ai-actions.test.ts b/packages/pieces/community/duckdb/test/ai-actions.test.tsnew file mode 100644index 00000000000..9dd6d2f3ddf--- /dev/null+++ b/packages/pieces/community/duckdb/test/ai-actions.test.ts@@ -0,0 +1,188 @@+import { describe, expect, it } from 'vitest';+import { duckdbCompareDatasets } from '../src/lib/actions/ai/compare/compare-datasets';+import { duckdbDeduplicateRows } from '../src/lib/actions/ai/compare/deduplicate-rows';+import { duckdbDescribeData } from '../src/lib/actions/ai/profile/describe-data';+import { duckdbRunQuery } from '../src/lib/actions/ai/query/run-query';++function runAction({+ action,+ propsValue,+}: {+ action: { run: unknown };+ propsValue: Record<string, unknown>;+}) {+ return Reflect.apply(Function.prototype.call, action.run, [+ action,+ { propsValue },+ ]);+}++describe('duckdb_run_query', () => {+ it('joins tables, binds scalar params and returns integers as numbers', async () => {+ const result = await runAction({+ action: duckdbRunQuery,+ propsValue: {+ tables: {+ orders: [+ { id: 1, customer_id: 1, total: 9.5 },+ { id: 2, customer_id: 1, total: 3 },+ { id: 3, customer_id: 2, total: 7.25 },+ ],+ customers: [{ id: 1, name: 'Ann' }, { id: 2, name: 'Bo' }],+ },+ query:+ 'SELECT c.name, count(*) AS n FROM orders o JOIN customers c ON c.id = o.customer_id WHERE o.total > $1 GROUP BY ALL ORDER BY c.name',+ params: [1],+ },+ });++ expect(result.rows).toEqual([{ name: 'Ann', n: 2 }, { name: 'Bo', n: 1 }]);+ expect(result.row_count).toBe(2);+ });++ it('binds null params', async () => {+ const result = await runAction({+ action: duckdbRunQuery,+ propsValue: {+ tables: { t: [{ a: 1 }] },+ query: 'SELECT $1 IS NULL AS is_null',+ params: [null],+ },+ });++ expect(result.rows).toEqual([{ is_null: true }]);+ });++ it('refuses file access', async () => {+ await expect(+ runAction({+ action: duckdbRunQuery,+ propsValue: {+ tables: { t: [{ a: 1 }] },+ query: "SELECT * FROM read_csv('/etc/passwd')",+ },+ })+ ).rejects.toThrow(/disabled/);+ });+});++describe('duckdb_describe_data', () => {+ it('keeps nested objects as STRUCT columns', async () => {+ const result = await runAction({+ action: duckdbDescribeData,+ propsValue: { rows: [{ id: 1, meta: { plan: 'pro' } }] },+ });++ expect(result.columns).toEqual([+ { name: 'id', type: 'UBIGINT' },+ { name: 'meta', type: 'STRUCT(plan VARCHAR)' },+ ]);+ });+});++describe('duckdb_compare_datasets', () => {+ it('splits rows into added, removed, changed and unchanged', async () => {+ const result = await runAction({+ action: duckdbCompareDatasets,+ propsValue: {+ before: [{ id: 1, name: 'a' }, { id: 2, name: 'b' }, { id: 3, name: 'c' }],+ after: [{ id: 2, name: 'B' }, { id: 3, name: 'c' }, { id: 4, name: 'd' }],+ key_columns: ['id'],+ },+ });++ expect(result.added).toEqual([{ id: 4, name: 'd' }]);+ expect(result.removed).toEqual([{ id: 1, name: 'a' }]);+ expect(result.changed).toEqual([+ {+ key: { id: 2 },+ changed_columns: ['name'],+ before: { id: 2, name: 'b' },+ after: { id: 2, name: 'B' },+ },+ ]);+ expect(result.unchanged_count).toBe(1);+ });++ it('treats a nested key present as null as a change from a missing key', async () => {+ const result = await runAction({+ action: duckdbCompareDatasets,+ propsValue: {+ before: [{ id: 1, meta: { a: 1 } }],+ after: [{ id: 1, meta: { a: 1, b: null } }],+ key_columns: ['id'],+ },+ });++ expect(result.changed).toHaveLength(1);+ expect(result.changed[0].changed_columns).toEqual(['meta']);+ expect(result.unchanged_count).toBe(0);+ });++ it('ignores key order inside nested objects', async () => {+ const result = await runAction({+ action: duckdbCompareDatasets,+ propsValue: {+ before: [{ id: 1, meta: { a: 1, b: 2 } }],+ after: [{ id: 1, meta: { b: 2, a: 1 } }],+ key_columns: ['id'],+ },+ });++ expect(result.changed).toEqual([]);+ expect(result.unchanged_count).toBe(1);+ });++ it('refuses a key that repeats within one side', async () => {+ await expect(+ runAction({+ action: duckdbCompareDatasets,+ propsValue: {+ before: [{ id: 1 }, { id: 1 }],+ after: [{ id: 1 }],+ key_columns: ['id'],+ },+ })+ ).rejects.toThrow(/more than one row/);+ });+});++describe('duckdb_deduplicate_rows', () => {+ const rows = [+ { email: '[email protected]', n: 1 },+ { email: '[email protected]', n: 2 },+ { email: '[email protected]', n: 3 },+ ];++ it('keeps the first occurrence by default', async () => {+ const result = await runAction({+ action: duckdbDeduplicateRows,+ propsValue: { rows, key_columns: ['email'] },+ });++ expect(result.rows).toEqual([rows[0], rows[1]]);+ expect(result.removed_count).toBe(1);+ });++ it('keeps the last occurrence when asked', async () => {+ const result = await runAction({+ action: duckdbDeduplicateRows,+ propsValue: { rows, key_columns: ['email'], keep: 'last' },+ });++ expect(result.rows).toEqual([rows[1], rows[2]]);+ });++ it('keeps rows that differ only by a nested null', async () => {+ const result = await runAction({+ action: duckdbDeduplicateRows,+ propsValue: {+ rows: [{ meta: { a: 1 } }, { meta: { a: 1, b: null } }, { meta: { a: 1 } }],+ key_columns: [],+ },+ });++ expect(result.rows).toEqual([{ meta: { a: 1 } }, { meta: { a: 1, b: null } }]);+ expect(result.removed_count).toBe(1);+ });+});