Skip to content

AI Agents & MCPs & AI Workflow Automation • (~400 MCP servers for AI agents) • AI Automation / AI Agent with MCPs • AI Workflows & AI Agents • MCPs for AI Agents

feat(apache-kafka): add a piece to publish and consume messages (#16291)

Eliseu Kadesh5da73236367b
diff --git a/bun.lock b/bun.lock
index 252f58cc410..2ce6c5bf7ff 100644
--- a/bun.lock
+++ b/bun.lock
@@ -716,6 +716,18 @@
"tslib": "2.6.2",
},
},
+ "packages/pieces/community/apache-kafka": {
+ "name": "@activepieces/piece-apache-kafka",
+ "version": "0.0.1",
+ "dependencies": {
+ "@activepieces/pieces-framework": "workspace:*",
+ "kafkajs": "2.2.4",
+ },
+ "devDependencies": {
+ "tslib": "2.6.2",
+ "vitest": "3.2.6",
+ },
+ },
"packages/pieces/community/apify": {
"name": "@activepieces/piece-apify",
"version": "0.4.1",
@@ -11467,6 +11479,8 @@
"@activepieces/piece-anyhook-websocket": ["@activepieces/piece-anyhook-websocket@workspace:packages/pieces/community/anyhook-websocket"],
+ "@activepieces/piece-apache-kafka": ["@activepieces/piece-apache-kafka@workspace:packages/pieces/community/apache-kafka"],
+
"@activepieces/piece-apify": ["@activepieces/piece-apify@workspace:packages/pieces/community/apify"],
"@activepieces/piece-apitable": ["@activepieces/piece-apitable@workspace:packages/pieces/community/apitable"],
@@ -16567,6 +16581,8 @@
"jwt-decode": ["[email protected]", "", {}, "sha512-+KJGIyHgkGuIq3IEBNftfhW/LfWhXUIY6OmyVWjliu5KH1y0fw7VQ8YndE2O4qZdMSd9SqbnC8GOcZEy0Om7sA=="],
+ "kafkajs": ["[email protected]", "", {}, "sha512-j/YeapB1vfPT2iOIUn/vxdyKEuhuY2PxMBvf5JWux6iSaukAccrMtXEY/Lb7OvavDhOWME589bpLrEdnVHjfjA=="],
+
"katex": ["[email protected]", "", { "dependencies": { "commander": "^2.19.0" }, "bin": { "katex": "cli.js" } }, "sha512-y+8btoc/CK70XqcHqjxiGWBOeIL8upbS0peTPXTvgrh21n1RiWWcIpSWM+4uXq+IAgNh9YYQWdc7LVDPDAEEAg=="],
"keyv": ["[email protected]", "", { "dependencies": { "json-buffer": "3.0.1" } }, "sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw=="],
diff --git a/packages/pieces/community/apache-kafka/.eslintrc.json b/packages/pieces/community/apache-kafka/.eslintrc.json
new file mode 100644
index 00000000000..01f1c80f85a
--- /dev/null
+++ b/packages/pieces/community/apache-kafka/.eslintrc.json
@@ -0,0 +1,47 @@
+{
+ "extends": [
+ "../../../../.eslintrc.base.json"
+ ],
+ "ignorePatterns": [
+ "!**/*"
+ ],
+ "overrides": [
+ {
+ "files": [
+ "*.ts",
+ "*.tsx",
+ "*.js",
+ "*.jsx"
+ ],
+ "rules": {}
+ },
+ {
+ "files": [
+ "*.ts",
+ "*.tsx"
+ ],
+ "rules": {
+ "no-restricted-imports": [
+ "error",
+ {
+ "patterns": [
+ "lodash",
+ "lodash/*",
+ "@activepieces/core-*",
+ "@activepieces/server*",
+ "@activepieces/engine",
+ "@activepieces/shared"
+ ]
+ }
+ ]
+ }
+ },
+ {
+ "files": [
+ "*.js",
+ "*.jsx"
+ ],
+ "rules": {}
+ }
+ ]
+}
diff --git a/packages/pieces/community/apache-kafka/package.json b/packages/pieces/community/apache-kafka/package.json
new file mode 100644
index 00000000000..d7e9e028f9f
--- /dev/null
+++ b/packages/pieces/community/apache-kafka/package.json
@@ -0,0 +1,21 @@
+{
+ "name": "@activepieces/piece-apache-kafka",
+ "version": "0.0.1",
+ "type": "commonjs",
+ "main": "./dist/src/index.js",
+ "types": "./dist/src/index.d.ts",
+ "dependencies": {
+ "@activepieces/pieces-framework": "workspace:*",
+ "kafkajs": "2.2.4"
+ },
+ "devDependencies": {
+ "tslib": "2.6.2",
+ "vitest": "3.2.6"
+ },
+ "scripts": {
+ "build": "tsc -p tsconfig.lib.json && cp package.json dist/",
+ "bundle": "node ../../../../dist/packages/cli/src/index.js pieces bundle",
+ "lint": "eslint 'src/**/*.ts'",
+ "test": "vitest run"
+ }
+}
diff --git a/packages/pieces/community/apache-kafka/src/index.ts b/packages/pieces/community/apache-kafka/src/index.ts
new file mode 100644
index 00000000000..3e017ec4f25
--- /dev/null
+++ b/packages/pieces/community/apache-kafka/src/index.ts
@@ -0,0 +1,18 @@
+import { createPiece, PieceCategory } from '@activepieces/pieces-framework'
+import { publishMessage } from './lib/actions/publish-message'
+import { publishMessages } from './lib/actions/publish-messages'
+import { kafkaAuth } from './lib/auth'
+import { newMessage } from './lib/triggers/new-message'
+import { newMessagesBatch } from './lib/triggers/new-messages-batch'
+
+export const apacheKafka = createPiece({
+ displayName: 'Apache Kafka',
+ description: 'Publish and consume messages on an Apache Kafka cluster.',
+ minimumSupportedRelease: '0.36.1',
+ logoUrl: 'https://cdn.activepieces.com/pieces/apache-kafka.png',
+ categories: [PieceCategory.DEVELOPER_TOOLS],
+ auth: kafkaAuth,
+ authors: ['eliseukadesh67'],
+ actions: [publishMessage, publishMessages],
+ triggers: [newMessage, newMessagesBatch],
+})
diff --git a/packages/pieces/community/apache-kafka/src/lib/actions/publish-message.ts b/packages/pieces/community/apache-kafka/src/lib/actions/publish-message.ts
new file mode 100644
index 00000000000..e174c179857
--- /dev/null
+++ b/packages/pieces/community/apache-kafka/src/lib/actions/publish-message.ts
@@ -0,0 +1,59 @@
+import { createAction, Property } from '@activepieces/pieces-framework'
+import { kafkaAuth } from '../auth'
+import { kafkaClient } from '../common/client'
+
+export const publishMessage = createAction({
+ auth: kafkaAuth,
+ name: 'publish_message',
+ displayName: 'Publish message',
+ description: 'Publishes one message to a Kafka topic and returns its partition and offset.',
+ audience: 'both',
+ classification: 'WRITE',
+ aiMetadata: {
+ description: 'Publish one message to an Apache Kafka topic. Use this for a single record; use Publish messages when several records should go out in the same step. The call waits for all in-sync replicas and returns the partition and offset. A retry publishes another message.',
+ idempotent: false,
+ },
+ props: {
+ topic: Property.ShortText({
+ displayName: 'Topic',
+ description: 'Topic to publish to, for example orders.',
+ required: true,
+ }),
+ key: Property.ShortText({
+ displayName: 'Key',
+ description: 'Optional message key. Messages with the same key stay in order on one partition.',
+ required: false,
+ }),
+ value: Property.LongText({
+ displayName: 'Value',
+ description: 'Message body sent as UTF-8 text. For JSON, paste the JSON document.',
+ required: true,
+ }),
+ headers: Property.Object({
+ displayName: 'Headers',
+ description: 'Optional text headers, for example source = checkout.',
+ required: false,
+ }),
+ partition: Property.Number({
+ displayName: 'Partition',
+ description: 'Optional partition number, starting at 0. Leave empty to let Kafka choose from the key.',
+ required: false,
+ }),
+ },
+ async run(context) {
+ const [result] = await kafkaClient.publishMessages({
+ auth: context.auth.props,
+ topic: context.propsValue.topic,
+ messages: [{
+ key: context.propsValue.key,
+ value: context.propsValue.value,
+ headers: context.propsValue.headers,
+ partition: context.propsValue.partition,
+ }],
+ })
+ if (result === undefined) {
+ throw new Error('Kafka did not return an offset for the published message.')
+ }
+ return result
+ },
+})
diff --git a/packages/pieces/community/apache-kafka/src/lib/actions/publish-messages.ts b/packages/pieces/community/apache-kafka/src/lib/actions/publish-messages.ts
new file mode 100644
index 00000000000..ef2ccb0d896
--- /dev/null
+++ b/packages/pieces/community/apache-kafka/src/lib/actions/publish-messages.ts
@@ -0,0 +1,52 @@
+import { createAction, Property } from '@activepieces/pieces-framework'
+import { kafkaAuth } from '../auth'
+import { kafkaClient } from '../common/client'
+
+export const publishMessages = createAction({
+ auth: kafkaAuth,
+ name: 'publish_messages',
+ displayName: 'Publish messages',
+ description: 'Publishes several messages to a Kafka topic in one step and returns a partition and offset for each.',
+ audience: 'both',
+ classification: 'WRITE',
+ aiMetadata: {
+ description: 'Publish multiple messages to one Apache Kafka topic in a single step and return each message partition and offset. Use Publish message when there is only one record. Messages are sent in order on one connection, waiting for all in-sync replicas. A retry publishes the batch again.',
+ idempotent: false,
+ },
+ props: {
+ topic: Property.ShortText({
+ displayName: 'Topic',
+ description: 'Topic to publish to, for example orders.',
+ required: true,
+ }),
+ messages: Property.Array({
+ displayName: 'Messages',
+ description: 'Messages published in order. Each result includes that message key, partition, and offset.',
+ required: true,
+ properties: {
+ key: Property.ShortText({
+ displayName: 'Key',
+ description: 'Optional message key. Messages with the same key stay in order on one partition.',
+ required: false,
+ }),
+ value: Property.LongText({
+ displayName: 'Value',
+ description: 'Message body sent as UTF-8 text. For JSON, paste the JSON document.',
+ required: true,
+ }),
+ partition: Property.Number({
+ displayName: 'Partition',
+ description: 'Optional partition number, starting at 0. Leave empty to let Kafka choose from the key.',
+ required: false,
+ }),
+ },
+ }),
+ },
+ async run(context) {
+ return kafkaClient.publishMessages({
+ auth: context.auth.props,
+ topic: context.propsValue.topic,
+ messages: context.propsValue.messages,
+ })
+ },
+})
diff --git a/packages/pieces/community/apache-kafka/src/lib/auth.ts b/packages/pieces/community/apache-kafka/src/lib/auth.ts
new file mode 100644
index 00000000000..b9048d49026
--- /dev/null
+++ b/packages/pieces/community/apache-kafka/src/lib/auth.ts
@@ -0,0 +1,69 @@
+import { PieceAuth, Property } from '@activepieces/pieces-framework'
+import { kafkaClient } from './common/client'
+
+export const kafkaAuth = PieceAuth.CustomAuth({
+ description: 'Connect with bootstrap servers and SASL or SSL credentials.',
+ required: true,
+ props: {
+ setup: Property.MarkDown({
+ value: `1. Copy the cluster bootstrap servers as host:port pairs, separated by commas. Example: kafka-1.internal:9092, kafka-2.internal:9092.
+2. Choose the security protocol the cluster uses.
+3. For SASL_PLAINTEXT or SASL_SSL, choose the mechanism and enter the username and password.
+4. For SSL or SASL_SSL, leave server certificate verification on unless the cluster uses a private CA. Paste that CA in the certificate field.`,
+ }),
+ brokers: Property.ShortText({
+ displayName: 'Bootstrap servers',
+ description: 'Comma-separated host:port pairs, for example kafka-1.internal:9092, kafka-2.internal:9092.',
+ required: true,
+ }),
+ securityProtocol: Property.StaticDropdown({
+ displayName: 'Security protocol',
+ description: 'How the worker connects to the brokers.',
+ required: true,
+ defaultValue: 'SASL_SSL',
+ options: {
+ options: [
+ { label: 'SASL_SSL', value: 'SASL_SSL' },
+ { label: 'SASL_PLAINTEXT', value: 'SASL_PLAINTEXT' },
+ { label: 'SSL', value: 'SSL' },
+ { label: 'PLAINTEXT', value: 'PLAINTEXT' },
+ ],
+ },
+ }),
+ saslMechanism: Property.StaticDropdown({
+ displayName: 'SASL mechanism',
+ description: 'Required for SASL_SSL and SASL_PLAINTEXT. Ignored for SSL and PLAINTEXT.',
+ required: false,
+ options: {
+ options: [
+ { label: 'PLAIN', value: 'plain' },
+ { label: 'SCRAM-SHA-256', value: 'scram-sha-256' },
+ { label: 'SCRAM-SHA-512', value: 'scram-sha-512' },
+ ],
+ },
+ }),
+ username: Property.ShortText({
+ displayName: 'Username',
+ description: 'SASL username. Required for SASL_SSL and SASL_PLAINTEXT.',
+ required: false,
+ }),
+ password: PieceAuth.SecretText({
+ displayName: 'Password',
+ description: 'SASL password. Required for SASL_SSL and SASL_PLAINTEXT.',
+ required: false,
+ }),
+ rejectUnauthorized: Property.Checkbox({
+ displayName: 'Verify server certificate',
+ description: 'Applies to SSL and SASL_SSL. Keep this on. For a private or self-signed certificate, paste its CA certificate instead. Turning this off disables certificate checks and is unsafe outside testing.',
+ required: true,
+ defaultValue: true,
+ }),
+ caCertificate: Property.LongText({
+ displayName: 'CA certificate',
+ description: 'Optional PEM CA certificate for a private certificate authority. Used with SSL and SASL_SSL.',
+ required: false,
+ }),
+ },
+ validate: async ({ auth }) => kafkaClient.validate(auth),
+ getConnectionIdentifier: async ({ auth }) => auth.brokers,
+})
diff --git a/packages/pieces/community/apache-kafka/src/lib/common/__tests__/client.test.ts b/packages/pieces/community/apache-kafka/src/lib/common/__tests__/client.test.ts
new file mode 100644
index 00000000000..4ae05a9f27f
--- /dev/null
+++ b/packages/pieces/community/apache-kafka/src/lib/common/__tests__/client.test.ts
@@ -0,0 +1,341 @@
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+
+const kafkaMock = vi.hoisted(() => {
+ const adminConnect = vi.fn(async () => undefined)
+ const adminDisconnect = vi.fn(async () => undefined)
+ const describeCluster = vi.fn(async () => ({ brokers: [], controller: null, clusterId: 'cluster' }))
+ const fetchTopicOffsets = vi.fn(async () => [
+ { partition: 0, offset: '20', low: '3', high: '20' },
+ { partition: 1, offset: '9', low: '0', high: '9' },
+ ])
+ const fetchOffsets = vi.fn(async () => [{
+ topic: 'orders',
+ partitions: [
+ { partition: 0, offset: '-1', metadata: null },
+ { partition: 1, offset: '8', metadata: null },
+ ],
+ }])
+ const setOffsets = vi.fn(async () => undefined)
+ const producerConnect = vi.fn(async () => undefined)
+ const producerDisconnect = vi.fn(async () => undefined)
+ const send = vi.fn(async () => [{
+ topicName: 'orders',
+ partition: 3,
+ errorCode: 0,
+ baseOffset: '15',
+ }])
+ const consumerConnect = vi.fn(async () => undefined)
+ const consumerDisconnect = vi.fn(async () => undefined)
+ const subscribe = vi.fn(async () => undefined)
+ const commitOffsets = vi.fn(async () => undefined)
+ const stop = vi.fn(async () => undefined)
+ const run = vi.fn(async (config: {
+ eachMessage: (payload: {
+ topic: string
+ partition: number
+ message: {
+ key: Buffer | null
+ value: Buffer | null
+ offset: string
+ timestamp: string
+ headers?: Record<string, Buffer>
+ }
+ }) => Promise<void>
+ }) => {
+ await config.eachMessage({
+ topic: 'orders',
+ partition: 1,
+ message: {
+ key: Buffer.from('order-1'),
+ value: Buffer.from('hello'),
+ offset: '8',
+ timestamp: '1710000000000',
+ headers: { source: Buffer.from('checkout') },
+ },
+ })
+ })
+ return {
+ adminConnect,
+ adminDisconnect,
+ describeCluster,
+ fetchTopicOffsets,
+ fetchOffsets,
+ setOffsets,
+ producerConnect,
+ producerDisconnect,
+ send,
+ consumerConnect,
+ consumerDisconnect,
+ subscribe,
+ commitOffsets,
+ stop,
+ run,
+ }
+})
+
+vi.mock('kafkajs', () => ({
+ logLevel: { NOTHING: 0 },
+ Partitioners: {
+ DefaultPartitioner: () => () => 0,
+ },
+ Kafka: class Kafka {
+ admin() {
+ return {
+ connect: kafkaMock.adminConnect,
+ disconnect: kafkaMock.adminDisconnect,
+ describeCluster: kafkaMock.describeCluster,
+ fetchTopicOffsets: kafkaMock.fetchTopicOffsets,
+ fetchOffsets: kafkaMock.fetchOffsets,
+ setOffsets: kafkaMock.setOffsets,
+ }
+ }
+
+ producer() {
+ return {
+ connect: kafkaMock.producerConnect,
+ disconnect: kafkaMock.producerDisconnect,
+ send: kafkaMock.send,
+ }
+ }
+
+ consumer() {
+ return {
+ connect: kafkaMock.consumerConnect,
+ disconnect: kafkaMock.consumerDisconnect,
+ subscribe: kafkaMock.subscribe,
+ commitOffsets: kafkaMock.commitOffsets,
+ stop: kafkaMock.stop,
+ run: kafkaMock.run,
+ }
+ }
+ },
+}))
+
+import { kafkaClient } from '../client'
+
+const auth = {
+ brokers: 'kafka.internal:9092',
+ securityProtocol: 'PLAINTEXT',
+}
+
+describe('kafkaClient', () => {
+ beforeEach(() => {
+ vi.clearAllMocks()
+ })
+
+ it('rejects a connection when the cluster cannot be described', async () => {
+ kafkaMock.describeCluster.mockRejectedValueOnce(new Error('SASL authentication failed'))
+ const result = await kafkaClient.validate({
+ ...auth,
+ securityProtocol: 'SASL_SSL',
+ saslMechanism: 'plain',
+ username: 'app',
+ password: 'secret',
+ })
+ expect(result).toEqual({
+ valid: false,
+ error: expect.stringContaining('Kafka authentication failed'),
+ })
+ expect(kafkaMock.adminDisconnect).toHaveBeenCalled()
+ })
+
+ it('does not open a socket when the broker list is invalid', async () => {
+ const result = await kafkaClient.validate({
+ brokers: 'not-a-broker',
+ securityProtocol: 'PLAINTEXT',
+ })
+ expect(result.valid).toBe(false)
+ expect(kafkaMock.adminConnect).not.toHaveBeenCalled()
+ })
+
+ it('returns the partition and offset for each published message', async () => {
+ const results = await kafkaClient.publishMessages({
+ auth,
+ topic: 'orders',
+ messages: [
+ { key: 'order-1', value: 'hello' },
+ { value: 'world' },
+ ],
+ })
+ expect(results).toEqual([
+ { topic: 'orders', key: 'order-1', partition: 3, offset: '15' },
+ { topic: 'orders', key: null, partition: 3, offset: '15' },
+ ])
+ expect(kafkaMock.send).toHaveBeenCalledTimes(2)
+ expect(kafkaMock.producerDisconnect).toHaveBeenCalled()
+ })
+
+ it('commits the next offset only when the trigger run asks it to', async () => {
+ const committed = await kafkaClient.consume({
+ auth,
+ topic: 'orders',
+ consumerGroup: 'activepieces-orders',
+ maxMessages: 1,
+ pollTimeoutSeconds: 1,
+ fromBeginning: false,
+ commit: true,
+ })
+ expect(committed).toEqual([{
+ topic: 'orders',
+ partition: 1,
+ offset: '8',
+ key: 'order-1',
+ payload: 'hello',
+ headers: { source: 'checkout' },
+ timestamp: '1710000000000',
+ }])
+ expect(kafkaMock.commitOffsets).toHaveBeenCalledWith([
+ { topic: 'orders', partition: 1, offset: '9' },
+ ])
+
+ kafkaMock.commitOffsets.mockClear()
+ kafkaMock.setOffsets.mockClear()
+ await kafkaClient.consume({
+ auth,
+ topic: 'orders',
+ consumerGroup: 'activepieces-orders',
+ maxMessages: 1,
+ pollTimeoutSeconds: 1,
+ fromBeginning: false,
+ commit: false,
+ })
+ expect(kafkaMock.commitOffsets).not.toHaveBeenCalled()
+ expect(kafkaMock.setOffsets).not.toHaveBeenCalled()
+ })
+
+ it('pins the start offset of every partition without a commit before a trigger run reads', async () => {
+ await kafkaClient.consume({
+ auth,
+ topic: 'orders',
+ consumerGroup: 'activepieces-orders',
+ maxMessages: 1,
+ pollTimeoutSeconds: 1,
+ fromBeginning: false,
+ commit: true,
+ })
+ expect(kafkaMock.setOffsets).toHaveBeenCalledWith({
+ groupId: 'activepieces-orders',
+ topic: 'orders',
+ partitions: [{ partition: 0, offset: '20' }],
+ })
+ expect(kafkaMock.setOffsets.mock.invocationCallOrder[0]).toBeLessThan(kafkaMock.consumerConnect.mock.invocationCallOrder[0])
+ })
+
+ it('leaves the group untouched when every partition already has a commit', async () => {
+ kafkaMock.fetchOffsets.mockResolvedValueOnce([{
+ topic: 'orders',
+ partitions: [
+ { partition: 0, offset: '20', metadata: null },
+ { partition: 1, offset: '9', metadata: null },
+ ],
+ }])
+ await kafkaClient.pinStartOffsets({
+ auth,
+ topic: 'orders',
+ consumerGroup: 'activepieces-orders',
+ fromBeginning: true,
+ })
+ expect(kafkaMock.setOffsets).not.toHaveBeenCalled()
+ expect(kafkaMock.adminDisconnect).toHaveBeenCalled()
+ })
+
+ it('returns only the committed batch when a message arrives while committing', async () => {
+ const late = {
+ topic: 'orders',
+ partition: 1,
+ message: {
+ key: null,
+ value: Buffer.from('late'),
+ offset: '9',
+ timestamp: '1710000000001',
+ },
+ }
+ let deliver: ((payload: typeof late) => Promise<void>) | undefined
+ kafkaMock.run.mockImplementationOnce(async (config) => {
+ deliver = config.eachMessage
+ await config.eachMessage({
+ topic: 'orders',
+ partition: 1,
+ message: {
+ key: Buffer.from('order-1'),
+ value: Buffer.from('hello'),
+ offset: '8',
+ timestamp: '1710000000000',
+ },
+ })
+ })
+ kafkaMock.commitOffsets.mockImplementationOnce(async () => {
+ await deliver?.(late)
+ })
+
+ const records = await kafkaClient.consume({
+ auth,
+ topic: 'orders',
+ consumerGroup: 'activepieces-orders',
+ maxMessages: 10,
+ pollTimeoutSeconds: 5,
+ fromBeginning: false,
+ commit: true,
+ })
+
+ expect(records.map((record) => record.offset)).toEqual(['8'])
+ expect(kafkaMock.commitOffsets).toHaveBeenCalledWith([
+ { topic: 'orders', partition: 1, offset: '9' },
+ ])
+ })
+
+ it('waits for the consumer to join the group before timing the poll', async () => {
+ kafkaMock.run.mockImplementationOnce(async (config) => {
+ await new Promise((resolve) => setTimeout(resolve, 1_500))
+ setTimeout(() => {
+ config.eachMessage({
+ topic: 'orders',
+ partition: 0,
+ message: { key: null, value: Buffer.from('after join'), offset: '3', timestamp: '1710000000000' },
+ }).catch(() => undefined)
+ }, 50)
+ })
+
+ const records = await kafkaClient.consume({
+ auth,
+ topic: 'orders',
+ consumerGroup: 'activepieces-orders',
+ maxMessages: 10,
+ pollTimeoutSeconds: 1,
+ fromBeginning: false,
+ commit: false,
+ })
+
+ expect(records.map((record) => record.payload)).toEqual(['after join'])
+ })
+
+ it('reports a consumer that fails to start', async () => {
+ kafkaMock.run.mockRejectedValueOnce(new Error('Group coordinator not available'))
+
+ await expect(kafkaClient.consume({
+ auth,
+ topic: 'orders',
+ consumerGroup: 'activepieces-orders',
+ maxMessages: 10,
+ pollTimeoutSeconds: 1,
+ fromBeginning: false,
+ commit: false,
+ })).rejects.toThrow()
+ expect(kafkaMock.consumerDisconnect).toHaveBeenCalled()
+ })
+
+ it('returns a partial batch once messages stop arriving', async () => {
+ const started = Date.now()
+ const records = await kafkaClient.consume({
+ auth,
+ topic: 'orders',
+ consumerGroup: 'activepieces-orders',
+ maxMessages: 10,
+ pollTimeoutSeconds: 20,
+ fromBeginning: false,
+ commit: false,
+ })
+ expect(records).toHaveLength(1)
+ expect(Date.now() - started).toBeLessThan(2_000)
+ })
+})
diff --git a/packages/pieces/community/apache-kafka/src/lib/common/__tests__/config.test.ts b/packages/pieces/community/apache-kafka/src/lib/common/__tests__/config.test.ts
new file mode 100644
index 00000000000..634704f0daa
--- /dev/null
+++ b/packages/pieces/community/apache-kafka/src/lib/common/__tests__/config.test.ts
@@ -0,0 +1,72 @@
+import { describe, expect, it } from 'vitest'
+import { kafkaConfig } from '../config'
+
+const saslAuth = {
+ brokers: 'kafka.internal:9092',
+ securityProtocol: 'SASL_SSL',
+ saslMechanism: 'plain',
+ username: 'app',
+ password: 'secret',
+ rejectUnauthorized: true,
+}
+
+describe('kafkaConfig', () => {
+ it('parses comma-separated brokers and rejects a missing port', () => {
+ expect(kafkaConfig.parseBrokers(' kafka-1.internal:9092, kafka-2.internal:9093 ')).toEqual([
+ 'kafka-1.internal:9092',
+ 'kafka-2.internal:9093',
+ ])
+ expect(() => kafkaConfig.parseBrokers('kafka.internal')).toThrow('host:port')
+ })
+
+ it('requires SASL credentials only for SASL protocols', () => {
+ expect(kafkaConfig.requirementError({
+ brokers: 'kafka.internal:9092',
+ securityProtocol: 'PLAINTEXT',
+ })).toBeUndefined()
+ expect(kafkaConfig.requirementError({
+ ...saslAuth,
+ username: ' ',
+ })).toMatch('SASL username and password')
+ expect(kafkaConfig.requirementError({
+ ...saslAuth,
+ saslMechanism: undefined,
+ })).toMatch('SASL mechanism')
+ })
+
+ it('builds SSL and SASL options from the security protocol', () => {
+ const plaintext = kafkaConfig.build({
+ brokers: 'kafka.internal:9092',
+ securityProtocol: 'PLAINTEXT',
+ })
+ expect(plaintext.brokers).toEqual(['kafka.internal:9092'])
+ expect(plaintext.ssl).toBeUndefined()
+ expect(plaintext.sasl).toBeUndefined()
+
+ const secured = kafkaConfig.build({
+ ...saslAuth,
+ caCertificate: '-----BEGIN CERTIFICATE-----\nMIIB\n-----END CERTIFICATE-----',
+ })
+ expect(secured.sasl).toEqual({ mechanism: 'plain', username: 'app', password: 'secret' })
+ expect(secured.ssl).toMatchObject({
+ rejectUnauthorized: true,
+ ca: ['-----BEGIN CERTIFICATE-----\nMIIB\n-----END CERTIFICATE-----'],
+ })
+ })
+
+ it('turns broker failures into an actionable message', () => {
+ expect(kafkaConfig.describeError({
+ error: new Error('Connection timeout'),
+ brokers: 'kafka.internal:9092',
+ topic: 'orders',
+ })).toMatch('Could not reach Kafka topic "orders"')
+ expect(kafkaConfig.describeError({
+ error: new Error('SASL authentication failed'),
+ brokers: 'kafka.internal:9092',
+ })).toMatch('Kafka authentication failed')
+ expect(kafkaConfig.describeError({
+ error: new Error('Enter a topic name, for example orders.'),
+ brokers: 'kafka.internal:9092',
+ })).toBe('Enter a topic name, for example orders.')
+ })
+})
diff --git a/packages/pieces/community/apache-kafka/src/lib/common/__tests__/records.test.ts b/packages/pieces/community/apache-kafka/src/lib/common/__tests__/records.test.ts
new file mode 100644
index 00000000000..9822ed9109c
--- /dev/null
+++ b/packages/pieces/community/apache-kafka/src/lib/common/__tests__/records.test.ts
@@ -0,0 +1,95 @@
+import { describe, expect, it } from 'vitest'
+import { kafkaRecords } from '../records'
+
+describe('kafkaRecords', () => {
+ it('flattens a consumed message into payload, key, partition, and offset', () => {
+ const record = kafkaRecords.toRecord({
+ topic: 'orders',
+ partition: 2,
+ message: {
+ key: Buffer.from('order-1'),
+ value: Buffer.from('{"id":"order-1"}'),
+ offset: '41',
+ timestamp: '1710000000000',
+ headers: {
+ source: Buffer.from('checkout'),
+ trace: ['a', Buffer.from('b')],
+ },
+ },
+ })
+
+ expect(record).toEqual({
+ topic: 'orders',
+ partition: 2,
+ offset: '41',
+ key: 'order-1',
+ payload: '{"id":"order-1"}',
+ headers: { source: 'checkout', trace: 'a,b' },
+ timestamp: '1710000000000',
+ })
+ })
+
+ it('commits the next offset on each partition', () => {
+ expect(kafkaRecords.commitPlan({
+ topic: 'orders',
+ records: [
+ { partition: 0, offset: '4' },
+ { partition: 0, offset: '5' },
+ { partition: 1, offset: '9' },
+ ],
+ })).toEqual([
+ { topic: 'orders', partition: 0, offset: '6' },
+ { topic: 'orders', partition: 1, offset: '10' },
+ ])
+ })
+
+ it('starts each partition without a commit at the chosen end of the topic', () => {
+ const topicOffsets = [
+ { partition: 0, low: '3', high: '20' },
+ { partition: 1, low: '0', high: '9' },
+ { partition: 2, low: '5', high: '5' },
+ ]
+ const committed = [
+ { partition: 0, offset: '-1' },
+ { partition: 1, offset: '7' },
+ ]
+ expect(kafkaRecords.startOffsetPlan({ topicOffsets, committed, fromBeginning: false })).toEqual([
+ { partition: 0, offset: '20' },
+ { partition: 2, offset: '5' },
+ ])
+ expect(kafkaRecords.startOffsetPlan({ topicOffsets, committed, fromBeginning: true })).toEqual([
+ { partition: 0, offset: '3' },
+ { partition: 2, offset: '5' },
+ ])
+ })
+
+ it('returns one batch item or one item per message', () => {
+ const records = [{
+ topic: 'orders',
+ partition: 0,
+ offset: '1',
+ key: null,
+ payload: 'hello',
+ headers: {},
+ timestamp: '1',
+ }]
+ expect(kafkaRecords.toTriggerOutput({ records, topic: 'orders', batch: false })).toEqual(records)
+ expect(kafkaRecords.toTriggerOutput({ records, topic: 'orders', batch: true })).toEqual([{
+ topic: 'orders',
+ message_count: 1,
+ messages: records,
+ }])
+ expect(kafkaRecords.toTriggerOutput({ records: [], topic: 'orders', batch: true })).toEqual([])
+ })
+
+ it('reads outbound messages and rejects a missing value', () => {
+ expect(kafkaRecords.readOutboundMessages([
+ { key: 'order-1', value: 'hello', headers: { source: 'checkout' }, partition: 1 },
+ ])).toEqual([
+ { key: 'order-1', value: 'hello', headers: { source: 'checkout' }, partition: 1 },
+ ])
+ expect(() => kafkaRecords.readOutboundMessages([])).toThrow('Add at least one message')
+ expect(() => kafkaRecords.readOutboundMessage({ value: '' }, 0)).toThrow('Message 1 needs a value')
+ expect(() => kafkaRecords.readOutboundMessage({ value: 'hello', partition: -1 }, 0)).toThrow('Partition must be a whole number')
+ })
+})
diff --git a/packages/pieces/community/apache-kafka/src/lib/common/client.ts b/packages/pieces/community/apache-kafka/src/lib/common/client.ts
new file mode 100644
index 00000000000..051c0838dcb
--- /dev/null
+++ b/packages/pieces/community/apache-kafka/src/lib/common/client.ts
@@ -0,0 +1,179 @@
+import { Kafka, Partitioners, type EachMessagePayload } from 'kafkajs'
+import { kafkaConfig, type KafkaAuthInput } from './config'
+import { kafkaRecords, type KafkaRecord } from './records'
+
+async function validate(auth: KafkaAuthInput): Promise<{ valid: true } | { valid: false, error: string }> {
+ const requirement = kafkaConfig.requirementError(auth)
+ if (requirement !== undefined) {
+ return { valid: false, error: requirement }
+ }
+ const admin = new Kafka(kafkaConfig.build(auth)).admin()
+ try {
+ await admin.connect()
+ await admin.describeCluster()
+ return { valid: true }
+ } catch (error) {
+ return {
+ valid: false,
+ error: kafkaConfig.describeError({ error, brokers: auth.brokers }),
+ }
+ } finally {
+ await admin.disconnect().catch(() => undefined)
+ }
+}
+
+async function publishMessages({ auth, topic, messages }: {
+ auth: KafkaAuthInput
+ topic: string
+ messages: unknown
+}): Promise<PublishResult[]> {
+ const topicName = kafkaConfig.readTopic(topic)
+ const outbound = kafkaRecords.readOutboundMessages(messages)
+ const producer = new Kafka(kafkaConfig.build(auth)).producer({
+ createPartitioner: Partitioners.DefaultPartitioner,
+ })
+ try {
+ await producer.connect()
+ const results: PublishResult[] = []
+ for (const message of outbound) {
+ const metadata = await producer.send({
+ topic: topicName,
+ acks: -1,
+ messages: [kafkaRecords.toKafkaMessage(message)],
+ })
+ const record = metadata[0]
+ if (record === undefined || record.errorCode !== 0) {
+ throw new Error(`Broker returned error code ${record?.errorCode ?? 'unknown'}.`)
+ }
+ const offset = record.baseOffset ?? record.offset
+ if (offset === undefined || offset.length === 0) {
+ throw new Error('Kafka did not return an offset for the published message.')
+ }
+ results.push({
+ topic: topicName,
+ key: message.key ?? null,
+ partition: record.partition,
+ offset,
+ })
+ }
+ return results
+ } catch (error) {
+ throw new Error(kafkaConfig.describeError({ error, brokers: auth.brokers, topic: topicName }))
+ } finally {
+ await producer.disconnect().catch(() => undefined)
+ }
+}
+
+async function pinStartOffsets({ auth, topic, consumerGroup, fromBeginning }: {
+ auth: KafkaAuthInput
+ topic: string
+ consumerGroup: string
+ fromBeginning: boolean
+}): Promise<void> {
+ const topicName = kafkaConfig.readTopic(topic)
+ const groupId = kafkaConfig.readConsumerGroup(consumerGroup)
+ const admin = new Kafka(kafkaConfig.build(auth)).admin()
+ try {
+ await admin.connect()
+ const [topicOffsets, groupOffsets] = await Promise.all([
+ admin.fetchTopicOffsets(topicName),
+ admin.fetchOffsets({ groupId, topics: [topicName] }),
+ ])
+ const partitions = kafkaRecords.startOffsetPlan({
+ topicOffsets,
+ committed: groupOffsets.find((entry) => entry.topic === topicName)?.partitions ?? [],
+ fromBeginning,
+ })
+ if (partitions.length > 0) {
+ await admin.setOffsets({ groupId, topic: topicName, partitions })
+ }
+ } catch (error) {
+ throw new Error(kafkaConfig.describeError({ error, brokers: auth.brokers, topic: topicName }))
+ } finally {
+ await admin.disconnect().catch(() => undefined)
+ }
+}
+
+async function consume({ auth, topic, consumerGroup, maxMessages, pollTimeoutSeconds, fromBeginning, commit }: {
+ auth: KafkaAuthInput
+ topic: string
+ consumerGroup: string
+ maxMessages: number
+ pollTimeoutSeconds: number
+ fromBeginning: boolean
+ commit: boolean
+}): Promise<KafkaRecord[]> {
+ const topicName = kafkaConfig.readTopic(topic)
+ const groupId = kafkaConfig.readConsumerGroup(consumerGroup)
+ const limit = kafkaConfig.readMaxMessages(maxMessages)
+ const pollTimeoutMs = kafkaConfig.readPollTimeoutMs(pollTimeoutSeconds)
+ if (commit) {
+ await pinStartOffsets({ auth, topic: topicName, consumerGroup: groupId, fromBeginning })
+ }
+ const consumer = new Kafka(kafkaConfig.build(auth)).consumer({
+ groupId,
+ maxWaitTimeInMs: 500,
+ sessionTimeout: 15_000,
+ rebalanceTimeout: 15_000,
+ heartbeatInterval: 3_000,
+ })
+ try {
+ await consumer.connect()
+ await consumer.subscribe({ topic: topicName, fromBeginning })
+ const records: KafkaRecord[] = []
+ let collecting = true
+ await consumer.run({
+ autoCommit: false,
+ eachMessage: async ({ topic: messageTopic, partition, message }: EachMessagePayload) => {
+ if (!collecting || records.length >= limit) {
+ return
+ }
+ records.push(kafkaRecords.toRecord({ topic: messageTopic, partition, message }))
+ },
+ })
+ const deadline = Date.now() + pollTimeoutMs
+ let lastCount = 0
+ let quietSince = Date.now()
+ while (records.length < limit && Date.now() < deadline) {
+ if (records.length > 0 && records.length === lastCount && Date.now() - quietSince >= 500) {
+ break
+ }
+ if (records.length !== lastCount) {
+ lastCount = records.length
+ quietSince = Date.now()
+ }
+ await delay(100)
+ }
+ collecting = false
+ const batch = [...records]
+ if (commit && batch.length > 0) {
+ await consumer.commitOffsets(kafkaRecords.commitPlan({ topic: topicName, records: batch }))
+ }
+ return batch
+ } catch (error) {
+ throw new Error(kafkaConfig.describeError({ error, brokers: auth.brokers, topic: topicName }))
+ } finally {
+ await consumer.stop().catch(() => undefined)
+ await consumer.disconnect().catch(() => undefined)
+ }
+}
+
+function delay(ms: number): Promise<void> {
+ return new Promise((resolve) => {
+ setTimeout(resolve, ms)
+ })
+}
+
+export const kafkaClient = {
+ validate,
+ publishMessages,
+ pinStartOffsets,
+ consume,
+}
+
+export type PublishResult = {
+ topic: string
+ key: string | null
+ partition: number
+ offset: string
+}
diff --git a/packages/pieces/community/apache-kafka/src/lib/common/config.ts b/packages/pieces/community/apache-kafka/src/lib/common/config.ts
new file mode 100644
index 00000000000..1ce9809134a
--- /dev/null
+++ b/packages/pieces/community/apache-kafka/src/lib/common/config.ts
@@ -0,0 +1,234 @@
+import { logLevel, type KafkaConfig, type SASLOptions } from 'kafkajs'
+
+function parseBrokers(brokers: string): string[] {
+ const parsed = brokers
+ .split(',')
+ .map((entry) => entry.trim())
+ .filter((entry) => entry.length > 0)
+ if (parsed.length === 0) {
+ throw new Error('Enter at least one broker as host:port, for example kafka.internal:9092.')
+ }
+ const invalid = parsed.find((entry) => !isHostPort(entry))
+ if (invalid !== undefined) {
+ throw new Error(`Broker "${invalid}" must be host:port, for example kafka.internal:9092.`)
+ }
+ return parsed
+}
+
+function requirementError(input: KafkaAuthInput): string | undefined {
+ try {
+ parseBrokers(input.brokers)
+ } catch (error) {
+ return error instanceof Error ? error.message : 'Enter at least one broker as host:port, for example kafka.internal:9092.'
+ }
+ if (!isSecurityProtocol(input.securityProtocol)) {
+ return 'Choose a security protocol: PLAINTEXT, SSL, SASL_PLAINTEXT, or SASL_SSL.'
+ }
+ if (!usesSasl(input.securityProtocol)) {
+ return undefined
+ }
+ if (!isSaslMechanism(input.saslMechanism ?? '')) {
+ return 'Choose a SASL mechanism: plain, scram-sha-256, or scram-sha-512.'
+ }
+ const username = input.username?.trim() ?? ''
+ if (username.length === 0 || (input.password ?? '').length === 0) {
+ return 'SASL username and password are required when the security protocol is SASL_SSL or SASL_PLAINTEXT.'
+ }
+ return undefined
+}
+
+function build(input: KafkaAuthInput): KafkaConfig {
+ const requirement = requirementError(input)
+ if (requirement !== undefined) {
+ throw new Error(requirement)
+ }
+ const securityProtocol = input.securityProtocol
+ if (!isSecurityProtocol(securityProtocol)) {
+ throw new Error('Choose a security protocol: PLAINTEXT, SSL, SASL_PLAINTEXT, or SASL_SSL.')
+ }
+ const caCertificate = input.caCertificate?.trim() ?? ''
+ return {
+ clientId: 'activepieces',
+ brokers: parseBrokers(input.brokers),
+ ssl: usesSsl(securityProtocol)
+ ? {
+ rejectUnauthorized: input.rejectUnauthorized !== false,
+ ...(caCertificate.length > 0 ? { ca: [caCertificate] } : {}),
+ }
+ : undefined,
+ sasl: usesSasl(securityProtocol) ? readSasl(input) : undefined,
+ logLevel: logLevel.NOTHING,
+ connectionTimeout: 10_000,
+ authenticationTimeout: 10_000,
+ requestTimeout: 15_000,
+ retry: {
+ retries: 2,
+ initialRetryTime: 300,
+ maxRetryTime: 3_000,
+ },
+ }
+}
+
+function describeError({ error, brokers, topic }: { error: unknown, brokers: string, topic?: string }): string {
+ const raw = errorText(error)
+ if (isPassthrough(raw)) {
+ return raw
+ }
+ const target = topic !== undefined && topic.trim().length > 0
+ ? `topic "${topic.trim()}" on ${brokers}`
+ : `brokers ${brokers}`
+ const lower = raw.toLowerCase()
+ if (lower.includes('sasl') || lower.includes('authentication') || lower.includes('invalid credentials')) {
+ return `Kafka authentication failed for ${target}. Check the SASL mechanism, username, and password. ${raw}`
+ }
+ if (lower.includes('ssl') || lower.includes('tls') || lower.includes('certificate')) {
+ return `Kafka SSL handshake failed for ${target}. Check the security protocol, the CA certificate, and whether server certificate verification is enabled. ${raw}`
+ }
+ if (
+ lower.includes('econnrefused')
+ || lower.includes('connection timeout')
+ || lower.includes('connection error')
+ || lower.includes('failed to connect')
+ || lower.includes('timed out')
+ ) {
+ return `Could not reach Kafka ${target}. Check the bootstrap servers and that this worker can open a connection to them. ${raw}`
+ }
+ if (lower.includes('unknown topic') || lower.includes('does not host this topic') || lower.includes('topic_or_partition') || lower.includes('this server does not host')) {
+ return `Kafka topic was not found for ${target}. Check the topic name and that it exists on the cluster. ${raw}`
+ }
+ return `Kafka request failed for ${target}. ${raw}`
+}
+
+function readTopic(topic: string): string {
+ const trimmed = topic.trim()
+ if (trimmed.length === 0) {
+ throw new Error('Enter a topic name, for example orders.')
+ }
+ return trimmed
+}
+
+function readConsumerGroup(consumerGroup: string): string {
+ const trimmed = consumerGroup.trim()
+ if (trimmed.length === 0) {
+ throw new Error('Enter a consumer group id. Use a different group for each flow that should receive every message.')
+ }
+ return trimmed
+}
+
+function readMaxMessages(value: number): number {
+ if (!Number.isInteger(value) || value < 1 || value > 500) {
+ throw new Error('Max messages must be a whole number from 1 to 500.')
+ }
+ return value
+}
+
+function readPollTimeoutMs(seconds: number): number {
+ if (!Number.isInteger(seconds) || seconds < 1 || seconds > 20) {
+ throw new Error('Poll timeout must be a whole number of seconds from 1 to 20.')
+ }
+ return seconds * 1000
+}
+
+function readStartFrom(value: string): boolean {
+ if (value === 'beginning') {
+ return true
+ }
+ if (value === 'latest') {
+ return false
+ }
+ throw new Error('Choose whether to start from new messages only or from the beginning of the topic.')
+}
+
+function isHostPort(value: string): boolean {
+ const separator = value.lastIndexOf(':')
+ if (separator <= 0 || separator === value.length - 1) {
+ return false
+ }
+ const port = Number(value.slice(separator + 1))
+ return Number.isInteger(port) && port > 0 && port <= 65535 && !value.slice(0, separator).includes(' ')
+}
+
+function usesSasl(protocol: SecurityProtocol): boolean {
+ return protocol === 'SASL_PLAINTEXT' || protocol === 'SASL_SSL'
+}
+
+function usesSsl(protocol: SecurityProtocol): boolean {
+ return protocol === 'SSL' || protocol === 'SASL_SSL'
+}
+
+function readSasl(input: KafkaAuthInput): SASLOptions {
+ const mechanism = input.saslMechanism ?? ''
+ const username = input.username?.trim() ?? ''
+ const password = input.password ?? ''
+ if (mechanism === 'plain' || mechanism === 'scram-sha-256' || mechanism === 'scram-sha-512') {
+ return { mechanism, username, password }
+ }
+ throw new Error('Choose a SASL mechanism: plain, scram-sha-256, or scram-sha-512.')
+}
+
+function errorText(error: unknown): string {
+ if (!(error instanceof Error)) {
+ return 'Unknown Kafka error'
+ }
+ if (error.cause instanceof Error && error.cause.message.length > 0) {
+ return `${error.message} ${error.cause.message}`
+ }
+ return error.message
+}
+
+function isPassthrough(message: string): boolean {
+ const prefixes = [
+ 'Enter at least one broker',
+ 'Broker "',
+ 'Choose a security protocol',
+ 'Choose a SASL',
+ 'SASL username',
+ 'Enter a topic',
+ 'Enter a consumer group',
+ 'Max messages',
+ 'Poll timeout',
+ 'Choose whether to start',
+ 'Message ',
+ 'Header "',
+ 'Headers must',
+ 'Partition must',
+ 'Kafka did not return an offset',
+ 'Broker returned error code',
+ 'Add at least one message',
+ ]
+ return prefixes.some((prefix) => message.startsWith(prefix))
+}
+
+function isSecurityProtocol(value: string): value is SecurityProtocol {
+ return value === 'PLAINTEXT' || value === 'SSL' || value === 'SASL_PLAINTEXT' || value === 'SASL_SSL'
+}
+
+function isSaslMechanism(value: string): value is SaslMechanism {
+ return value === 'plain' || value === 'scram-sha-256' || value === 'scram-sha-512'
+}
+
+export const kafkaConfig = {
+ parseBrokers,
+ requirementError,
+ build,
+ describeError,
+ readTopic,
+ readConsumerGroup,
+ readMaxMessages,
+ readPollTimeoutMs,
+ readStartFrom,
+}
+
+export type KafkaAuthInput = {
+ brokers: string
+ securityProtocol: string
+ saslMechanism?: string
+ username?: string
+ password?: string
+ rejectUnauthorized?: boolean
+ caCertificate?: string
+}
+
+export type SecurityProtocol = 'PLAINTEXT' | 'SSL' | 'SASL_PLAINTEXT' | 'SASL_SSL'
+
+export type SaslMechanism = 'plain' | 'scram-sha-256' | 'scram-sha-512'
diff --git a/packages/pieces/community/apache-kafka/src/lib/common/records.ts b/packages/pieces/community/apache-kafka/src/lib/common/records.ts
new file mode 100644
index 00000000000..68f2f1f50b6
--- /dev/null
+++ b/packages/pieces/community/apache-kafka/src/lib/common/records.ts
@@ -0,0 +1,205 @@
+import { type Message } from 'kafkajs'
+
+function toRecord({ topic, partition, message }: {
+ topic: string
+ partition: number
+ message: {
+ key: Buffer | null
+ value: Buffer | null
+ offset: string
+ timestamp: string
+ headers?: Record<string, Buffer | string | (Buffer | string)[] | undefined>
+ }
+}): KafkaRecord {
+ return {
+ topic,
+ partition,
+ offset: message.offset,
+ key: message.key === null ? null : message.key.toString('utf8'),
+ payload: message.value === null ? null : message.value.toString('utf8'),
+ headers: headerRecord(message.headers),
+ timestamp: message.timestamp,
+ }
+}
+
+function commitPlan({ topic, records }: {
+ topic: string
+ records: { partition: number, offset: string }[]
+}): { topic: string, partition: number, offset: string }[] {
+ const highest = new Map<number, bigint>()
+ for (const record of records) {
+ const offset = BigInt(record.offset)
+ const current = highest.get(record.partition)
+ if (current === undefined || offset > current) {
+ highest.set(record.partition, offset)
+ }
+ }
+ return [...highest.entries()].map(([partition, offset]) => ({
+ topic,
+ partition,
+ offset: (offset + BigInt(1)).toString(),
+ }))
+}
+
+function startOffsetPlan({ topicOffsets, committed, fromBeginning }: {
+ topicOffsets: { partition: number, low: string, high: string }[]
+ committed: { partition: number, offset: string }[]
+ fromBeginning: boolean
+}): { partition: number, offset: string }[] {
+ const pinned = new Set(committed.filter((entry) => BigInt(entry.offset) >= BigInt(0)).map((entry) => entry.partition))
+ return topicOffsets
+ .filter((entry) => !pinned.has(entry.partition))
+ .map((entry) => ({ partition: entry.partition, offset: fromBeginning ? entry.low : entry.high }))
+}
+
+function readOutboundMessages(value: unknown): OutboundMessage[] {
+ if (!Array.isArray(value) || value.length === 0) {
+ throw new Error('Add at least one message. Each message needs a value.')
+ }
+ return value.map((item, index) => readOutboundMessage(item, index))
+}
+
+function readOutboundMessage(value: unknown, index: number): OutboundMessage {
+ if (typeof value !== 'object' || value === null || Array.isArray(value)) {
+ throw new Error(`${messageLabel(index)} must include a value.`)
+ }
+ const rawValue = readField(value, 'value')
+ if (typeof rawValue !== 'string' || rawValue.length === 0) {
+ throw new Error(`${messageLabel(index)} needs a value.`)
+ }
+ return {
+ value: rawValue,
+ key: readOptionalString(readField(value, 'key'), `${messageLabel(index)} key`),
+ partition: readOptionalPartition(readField(value, 'partition'), index),
+ headers: readHeadersInput(readField(value, 'headers'), index),
+ }
+}
+
+function toKafkaMessage(message: OutboundMessage): Message {
+ return {
+ key: message.key,
+ value: message.value,
+ headers: message.headers,
+ partition: message.partition,
+ }
+}
+
+function toTriggerOutput({ records, topic, batch }: {
+ records: KafkaRecord[]
+ topic: string
+ batch: boolean
+}): KafkaRecord[] | KafkaBatch[] {
+ if (!batch) {
+ return records
+ }
+ if (records.length === 0) {
+ return []
+ }
+ return [{
+ topic,
+ message_count: records.length,
+ messages: records,
+ }]
+}
+
+function headerRecord(headers: Record<string, Buffer | string | (Buffer | string)[] | undefined> | undefined): Record<string, string> {
+ if (headers === undefined) {
+ return {}
+ }
+ return Object.entries(headers).reduce<Record<string, string>>((collected, [key, value]) => {
+ if (value !== undefined) {
+ collected[key] = headerValue(value)
+ }
+ return collected
+ }, {})
+}
+
+function headerValue(value: Buffer | string | (Buffer | string)[]): string {
+ if (typeof value === 'string') {
+ return value
+ }
+ if (Buffer.isBuffer(value)) {
+ return value.toString('utf8')
+ }
+ return value.map((entry) => (Buffer.isBuffer(entry) ? entry.toString('utf8') : entry)).join(',')
+}
+
+function readHeadersInput(value: unknown, index: number): Record<string, string> | undefined {
+ if (value === undefined || value === null) {
+ return undefined
+ }
+ if (typeof value !== 'object' || Array.isArray(value)) {
+ throw new Error(`Headers must be an object of text values on ${messageLabel(index)}.`)
+ }
+ return Object.entries(value).reduce<Record<string, string>>((collected, [key, entry]) => {
+ if (typeof entry !== 'string') {
+ throw new Error(`Header "${key}" must be text on ${messageLabel(index)}.`)
+ }
+ collected[key] = entry
+ return collected
+ }, {})
+}
+
+function readOptionalString(value: unknown, label: string): string | undefined {
+ if (value === undefined || value === null || value === '') {
+ return undefined
+ }
+ if (typeof value !== 'string') {
+ throw new Error(`${label} must be text.`)
+ }
+ return value
+}
+
+function readOptionalPartition(value: unknown, index: number): number | undefined {
+ if (value === undefined || value === null || value === '') {
+ return undefined
+ }
+ if (typeof value !== 'number' || !Number.isInteger(value) || value < 0) {
+ throw new Error(`Partition must be a whole number starting at 0 on ${messageLabel(index)}.`)
+ }
+ return value
+}
+
+function readField(record: object, key: string): unknown {
+ if (!(key in record)) {
+ return undefined
+ }
+ return Reflect.get(record, key)
+}
+
+function messageLabel(index: number): string {
+ return `Message ${index + 1}`
+}
+
+export const kafkaRecords = {
+ toRecord,
+ commitPlan,
+ startOffsetPlan,
+ readOutboundMessage,
+ readOutboundMessages,
+ toKafkaMessage,
+ toTriggerOutput,
+}
+
+export type KafkaRecord = {
+ topic: string
+ partition: number
+ offset: string
+ key: string | null
+ payload: string | null
+ headers: Record<string, string>
+ timestamp: string
+}
+
+export type KafkaBatch = {
+ topic: string
+ message_count: number
+ messages: KafkaRecord[]
+}
+
+export type OutboundMessage = {
+ key?: string
+ value: string
+ headers?: Record<string, string>
+ partition?: number
+}
diff --git a/packages/pieces/community/apache-kafka/src/lib/triggers/new-message.ts b/packages/pieces/community/apache-kafka/src/lib/triggers/new-message.ts
new file mode 100644
index 00000000000..52cc5f9c4f6
--- /dev/null
+++ b/packages/pieces/community/apache-kafka/src/lib/triggers/new-message.ts
@@ -0,0 +1,85 @@
+import { createTrigger, TriggerStrategy } from '@activepieces/pieces-framework'
+import { kafkaAuth } from '../auth'
+import { kafkaClient } from '../common/client'
+import { kafkaConfig } from '../common/config'
+import { kafkaRecords } from '../common/records'
+import { kafkaTriggerProps } from './props'
+
+async function readMessages({ auth, topic, consumerGroup, maxMessages, pollTimeoutSeconds, startFrom, commit, batch }: {
+ auth: Parameters<typeof kafkaClient.consume>[0]['auth']
+ topic: string
+ consumerGroup: string
+ maxMessages: number
+ pollTimeoutSeconds: number
+ startFrom: string
+ commit: boolean
+ batch: boolean
+}) {
+ const records = await kafkaClient.consume({
+ auth,
+ topic,
+ consumerGroup,
+ maxMessages,
+ pollTimeoutSeconds,
+ fromBeginning: kafkaConfig.readStartFrom(startFrom),
+ commit,
+ })
+ return kafkaRecords.toTriggerOutput({ records, topic: kafkaConfig.readTopic(topic), batch })
+}
+
+export const newMessage = createTrigger({
+ auth: kafkaAuth,
+ name: 'new_message',
+ displayName: 'New message in topic',
+ description: 'Reads each new message on a topic, including its payload, key, partition, and offset. Messages are marked as read before the flow runs, so if a run fails, that message is not delivered again.',
+ aiMetadata: {
+ description: 'Read new Apache Kafka messages from a topic, one flow run per message. Choose the batch trigger when one run should receive the whole group of messages. Offsets are committed to the consumer group before the flow runs, so a message from a failed run is not delivered again, and a shared group splits messages across flows. The flow polling schedule decides how often the topic is checked.',
+ },
+ classification: 'READ',
+ props: kafkaTriggerProps.fields({ batch: false }),
+ sampleData: {
+ topic: 'orders',
+ partition: 0,
+ offset: '42',
+ key: 'order-1',
+ payload: '{"id":"order-1"}',
+ headers: { source: 'checkout' },
+ timestamp: '1710000000000',
+ },
+ type: TriggerStrategy.POLLING,
+ async test(context) {
+ return readMessages({
+ auth: context.auth.props,
+ topic: context.propsValue.topic,
+ consumerGroup: context.propsValue.consumerGroup,
+ maxMessages: context.propsValue.maxMessages,
+ pollTimeoutSeconds: context.propsValue.pollTimeoutSeconds,
+ startFrom: context.propsValue.startFrom,
+ commit: false,
+ batch: false,
+ })
+ },
+ async onEnable(context) {
+ await kafkaClient.pinStartOffsets({
+ auth: context.auth.props,
+ topic: context.propsValue.topic,
+ consumerGroup: context.propsValue.consumerGroup,
+ fromBeginning: kafkaConfig.readStartFrom(context.propsValue.startFrom),
+ })
+ },
+ async onDisable() {
+ return
+ },
+ async run(context) {
+ return readMessages({
+ auth: context.auth.props,
+ topic: context.propsValue.topic,
+ consumerGroup: context.propsValue.consumerGroup,
+ maxMessages: context.propsValue.maxMessages,
+ pollTimeoutSeconds: context.propsValue.pollTimeoutSeconds,
+ startFrom: context.propsValue.startFrom,
+ commit: true,
+ batch: false,
+ })
+ },
+})
diff --git a/packages/pieces/community/apache-kafka/src/lib/triggers/new-messages-batch.ts b/packages/pieces/community/apache-kafka/src/lib/triggers/new-messages-batch.ts
new file mode 100644
index 00000000000..9076f85eb2b
--- /dev/null
+++ b/packages/pieces/community/apache-kafka/src/lib/triggers/new-messages-batch.ts
@@ -0,0 +1,86 @@
+import { createTrigger, TriggerStrategy } from '@activepieces/pieces-framework'
+import { kafkaAuth } from '../auth'
+import { kafkaClient } from '../common/client'
+import { kafkaConfig } from '../common/config'
+import { kafkaRecords } from '../common/records'
+import { kafkaTriggerProps } from './props'
+
+async function readBatch({ auth, topic, consumerGroup, maxMessages, pollTimeoutSeconds, startFrom, commit }: {
+ auth: Parameters<typeof kafkaClient.consume>[0]['auth']
+ topic: string
+ consumerGroup: string
+ maxMessages: number
+ pollTimeoutSeconds: number
+ startFrom: string
+ commit: boolean
+}) {
+ const records = await kafkaClient.consume({
+ auth,
+ topic,
+ consumerGroup,
+ maxMessages,
+ pollTimeoutSeconds,
+ fromBeginning: kafkaConfig.readStartFrom(startFrom),
+ commit,
+ })
+ return kafkaRecords.toTriggerOutput({ records, topic: kafkaConfig.readTopic(topic), batch: true })
+}
+
+export const newMessagesBatch = createTrigger({
+ auth: kafkaAuth,
+ name: 'new_messages_batch',
+ displayName: 'New messages in topic',
+ description: 'Reads a batch of new messages on a topic in one run. Messages are marked as read before the flow runs, so if a run fails, those messages are not delivered again.',
+ aiMetadata: {
+ description: 'Read a batch of new Apache Kafka messages from a topic in one flow run. Choose the single-message trigger when each message should start its own run. Offsets are committed to the consumer group before the flow runs, so messages from a failed run are not delivered again. The flow polling schedule decides how often the topic is checked.',
+ },
+ classification: 'READ',
+ props: kafkaTriggerProps.fields({ batch: true }),
+ sampleData: {
+ topic: 'orders',
+ message_count: 1,
+ messages: [{
+ topic: 'orders',
+ partition: 0,
+ offset: '42',
+ key: 'order-1',
+ payload: '{"id":"order-1"}',
+ headers: { source: 'checkout' },
+ timestamp: '1710000000000',
+ }],
+ },
+ type: TriggerStrategy.POLLING,
+ async test(context) {
+ return readBatch({
+ auth: context.auth.props,
+ topic: context.propsValue.topic,
+ consumerGroup: context.propsValue.consumerGroup,
+ maxMessages: context.propsValue.maxMessages,
+ pollTimeoutSeconds: context.propsValue.pollTimeoutSeconds,
+ startFrom: context.propsValue.startFrom,
+ commit: false,
+ })
+ },
+ async onEnable(context) {
+ await kafkaClient.pinStartOffsets({
+ auth: context.auth.props,
+ topic: context.propsValue.topic,
+ consumerGroup: context.propsValue.consumerGroup,
+ fromBeginning: kafkaConfig.readStartFrom(context.propsValue.startFrom),
+ })
+ },
+ async onDisable() {
+ return
+ },
+ async run(context) {
+ return readBatch({
+ auth: context.auth.props,
+ topic: context.propsValue.topic,
+ consumerGroup: context.propsValue.consumerGroup,
+ maxMessages: context.propsValue.maxMessages,
+ pollTimeoutSeconds: context.propsValue.pollTimeoutSeconds,
+ startFrom: context.propsValue.startFrom,
+ commit: true,
+ })
+ },
+})
diff --git a/packages/pieces/community/apache-kafka/src/lib/triggers/props.ts b/packages/pieces/community/apache-kafka/src/lib/triggers/props.ts
new file mode 100644
index 00000000000..62f4f45f418
--- /dev/null
+++ b/packages/pieces/community/apache-kafka/src/lib/triggers/props.ts
@@ -0,0 +1,44 @@
+import { Property } from '@activepieces/pieces-framework'
+
+export const kafkaTriggerProps = {
+ fields({ batch }: { batch: boolean }) {
+ return {
+ topic: Property.ShortText({
+ displayName: 'Topic',
+ description: 'Topic to read, for example orders.',
+ required: true,
+ }),
+ consumerGroup: Property.ShortText({
+ displayName: 'Consumer group',
+ description: 'Kafka consumer group id, for example activepieces-orders. Use a different group for each flow that should receive every message. Flows that share a group split the topic between them.',
+ required: true,
+ }),
+ maxMessages: Property.Number({
+ displayName: 'Max messages',
+ description: batch
+ ? 'Maximum messages included in one batch. Whole number from 1 to 500.'
+ : 'Maximum messages to read each time the flow checks the topic. Each message starts its own run. Whole number from 1 to 500.',
+ required: true,
+ defaultValue: batch ? 100 : 50,
+ }),
+ pollTimeoutSeconds: Property.Number({
+ displayName: 'Poll timeout (seconds)',
+ description: 'How long to wait for messages on each check. Whole number from 1 to 20. The flow schedule controls how often checks run.',
+ required: true,
+ defaultValue: 10,
+ }),
+ startFrom: Property.StaticDropdown({
+ displayName: 'Start from',
+ description: 'Used only when this consumer group has no committed offset yet. New messages only skips history. Beginning of the topic reads what is still retained.',
+ required: true,
+ defaultValue: 'latest',
+ options: {
+ options: [
+ { label: 'New messages only', value: 'latest' },
+ { label: 'Beginning of the topic', value: 'beginning' },
+ ],
+ },
+ }),
+ }
+ },
+}
diff --git a/packages/pieces/community/apache-kafka/tsconfig.json b/packages/pieces/community/apache-kafka/tsconfig.json
new file mode 100644
index 00000000000..eff240ac143
--- /dev/null
+++ b/packages/pieces/community/apache-kafka/tsconfig.json
@@ -0,0 +1,20 @@
+{
+ "extends": "../../../../tsconfig.base.json",
+ "compilerOptions": {
+ "module": "commonjs",
+ "forceConsistentCasingInFileNames": true,
+ "strict": true,
+ "importHelpers": true,
+ "noImplicitOverride": true,
+ "noImplicitReturns": true,
+ "noFallthroughCasesInSwitch": true,
+ "noPropertyAccessFromIndexSignature": true
+ },
+ "files": [],
+ "include": [],
+ "references": [
+ {
+ "path": "./tsconfig.lib.json"
+ }
+ ]
+}
diff --git a/packages/pieces/community/apache-kafka/tsconfig.lib.json b/packages/pieces/community/apache-kafka/tsconfig.lib.json
new file mode 100644
index 00000000000..0ba4caeb858
--- /dev/null
+++ b/packages/pieces/community/apache-kafka/tsconfig.lib.json
@@ -0,0 +1,15 @@
+{
+ "extends": "./tsconfig.json",
+ "compilerOptions": {
+ "module": "commonjs",
+ "rootDir": ".",
+ "baseUrl": ".",
+ "paths": {},
+ "outDir": "./dist",
+ "declaration": true,
+ "declarationMap": true,
+ "types": ["node"]
+ },
+ "exclude": ["jest.config.ts", "src/**/*.spec.ts", "src/**/*.test.ts"],
+ "include": ["src/**/*.ts"]
+}
diff --git a/packages/pieces/community/apache-kafka/vitest.config.ts b/packages/pieces/community/apache-kafka/vitest.config.ts
new file mode 100644
index 00000000000..ba8ade4a178
--- /dev/null
+++ b/packages/pieces/community/apache-kafka/vitest.config.ts
@@ -0,0 +1,17 @@
+import path from 'path'
+import { defineConfig } from 'vitest/config'
+
+const repoRoot = path.resolve(__dirname, '../../../..')
+
+export default defineConfig({
+ test: {
+ globals: true,
+ environment: 'node',
+ },
+ resolve: {
+ alias: {
+ '@activepieces/pieces-framework': path.resolve(repoRoot, 'packages/pieces/framework/src/index.ts'),
+ '@activepieces/pieces-common': path.resolve(repoRoot, 'packages/pieces/common/src/index.ts'),
+ },
+ },
+})
diff --git a/tsconfig.base.json b/tsconfig.base.json
index 1dd2d29e5cd..09c041554df 100644
--- a/tsconfig.base.json
+++ b/tsconfig.base.json
@@ -99,6 +99,9 @@
"@activepieces/piece-anyhook-websocket": [
"packages/pieces/community/anyhook-websocket/src/index.ts"
],
+ "@activepieces/piece-apache-kafka": [
+ "packages/pieces/community/apache-kafka/src/index.ts"
+ ],
"@activepieces/piece-apify": [
"packages/pieces/community/apify/src/index.ts"
],