Skip to content

AI Agents & MCPs & AI Workflow Automation • (~400 MCP servers for AI agents) • AI Automation / AI Agent with MCPs • AI Workflows & AI Agents • MCPs for AI Agents

feat(apache-kafka): react to new Kafka messages within a second and publish batches in one request (#16732)

Odai Ahmad51b7d5462fc9
diff --git a/bun.lock b/bun.lock
index 578bb7e5339..eaf315260ae 100644
--- a/bun.lock
+++ b/bun.lock
@@ -720,7 +720,7 @@
},
"packages/pieces/community/apache-kafka": {
"name": "@activepieces/piece-apache-kafka",
- "version": "0.0.1",
+ "version": "0.1.0",
"dependencies": {
"@activepieces/pieces-framework": "workspace:*",
"kafkajs": "2.2.4",
diff --git a/docs/install/reference/breaking-changes-unreleased.mdx b/docs/install/reference/breaking-changes-unreleased.mdx
index 5da8e6bb242..37797cabf91 100644
--- a/docs/install/reference/breaking-changes-unreleased.mdx
+++ b/docs/install/reference/breaking-changes-unreleased.mdx
@@ -8,6 +8,14 @@ Entries here move to [Breaking Changes](/install/reference/breaking-changes) aut
## What has changed?
+#### Apache Kafka 0.1.0: Publish messages is no longer an AI agent tool
+
+**Publish messages** (several messages in one request) is no longer offered to AI agents as a tool; flow steps that use it keep working unchanged. Agents publish with **Publish message**, one message per call, so there is one agent tool for publishing.
+
+#### What you need to do
+
+If a Run Agent step or MCP server used Publish messages as a tool, switch it to Publish message.
+
#### PandaDoc 0.3.0: seven actions are builder-only, and the triggers now fire
Create Document from Template, Create Attachment, Create or Update Contact, Find Document(s), Get Document Attachment, Get Document and Download Document are no longer offered to AI agents. They keep working in flows, but an agent that used them as tools loses them after the upgrade.
diff --git a/packages/pieces/community/apache-kafka/package.json b/packages/pieces/community/apache-kafka/package.json
index d7e9e028f9f..1860b231b2d 100644
--- a/packages/pieces/community/apache-kafka/package.json
+++ b/packages/pieces/community/apache-kafka/package.json
@@ -1,6 +1,6 @@
{
"name": "@activepieces/piece-apache-kafka",
- "version": "0.0.1",
+ "version": "0.1.0",
"type": "commonjs",
"main": "./dist/src/index.js",
"types": "./dist/src/index.d.ts",
diff --git a/packages/pieces/community/apache-kafka/src/i18n/translation.json b/packages/pieces/community/apache-kafka/src/i18n/translation.json
index d60956d5977..6bf3161e33b 100644
--- a/packages/pieces/community/apache-kafka/src/i18n/translation.json
+++ b/packages/pieces/community/apache-kafka/src/i18n/translation.json
@@ -55,5 +55,6 @@
"Used only when this consumer group has no committed offset yet. New messages only skips history. Beginning of the topic reads what is still retained.": "Used only when this consumer group has no committed offset yet. New messages only skips history. Beginning of the topic reads what is still retained.",
"Maximum messages included in one batch. Whole number from 1 to 500.": "Maximum messages included in one batch. Whole number from 1 to 500.",
"New messages only": "New messages only",
- "Beginning of the topic": "Beginning of the topic"
-}
\ No newline at end of file
+ "Beginning of the topic": "Beginning of the topic",
+ "Longest time one check spends reading. A check with no new messages ends at once. Whole number from 1 to 20.": "Longest time one check spends reading. A check with no new messages ends at once. Whole number from 1 to 20."
+}
diff --git a/packages/pieces/community/apache-kafka/src/lib/actions/publish-message.ts b/packages/pieces/community/apache-kafka/src/lib/actions/publish-message.ts
index e174c179857..aa24cca835e 100644
--- a/packages/pieces/community/apache-kafka/src/lib/actions/publish-message.ts
+++ b/packages/pieces/community/apache-kafka/src/lib/actions/publish-message.ts
@@ -1,6 +1,7 @@
import { createAction, Property } from '@activepieces/pieces-framework'
import { kafkaAuth } from '../auth'
import { kafkaClient } from '../common/client'
+import { kafkaOutputSchemas } from '../output-schemas'
export const publishMessage = createAction({
auth: kafkaAuth,
@@ -10,9 +11,10 @@ export const publishMessage = createAction({
audience: 'both',
classification: 'WRITE',
aiMetadata: {
- description: 'Publish one message to an Apache Kafka topic. Use this for a single record; use Publish messages when several records should go out in the same step. The call waits for all in-sync replicas and returns the partition and offset. A retry publishes another message.',
+ description: 'Publish one message to an Apache Kafka topic. Call it once per record. The call waits for all in-sync replicas and returns the partition and offset. A retry publishes another message.',
idempotent: false,
},
+ outputSchema: kafkaOutputSchemas.publishMessage,
props: {
topic: Property.ShortText({
displayName: 'Topic',
diff --git a/packages/pieces/community/apache-kafka/src/lib/actions/publish-messages.ts b/packages/pieces/community/apache-kafka/src/lib/actions/publish-messages.ts
index ef2ccb0d896..f9115e93626 100644
--- a/packages/pieces/community/apache-kafka/src/lib/actions/publish-messages.ts
+++ b/packages/pieces/community/apache-kafka/src/lib/actions/publish-messages.ts
@@ -1,18 +1,20 @@
import { createAction, Property } from '@activepieces/pieces-framework'
import { kafkaAuth } from '../auth'
import { kafkaClient } from '../common/client'
+import { kafkaOutputSchemas } from '../output-schemas'
export const publishMessages = createAction({
auth: kafkaAuth,
name: 'publish_messages',
displayName: 'Publish messages',
description: 'Publishes several messages to a Kafka topic in one step and returns a partition and offset for each.',
- audience: 'both',
+ audience: 'human',
classification: 'WRITE',
aiMetadata: {
- description: 'Publish multiple messages to one Apache Kafka topic in a single step and return each message partition and offset. Use Publish message when there is only one record. Messages are sent in order on one connection, waiting for all in-sync replicas. A retry publishes the batch again.',
+ description: 'Publish multiple messages to one Apache Kafka topic in a single step and return each message partition and offset. Use Publish message when there is only one record. All messages go to the brokers in one request, in order within each partition, waiting for all in-sync replicas. A retry publishes the batch again.',
idempotent: false,
},
+ outputSchema: kafkaOutputSchemas.publishMessages,
props: {
topic: Property.ShortText({
displayName: 'Topic',
diff --git a/packages/pieces/community/apache-kafka/src/lib/common/__tests__/client.test.ts b/packages/pieces/community/apache-kafka/src/lib/common/__tests__/client.test.ts
index 4ae05a9f27f..fbb2a5e2c81 100644
--- a/packages/pieces/community/apache-kafka/src/lib/common/__tests__/client.test.ts
+++ b/packages/pieces/community/apache-kafka/src/lib/common/__tests__/client.test.ts
@@ -1,5 +1,15 @@
import { beforeEach, describe, expect, it, vi } from 'vitest'
+type MockMessage = {
+ key: Buffer | null
+ value: Buffer | null
+ offset: string
+ timestamp: string
+ headers?: Record<string, Buffer>
+}
+
+type MockBatch = { topic: string, partition: number, messages: MockMessage[], lastOffset: () => string }
+
const kafkaMock = vi.hoisted(() => {
const adminConnect = vi.fn(async () => undefined)
const adminDisconnect = vi.fn(async () => undefined)
@@ -16,6 +26,11 @@ const kafkaMock = vi.hoisted(() => {
],
}])
const setOffsets = vi.fn(async () => undefined)
+ const deleteGroups = vi.fn(async () => undefined)
+ const fetchTopicMetadata = vi.fn(async () => ({
+ topics: [{ name: 'orders', partitions: [{ partitionId: 0, leader: 1 }, { partitionId: 1, leader: 1 }] }],
+ }))
+ const kafkaConfigs: unknown[] = []
const producerConnect = vi.fn(async () => undefined)
const producerDisconnect = vi.fn(async () => undefined)
const send = vi.fn(async () => [{
@@ -29,20 +44,21 @@ const kafkaMock = vi.hoisted(() => {
const subscribe = vi.fn(async () => undefined)
const commitOffsets = vi.fn(async () => undefined)
const stop = vi.fn(async () => undefined)
+ const assignment: { orders: number[] } = { orders: [0, 1] }
+ const joinListeners: ((event: { payload: { memberAssignment: Record<string, number[]> } }) => void)[] = []
+ const on = vi.fn((_event: string, listener: (event: { payload: { memberAssignment: Record<string, number[]> } }) => void) => {
+ joinListeners.push(listener)
+ })
+ const resolveOffset = vi.fn()
+ const asBatch = ({ topic, partition, message, lastOffset }: { topic: string, partition: number, message: MockMessage, lastOffset?: string }) => ({
+ batch: { topic, partition, messages: [message], lastOffset: () => lastOffset ?? message.offset },
+ resolveOffset,
+ })
const run = vi.fn(async (config: {
- eachMessage: (payload: {
- topic: string
- partition: number
- message: {
- key: Buffer | null
- value: Buffer | null
- offset: string
- timestamp: string
- headers?: Record<string, Buffer>
- }
- }) => Promise<void>
+ eachBatch: (payload: { batch: MockBatch, resolveOffset: (offset: string) => void }) => Promise<void>
}) => {
- await config.eachMessage({
+ joinListeners.forEach((listener) => listener({ payload: { memberAssignment: assignment } }))
+ await config.eachBatch(asBatch({
topic: 'orders',
partition: 1,
message: {
@@ -52,15 +68,20 @@ const kafkaMock = vi.hoisted(() => {
timestamp: '1710000000000',
headers: { source: Buffer.from('checkout') },
},
- })
+ }))
})
return {
+ asBatch,
+ resolveOffset,
adminConnect,
adminDisconnect,
describeCluster,
fetchTopicOffsets,
fetchOffsets,
setOffsets,
+ deleteGroups,
+ fetchTopicMetadata,
+ kafkaConfigs,
producerConnect,
producerDisconnect,
send,
@@ -69,6 +90,9 @@ const kafkaMock = vi.hoisted(() => {
subscribe,
commitOffsets,
stop,
+ on,
+ assignment,
+ joinListeners,
run,
}
})
@@ -76,9 +100,13 @@ const kafkaMock = vi.hoisted(() => {
vi.mock('kafkajs', () => ({
logLevel: { NOTHING: 0 },
Partitioners: {
- DefaultPartitioner: () => () => 0,
+ DefaultPartitioner: () => ({ message }: { message: { key?: string } }) => (message.key === 'a' ? 0 : 1),
},
Kafka: class Kafka {
+ constructor(config: unknown) {
+ kafkaMock.kafkaConfigs.push(config)
+ }
+
admin() {
return {
connect: kafkaMock.adminConnect,
@@ -87,6 +115,8 @@ vi.mock('kafkajs', () => ({
fetchTopicOffsets: kafkaMock.fetchTopicOffsets,
fetchOffsets: kafkaMock.fetchOffsets,
setOffsets: kafkaMock.setOffsets,
+ deleteGroups: kafkaMock.deleteGroups,
+ fetchTopicMetadata: kafkaMock.fetchTopicMetadata,
}
}
@@ -105,6 +135,8 @@ vi.mock('kafkajs', () => ({
subscribe: kafkaMock.subscribe,
commitOffsets: kafkaMock.commitOffsets,
stop: kafkaMock.stop,
+ on: kafkaMock.on,
+ events: { GROUP_JOIN: 'consumer.group_join' },
run: kafkaMock.run,
}
}
@@ -118,9 +150,21 @@ const auth = {
securityProtocol: 'PLAINTEXT',
}
+const pollInput = {
+ auth,
+ topic: 'orders',
+ consumerGroup: 'activepieces-orders',
+ maxMessages: 10,
+ pollTimeoutSeconds: 20,
+ fromBeginning: false,
+}
+
describe('kafkaClient', () => {
beforeEach(() => {
vi.clearAllMocks()
+ kafkaMock.joinListeners.length = 0
+ kafkaMock.assignment.orders = [0, 1]
+ kafkaMock.kafkaConfigs.length = 0
})
it('rejects a connection when the cluster cannot be described', async () => {
@@ -139,6 +183,14 @@ describe('kafkaClient', () => {
expect(kafkaMock.adminDisconnect).toHaveBeenCalled()
})
+ it('validates with a single short connection attempt', async () => {
+ await kafkaClient.validate(auth)
+ expect(kafkaMock.kafkaConfigs[0]).toMatchObject({
+ connectionTimeout: 5_000,
+ retry: { retries: 0 },
+ })
+ })
+
it('does not open a socket when the broker list is invalid', async () => {
const result = await kafkaClient.validate({
brokers: 'not-a-broker',
@@ -148,34 +200,78 @@ describe('kafkaClient', () => {
expect(kafkaMock.adminConnect).not.toHaveBeenCalled()
})
- it('returns the partition and offset for each published message', async () => {
+ it('publishes every message in one send and derives each offset from the partition base offset', async () => {
+ kafkaMock.send.mockResolvedValueOnce([
+ { topicName: 'orders', partition: 0, errorCode: 0, baseOffset: '10' },
+ { topicName: 'orders', partition: 1, errorCode: 0, baseOffset: '20' },
+ ])
const results = await kafkaClient.publishMessages({
auth,
topic: 'orders',
messages: [
- { key: 'order-1', value: 'hello' },
- { value: 'world' },
+ { key: 'a', value: 'first' },
+ { key: 'b', value: 'second' },
+ { key: 'a', value: 'third' },
+ ],
+ })
+ expect(kafkaMock.send).toHaveBeenCalledTimes(1)
+ expect(kafkaMock.send).toHaveBeenCalledWith({
+ topic: 'orders',
+ acks: -1,
+ messages: [
+ expect.objectContaining({ key: 'a', value: 'first', partition: 0 }),
+ expect.objectContaining({ key: 'b', value: 'second', partition: 1 }),
+ expect.objectContaining({ key: 'a', value: 'third', partition: 0 }),
],
})
expect(results).toEqual([
- { topic: 'orders', key: 'order-1', partition: 3, offset: '15' },
- { topic: 'orders', key: null, partition: 3, offset: '15' },
+ { topic: 'orders', key: 'a', partition: 0, offset: '10' },
+ { topic: 'orders', key: 'b', partition: 1, offset: '20' },
+ { topic: 'orders', key: 'a', partition: 0, offset: '11' },
])
- expect(kafkaMock.send).toHaveBeenCalledTimes(2)
expect(kafkaMock.producerDisconnect).toHaveBeenCalled()
})
- it('commits the next offset only when the trigger run asks it to', async () => {
- const committed = await kafkaClient.consume({
+ it('publishes a single message without fetching topic metadata', async () => {
+ const results = await kafkaClient.publishMessages({
auth,
topic: 'orders',
- consumerGroup: 'activepieces-orders',
- maxMessages: 1,
- pollTimeoutSeconds: 1,
- fromBeginning: false,
- commit: true,
+ messages: [{ value: 'hello' }],
})
- expect(committed).toEqual([{
+ expect(results).toEqual([{ topic: 'orders', key: null, partition: 3, offset: '15' }])
+ expect(kafkaMock.fetchTopicMetadata).not.toHaveBeenCalled()
+ })
+
+ it('fails when the broker rejects a partition', async () => {
+ kafkaMock.send.mockResolvedValueOnce([
+ { topicName: 'orders', partition: 0, errorCode: 0, baseOffset: '10' },
+ { topicName: 'orders', partition: 1, errorCode: 6, baseOffset: '-1' },
+ ])
+ await expect(kafkaClient.publishMessages({
+ auth,
+ topic: 'orders',
+ messages: [{ key: 'a', value: 'first' }, { key: 'b', value: 'second' }],
+ })).rejects.toThrow('Broker returned error code 6')
+ })
+
+ it('returns at once without joining the group when nothing is pending', async () => {
+ kafkaMock.fetchOffsets.mockResolvedValueOnce([{
+ topic: 'orders',
+ partitions: [
+ { partition: 0, offset: '20', metadata: null },
+ { partition: 1, offset: '9', metadata: null },
+ ],
+ }])
+ const records = await kafkaClient.poll(pollInput)
+ expect(records).toEqual([])
+ expect(kafkaMock.consumerConnect).not.toHaveBeenCalled()
+ expect(kafkaMock.setOffsets).not.toHaveBeenCalled()
+ })
+
+ it('reads up to the pending end, commits the next offset and returns without waiting for the timeout', async () => {
+ const started = Date.now()
+ const records = await kafkaClient.poll(pollInput)
+ expect(records).toEqual([{
topic: 'orders',
partition: 1,
offset: '8',
@@ -187,32 +283,11 @@ describe('kafkaClient', () => {
expect(kafkaMock.commitOffsets).toHaveBeenCalledWith([
{ topic: 'orders', partition: 1, offset: '9' },
])
-
- kafkaMock.commitOffsets.mockClear()
- kafkaMock.setOffsets.mockClear()
- await kafkaClient.consume({
- auth,
- topic: 'orders',
- consumerGroup: 'activepieces-orders',
- maxMessages: 1,
- pollTimeoutSeconds: 1,
- fromBeginning: false,
- commit: false,
- })
- expect(kafkaMock.commitOffsets).not.toHaveBeenCalled()
- expect(kafkaMock.setOffsets).not.toHaveBeenCalled()
+ expect(Date.now() - started).toBeLessThan(1_000)
})
it('pins the start offset of every partition without a commit before a trigger run reads', async () => {
- await kafkaClient.consume({
- auth,
- topic: 'orders',
- consumerGroup: 'activepieces-orders',
- maxMessages: 1,
- pollTimeoutSeconds: 1,
- fromBeginning: false,
- commit: true,
- })
+ await kafkaClient.poll(pollInput)
expect(kafkaMock.setOffsets).toHaveBeenCalledWith({
groupId: 'activepieces-orders',
topic: 'orders',
@@ -221,6 +296,28 @@ describe('kafkaClient', () => {
expect(kafkaMock.setOffsets.mock.invocationCallOrder[0]).toBeLessThan(kafkaMock.consumerConnect.mock.invocationCallOrder[0])
})
+ it('reads the group commit before the topic end so another member committing past an old end is not reset', async () => {
+ kafkaMock.fetchOffsets.mockResolvedValueOnce([{
+ topic: 'orders',
+ partitions: [
+ { partition: 0, offset: '25', metadata: null },
+ { partition: 1, offset: '9', metadata: null },
+ ],
+ }])
+ kafkaMock.fetchTopicOffsets.mockResolvedValueOnce([
+ { partition: 0, offset: '25', low: '3', high: '25' },
+ { partition: 1, offset: '9', low: '0', high: '9' },
+ ])
+ await kafkaClient.pinStartOffsets({
+ auth,
+ topic: 'orders',
+ consumerGroup: 'activepieces-orders',
+ fromBeginning: true,
+ })
+ expect(kafkaMock.fetchOffsets.mock.invocationCallOrder[0]).toBeLessThan(kafkaMock.fetchTopicOffsets.mock.invocationCallOrder[0])
+ expect(kafkaMock.setOffsets).not.toHaveBeenCalled()
+ })
+
it('leaves the group untouched when every partition already has a commit', async () => {
kafkaMock.fetchOffsets.mockResolvedValueOnce([{
topic: 'orders',
@@ -239,44 +336,63 @@ describe('kafkaClient', () => {
expect(kafkaMock.adminDisconnect).toHaveBeenCalled()
})
+ it('samples recent messages through a throwaway group without committing or touching the live group', async () => {
+ const records = await kafkaClient.sample({
+ auth,
+ topic: 'orders',
+ consumerGroup: 'activepieces-orders',
+ count: 5,
+ pollTimeoutSeconds: 1,
+ })
+ expect(records.map((record) => record.offset)).toEqual(['8'])
+ const [setCall] = kafkaMock.setOffsets.mock.calls
+ expect(setCall[0]).toMatchObject({
+ topic: 'orders',
+ partitions: [{ partition: 0, offset: '15' }, { partition: 1, offset: '4' }],
+ })
+ expect(setCall[0].groupId).toMatch(/^activepieces-orders-activepieces-test-/)
+ expect(kafkaMock.fetchOffsets).not.toHaveBeenCalled()
+ expect(kafkaMock.commitOffsets).not.toHaveBeenCalled()
+ expect(kafkaMock.deleteGroups).toHaveBeenCalledWith([setCall[0].groupId])
+ })
+
+ it('samples nothing from an empty topic without starting a consumer', async () => {
+ kafkaMock.fetchTopicOffsets.mockResolvedValueOnce([{ partition: 0, offset: '0', low: '0', high: '0' }])
+ const records = await kafkaClient.sample({
+ auth,
+ topic: 'orders',
+ consumerGroup: 'activepieces-orders',
+ count: 5,
+ pollTimeoutSeconds: 5,
+ })
+ expect(records).toEqual([])
+ expect(kafkaMock.consumerConnect).not.toHaveBeenCalled()
+ })
+
it('returns only the committed batch when a message arrives while committing', async () => {
+ kafkaMock.fetchTopicOffsets.mockResolvedValueOnce([
+ { partition: 0, offset: '20', low: '3', high: '20' },
+ { partition: 1, offset: '12', low: '0', high: '12' },
+ ])
const late = {
topic: 'orders',
partition: 1,
- message: {
- key: null,
- value: Buffer.from('late'),
- offset: '9',
- timestamp: '1710000000001',
- },
+ message: { key: null, value: Buffer.from('late'), offset: '9', timestamp: '1710000000001' },
}
- let deliver: ((payload: typeof late) => Promise<void>) | undefined
+ let deliver: ((payload: { batch: MockBatch, resolveOffset: (offset: string) => void }) => Promise<void>) | undefined
kafkaMock.run.mockImplementationOnce(async (config) => {
- deliver = config.eachMessage
- await config.eachMessage({
+ deliver = config.eachBatch
+ await config.eachBatch(kafkaMock.asBatch({
topic: 'orders',
partition: 1,
- message: {
- key: Buffer.from('order-1'),
- value: Buffer.from('hello'),
- offset: '8',
- timestamp: '1710000000000',
- },
- })
+ message: { key: Buffer.from('order-1'), value: Buffer.from('hello'), offset: '8', timestamp: '1710000000000' },
+ }))
})
kafkaMock.commitOffsets.mockImplementationOnce(async () => {
- await deliver?.(late)
+ await deliver?.(kafkaMock.asBatch(late))
})
- const records = await kafkaClient.consume({
- auth,
- topic: 'orders',
- consumerGroup: 'activepieces-orders',
- maxMessages: 10,
- pollTimeoutSeconds: 5,
- fromBeginning: false,
- commit: true,
- })
+ const records = await kafkaClient.poll({ ...pollInput, pollTimeoutSeconds: 1 })
expect(records.map((record) => record.offset)).toEqual(['8'])
expect(kafkaMock.commitOffsets).toHaveBeenCalledWith([
@@ -287,55 +403,91 @@ describe('kafkaClient', () => {
it('waits for the consumer to join the group before timing the poll', async () => {
kafkaMock.run.mockImplementationOnce(async (config) => {
await new Promise((resolve) => setTimeout(resolve, 1_500))
+ kafkaMock.joinListeners.forEach((listener) => listener({ payload: { memberAssignment: kafkaMock.assignment } }))
setTimeout(() => {
- config.eachMessage({
+ config.eachBatch(kafkaMock.asBatch({
topic: 'orders',
- partition: 0,
- message: { key: null, value: Buffer.from('after join'), offset: '3', timestamp: '1710000000000' },
- }).catch(() => undefined)
+ partition: 1,
+ message: { key: null, value: Buffer.from('after join'), offset: '8', timestamp: '1710000000000' },
+ })).catch(() => undefined)
}, 50)
})
- const records = await kafkaClient.consume({
- auth,
+ const records = await kafkaClient.poll({ ...pollInput, pollTimeoutSeconds: 1 })
+
+ expect(records.map((record) => record.payload)).toEqual(['after join'])
+ })
+
+ it('does not wait for partitions another group member owns', async () => {
+ kafkaMock.fetchOffsets.mockResolvedValueOnce([{
topic: 'orders',
- consumerGroup: 'activepieces-orders',
- maxMessages: 10,
- pollTimeoutSeconds: 1,
- fromBeginning: false,
- commit: false,
+ partitions: [
+ { partition: 0, offset: '10', metadata: null },
+ { partition: 1, offset: '8', metadata: null },
+ ],
+ }])
+ kafkaMock.assignment.orders = [1]
+ const startedAt = Date.now()
+
+ const records = await kafkaClient.poll({ ...pollInput, pollTimeoutSeconds: 5 })
+
+ expect(records.map((record) => record.offset)).toEqual(['8'])
+ expect(Date.now() - startedAt).toBeLessThan(2_000)
+ })
+
+ it('explains the prefixed ACL when the broker refuses the test group', async () => {
+ kafkaMock.setOffsets.mockRejectedValueOnce(Object.assign(new Error('Not authorized to access group'), { type: 'GROUP_AUTHORIZATION_FAILED' }))
+
+ await expect(kafkaClient.sample({ auth, topic: 'orders', consumerGroup: 'activepieces-orders', count: 5, pollTimeoutSeconds: 1 }))
+ .rejects.toThrow(/PREFIXED group ACL on "activepieces-orders"/)
+ })
+
+ it('finishes and commits past a trailing transaction marker without waiting for the timeout', async () => {
+ kafkaMock.fetchTopicOffsets.mockResolvedValueOnce([
+ { partition: 0, offset: '20', low: '3', high: '20' },
+ { partition: 1, offset: '10', low: '0', high: '10' },
+ ])
+ kafkaMock.run.mockImplementationOnce(async (config) => {
+ kafkaMock.joinListeners.forEach((listener) => listener({ payload: { memberAssignment: kafkaMock.assignment } }))
+ await config.eachBatch(kafkaMock.asBatch({
+ topic: 'orders',
+ partition: 1,
+ message: { key: null, value: Buffer.from('last real'), offset: '8', timestamp: '1710000000000' },
+ lastOffset: '9',
+ }))
})
+ const startedAt = Date.now()
- expect(records.map((record) => record.payload)).toEqual(['after join'])
+ const records = await kafkaClient.poll({ ...pollInput, pollTimeoutSeconds: 5 })
+
+ expect(records.map((record) => record.offset)).toEqual(['8'])
+ expect(kafkaMock.commitOffsets).toHaveBeenCalledWith([{ topic: 'orders', partition: 1, offset: '10' }])
+ expect(kafkaMock.resolveOffset.mock.calls.map(([offset]) => offset)).toEqual(['8', '9'])
+ expect(Date.now() - startedAt).toBeLessThan(2_000)
+ })
+
+ it('reads a partition across two fetched batches and resolves each offset', async () => {
+ kafkaMock.fetchTopicOffsets.mockResolvedValueOnce([
+ { partition: 0, offset: '20', low: '3', high: '20' },
+ { partition: 1, offset: '10', low: '0', high: '10' },
+ ])
+ kafkaMock.run.mockImplementationOnce(async (config) => {
+ kafkaMock.joinListeners.forEach((listener) => listener({ payload: { memberAssignment: kafkaMock.assignment } }))
+ await config.eachBatch(kafkaMock.asBatch({ topic: 'orders', partition: 1, message: { key: null, value: Buffer.from('a'), offset: '8', timestamp: '1710000000000' } }))
+ await config.eachBatch(kafkaMock.asBatch({ topic: 'orders', partition: 1, message: { key: null, value: Buffer.from('b'), offset: '9', timestamp: '1710000000001' } }))
+ })
+
+ const records = await kafkaClient.poll({ ...pollInput, pollTimeoutSeconds: 5 })
+
+ expect(records.map((record) => record.payload)).toEqual(['a', 'b'])
+ expect(kafkaMock.resolveOffset.mock.calls.map(([offset]) => offset)).toEqual(['8', '8', '9', '9'])
+ expect(kafkaMock.commitOffsets).toHaveBeenCalledWith([{ topic: 'orders', partition: 1, offset: '10' }])
})
it('reports a consumer that fails to start', async () => {
kafkaMock.run.mockRejectedValueOnce(new Error('Group coordinator not available'))
- await expect(kafkaClient.consume({
- auth,
- topic: 'orders',
- consumerGroup: 'activepieces-orders',
- maxMessages: 10,
- pollTimeoutSeconds: 1,
- fromBeginning: false,
- commit: false,
- })).rejects.toThrow()
+ await expect(kafkaClient.poll({ ...pollInput, pollTimeoutSeconds: 1 })).rejects.toThrow()
expect(kafkaMock.consumerDisconnect).toHaveBeenCalled()
})
-
- it('returns a partial batch once messages stop arriving', async () => {
- const started = Date.now()
- const records = await kafkaClient.consume({
- auth,
- topic: 'orders',
- consumerGroup: 'activepieces-orders',
- maxMessages: 10,
- pollTimeoutSeconds: 20,
- fromBeginning: false,
- commit: false,
- })
- expect(records).toHaveLength(1)
- expect(Date.now() - started).toBeLessThan(2_000)
- })
})
diff --git a/packages/pieces/community/apache-kafka/src/lib/common/__tests__/config.test.ts b/packages/pieces/community/apache-kafka/src/lib/common/__tests__/config.test.ts
index 634704f0daa..07c85f6dc59 100644
--- a/packages/pieces/community/apache-kafka/src/lib/common/__tests__/config.test.ts
+++ b/packages/pieces/community/apache-kafka/src/lib/common/__tests__/config.test.ts
@@ -69,4 +69,23 @@ describe('kafkaConfig', () => {
brokers: 'kafka.internal:9092',
})).toBe('Enter a topic name, for example orders.')
})
+
+ it('builds a fail-fast config for connection checks', () => {
+ expect(kafkaConfig.buildForValidation({
+ brokers: 'kafka.internal:9092',
+ securityProtocol: 'PLAINTEXT',
+ })).toMatchObject({
+ brokers: ['kafka.internal:9092'],
+ connectionTimeout: 5_000,
+ retry: { retries: 0 },
+ })
+ })
+
+ it('does not repeat a cause that is already in the error message', () => {
+ const error = new Error('Connection error: connect ECONNREFUSED 127.0.0.1:9999', {
+ cause: new Error('Connection error: connect ECONNREFUSED 127.0.0.1:9999'),
+ })
+ const message = kafkaConfig.describeError({ error, brokers: 'localhost:9999' })
+ expect(message.match(/ECONNREFUSED/g)).toHaveLength(1)
+ })
})
diff --git a/packages/pieces/community/apache-kafka/src/lib/common/__tests__/records.test.ts b/packages/pieces/community/apache-kafka/src/lib/common/__tests__/records.test.ts
index 9822ed9109c..134eefbeb67 100644
--- a/packages/pieces/community/apache-kafka/src/lib/common/__tests__/records.test.ts
+++ b/packages/pieces/community/apache-kafka/src/lib/common/__tests__/records.test.ts
@@ -89,7 +89,79 @@ describe('kafkaRecords', () => {
{ key: 'order-1', value: 'hello', headers: { source: 'checkout' }, partition: 1 },
])
expect(() => kafkaRecords.readOutboundMessages([])).toThrow('Add at least one message')
- expect(() => kafkaRecords.readOutboundMessage({ value: '' }, 0)).toThrow('Message 1 needs a value')
- expect(() => kafkaRecords.readOutboundMessage({ value: 'hello', partition: -1 }, 0)).toThrow('Partition must be a whole number')
+ expect(() => kafkaRecords.readOutboundMessage({ value: { value: '' }, index: 0 })).toThrow('Message 1 needs a value')
+ expect(() => kafkaRecords.readOutboundMessage({ value: { value: 'hello', partition: -1 }, index: 0 })).toThrow('Partition must be a whole number')
+ })
+
+ it('plans pending reads from committed offsets and skips caught-up partitions', () => {
+ expect(kafkaRecords.pendingTargets({
+ topicOffsets: [
+ { partition: 0, low: '0', high: '10' },
+ { partition: 1, low: '5', high: '7' },
+ { partition: 2, low: '0', high: '4' },
+ ],
+ committed: [
+ { partition: 0, offset: '10' },
+ { partition: 1, offset: '2' },
+ { partition: 2, offset: '1' },
+ ],
+ })).toEqual([
+ { partition: 1, offset: '5', end: '7' },
+ { partition: 2, offset: '1', end: '4' },
+ ])
+ })
+
+ it('plans recent reads per partition and skips empty partitions', () => {
+ expect(kafkaRecords.recentTargets({
+ topicOffsets: [
+ { partition: 0, low: '0', high: '10' },
+ { partition: 1, low: '2', high: '3' },
+ { partition: 2, low: '4', high: '4' },
+ ],
+ count: 5,
+ })).toEqual([
+ { partition: 0, offset: '5', end: '10' },
+ { partition: 1, offset: '2', end: '3' },
+ ])
+ })
+
+ it('orders sampled records newest first and keeps the requested count', () => {
+ const record = { topic: 't', partition: 0, key: null, payload: 'x', headers: {} }
+ expect(kafkaRecords.newestFirst({
+ records: [
+ { ...record, offset: '1', timestamp: '100' },
+ { ...record, offset: '2', timestamp: '300' },
+ { ...record, offset: '3', timestamp: '200' },
+ ],
+ count: 2,
+ }).map((entry) => entry.offset)).toEqual(['2', '3'])
+ })
+
+ it('maps one send response to per-message offsets', () => {
+ expect(kafkaRecords.publishResults({
+ topic: 'orders',
+ messages: [
+ { value: 'a', partition: 1 },
+ { value: 'b', partition: 1, key: 'k' },
+ ],
+ metadata: [{ topicName: 'orders', partition: 1, errorCode: 0, baseOffset: '41' }],
+ })).toEqual([
+ { topic: 'orders', key: null, partition: 1, offset: '41' },
+ { topic: 'orders', key: 'k', partition: 1, offset: '42' },
+ ])
+ })
+ it('re-applies the start policy when the committed offset is past the end of a truncated log', () => {
+ const plan = kafkaRecords.startOffsetPlan({
+ topicOffsets: [{ partition: 0, low: '0', high: '100' }, { partition: 1, low: '0', high: '50' }],
+ committed: [{ partition: 0, offset: '1000' }, { partition: 1, offset: '40' }],
+ fromBeginning: false,
+ })
+ expect(plan).toEqual([{ partition: 0, offset: '100' }])
+ })
+
+ it('knows when a sample window already starts at the beginning of every partition', () => {
+ const topicOffsets = [{ partition: 0, low: '0', high: '3' }, { partition: 1, low: '5', high: '100' }]
+ expect(kafkaRecords.reachesLogStart({ topicOffsets, targets: kafkaRecords.recentTargets({ topicOffsets, count: 5 }) })).toBe(false)
+ expect(kafkaRecords.reachesLogStart({ topicOffsets, targets: kafkaRecords.recentTargets({ topicOffsets, count: 500 }) })).toBe(true)
})
})
diff --git a/packages/pieces/community/apache-kafka/src/lib/common/client.ts b/packages/pieces/community/apache-kafka/src/lib/common/client.ts
index 051c0838dcb..8d2bf5a2275 100644
--- a/packages/pieces/community/apache-kafka/src/lib/common/client.ts
+++ b/packages/pieces/community/apache-kafka/src/lib/common/client.ts
@@ -1,13 +1,14 @@
-import { Kafka, Partitioners, type EachMessagePayload } from 'kafkajs'
+import { randomUUID } from 'node:crypto'
+import type { Admin, EachBatchPayload, Kafka, KafkaConfig } from 'kafkajs'
import { kafkaConfig, type KafkaAuthInput } from './config'
-import { kafkaRecords, type KafkaRecord } from './records'
+import { kafkaRecords, type KafkaRecord, type OutboundMessage, type PublishResult, type ReadTarget } from './records'
async function validate(auth: KafkaAuthInput): Promise<{ valid: true } | { valid: false, error: string }> {
const requirement = kafkaConfig.requirementError(auth)
if (requirement !== undefined) {
return { valid: false, error: requirement }
}
- const admin = new Kafka(kafkaConfig.build(auth)).admin()
+ const admin = (await createKafka(kafkaConfig.buildForValidation(auth))).admin()
try {
await admin.connect()
await admin.describeCluster()
@@ -29,34 +30,20 @@ async function publishMessages({ auth, topic, messages }: {
}): Promise<PublishResult[]> {
const topicName = kafkaConfig.readTopic(topic)
const outbound = kafkaRecords.readOutboundMessages(messages)
- const producer = new Kafka(kafkaConfig.build(auth)).producer({
+ const { Partitioners } = await import('kafkajs')
+ const kafka = await createKafka(kafkaConfig.build(auth))
+ const producer = kafka.producer({
createPartitioner: Partitioners.DefaultPartitioner,
})
try {
+ const assigned = await assignPartitions({ kafka, topic: topicName, messages: outbound })
await producer.connect()
- const results: PublishResult[] = []
- for (const message of outbound) {
- const metadata = await producer.send({
- topic: topicName,
- acks: -1,
- messages: [kafkaRecords.toKafkaMessage(message)],
- })
- const record = metadata[0]
- if (record === undefined || record.errorCode !== 0) {
- throw new Error(`Broker returned error code ${record?.errorCode ?? 'unknown'}.`)
- }
- const offset = record.baseOffset ?? record.offset
- if (offset === undefined || offset.length === 0) {
- throw new Error('Kafka did not return an offset for the published message.')
- }
- results.push({
- topic: topicName,
- key: message.key ?? null,
- partition: record.partition,
- offset,
- })
- }
- return results
+ const metadata = await producer.send({
+ topic: topicName,
+ acks: -1,
+ messages: assigned.map((message) => kafkaRecords.toKafkaMessage(message)),
+ })
+ return kafkaRecords.publishResults({ topic: topicName, messages: assigned, metadata })
} catch (error) {
throw new Error(kafkaConfig.describeError({ error, brokers: auth.brokers, topic: topicName }))
} finally {
@@ -72,21 +59,10 @@ async function pinStartOffsets({ auth, topic, consumerGroup, fromBeginning }: {
}): Promise<void> {
const topicName = kafkaConfig.readTopic(topic)
const groupId = kafkaConfig.readConsumerGroup(consumerGroup)
- const admin = new Kafka(kafkaConfig.build(auth)).admin()
+ const admin = (await createKafka(kafkaConfig.build(auth))).admin()
try {
await admin.connect()
- const [topicOffsets, groupOffsets] = await Promise.all([
- admin.fetchTopicOffsets(topicName),
- admin.fetchOffsets({ groupId, topics: [topicName] }),
- ])
- const partitions = kafkaRecords.startOffsetPlan({
- topicOffsets,
- committed: groupOffsets.find((entry) => entry.topic === topicName)?.partitions ?? [],
- fromBeginning,
- })
- if (partitions.length > 0) {
- await admin.setOffsets({ groupId, topic: topicName, partitions })
- }
+ await seedOffsets({ admin, topic: topicName, groupId, fromBeginning })
} catch (error) {
throw new Error(kafkaConfig.describeError({ error, brokers: auth.brokers, topic: topicName }))
} finally {
@@ -94,86 +70,228 @@ async function pinStartOffsets({ auth, topic, consumerGroup, fromBeginning }: {
}
}
-async function consume({ auth, topic, consumerGroup, maxMessages, pollTimeoutSeconds, fromBeginning, commit }: {
+async function poll({ auth, topic, consumerGroup, maxMessages, pollTimeoutSeconds, fromBeginning }: {
auth: KafkaAuthInput
topic: string
consumerGroup: string
maxMessages: number
pollTimeoutSeconds: number
fromBeginning: boolean
- commit: boolean
}): Promise<KafkaRecord[]> {
const topicName = kafkaConfig.readTopic(topic)
const groupId = kafkaConfig.readConsumerGroup(consumerGroup)
const limit = kafkaConfig.readMaxMessages(maxMessages)
- const pollTimeoutMs = kafkaConfig.readPollTimeoutMs(pollTimeoutSeconds)
- if (commit) {
- await pinStartOffsets({ auth, topic: topicName, consumerGroup: groupId, fromBeginning })
+ const timeoutMs = kafkaConfig.readPollTimeoutMs(pollTimeoutSeconds)
+ const kafka = await createKafka(kafkaConfig.build(auth))
+ const admin = kafka.admin()
+ try {
+ await admin.connect()
+ const targets = await seedOffsets({ admin, topic: topicName, groupId, fromBeginning })
+ await admin.disconnect().catch(() => undefined)
+ if (targets.length === 0) {
+ return []
+ }
+ return await readTargets({ kafka, topic: topicName, groupId, targets, limit, timeoutMs, fromBeginning, commit: true })
+ } catch (error) {
+ throw new Error(kafkaConfig.describeError({ error, brokers: auth.brokers, topic: topicName }))
+ } finally {
+ await admin.disconnect().catch(() => undefined)
+ }
+}
+
+async function sample({ auth, topic, consumerGroup, count, pollTimeoutSeconds }: {
+ auth: KafkaAuthInput
+ topic: string
+ consumerGroup: string
+ count: number
+ pollTimeoutSeconds: number
+}): Promise<KafkaRecord[]> {
+ const topicName = kafkaConfig.readTopic(topic)
+ const liveGroup = kafkaConfig.readConsumerGroup(consumerGroup)
+ const limit = kafkaConfig.readMaxMessages(count)
+ const timeoutMs = kafkaConfig.readPollTimeoutMs(pollTimeoutSeconds)
+ const groupId = `${liveGroup}-activepieces-test-${randomUUID()}`
+ const kafka = await createKafka(kafkaConfig.build(auth))
+ const admin = kafka.admin()
+ try {
+ await admin.connect()
+ const topicOffsets = await admin.fetchTopicOffsets(topicName)
+ let records: KafkaRecord[] = []
+ for (const factor of SAMPLE_WINDOW_FACTORS) {
+ const targets = kafkaRecords.recentTargets({ topicOffsets, count: limit * factor })
+ if (targets.length === 0) {
+ return []
+ }
+ await admin.setOffsets({
+ groupId,
+ topic: topicName,
+ partitions: targets.map((target) => ({ partition: target.partition, offset: target.offset })),
+ })
+ records = await readTargets({ kafka, topic: topicName, groupId, targets, limit: limit * factor * targets.length, timeoutMs, fromBeginning: true, commit: false })
+ if (records.length >= limit || kafkaRecords.reachesLogStart({ topicOffsets, targets })) {
+ break
+ }
+ }
+ return kafkaRecords.newestFirst({ records, count: limit })
+ } catch (error) {
+ if (kafkaConfig.isGroupAuthorizationError(error)) {
+ throw new Error(`Kafka refused the test consumer group "${groupId}". Testing reads with a temporary group whose name starts with "${liveGroup}-activepieces-test-". Grant this user a PREFIXED group ACL on "${liveGroup}" (Read and Delete), or test by publishing a message and letting the flow run.`)
+ }
+ throw new Error(kafkaConfig.describeError({ error, brokers: auth.brokers, topic: topicName }))
+ } finally {
+ await admin.deleteGroups([groupId]).catch((error: unknown) => {
+ console.warn(`Could not delete the temporary Kafka consumer group "${groupId}" after the test. Grant Delete on the PREFIXED group ACL "${liveGroup}" so test groups are removed; until then Kafka keeps it until its offsets expire.`, error instanceof Error ? error.message : String(error))
+ })
+ await admin.disconnect().catch(() => undefined)
+ }
+}
+
+async function seedOffsets({ admin, topic, groupId, fromBeginning }: {
+ admin: Admin
+ topic: string
+ groupId: string
+ fromBeginning: boolean
+}): Promise<ReadTarget[]> {
+ const groupOffsets = await admin.fetchOffsets({ groupId, topics: [topic] })
+ const topicOffsets = await admin.fetchTopicOffsets(topic)
+ const committed = (groupOffsets.find((entry) => entry.topic === topic)?.partitions ?? [])
+ .filter((entry) => BigInt(entry.offset) >= BigInt(0))
+ const seeded = kafkaRecords.startOffsetPlan({ topicOffsets, committed, fromBeginning })
+ if (seeded.length > 0) {
+ await admin.setOffsets({ groupId, topic, partitions: seeded })
}
- const consumer = new Kafka(kafkaConfig.build(auth)).consumer({
+ return kafkaRecords.pendingTargets({ topicOffsets, committed: [...committed, ...seeded] })
+}
+
+async function readTargets({ kafka, topic, groupId, targets, limit, timeoutMs, fromBeginning, commit }: {
+ kafka: Kafka
+ topic: string
+ groupId: string
+ targets: ReadTarget[]
+ limit: number
+ timeoutMs: number
+ fromBeginning: boolean
+ commit: boolean
+}): Promise<KafkaRecord[]> {
+ const ends = new Map(targets.map((target) => [target.partition, BigInt(target.end)]))
+ const positions = new Map<number, bigint>()
+ let assigned: Set<number> | undefined
+ const consumer = kafka.consumer({
groupId,
- maxWaitTimeInMs: 500,
+ maxWaitTimeInMs: 250,
sessionTimeout: 15_000,
rebalanceTimeout: 15_000,
heartbeatInterval: 3_000,
})
+ consumer.on(consumer.events.GROUP_JOIN, ({ payload }) => {
+ assigned = new Set(payload.memberAssignment[topic] ?? [])
+ })
try {
await consumer.connect()
- await consumer.subscribe({ topic: topicName, fromBeginning })
+ await consumer.subscribe({ topic, fromBeginning })
const records: KafkaRecord[] = []
let collecting = true
await consumer.run({
autoCommit: false,
- eachMessage: async ({ topic: messageTopic, partition, message }: EachMessagePayload) => {
- if (!collecting || records.length >= limit) {
+ eachBatchAutoResolve: false,
+ eachBatch: async ({ batch, resolveOffset }: EachBatchPayload) => {
+ const end = ends.get(batch.partition)
+ if (!collecting || end === undefined) {
return
}
- records.push(kafkaRecords.toRecord({ topic: messageTopic, partition, message }))
+ let position = positions.get(batch.partition)
+ for (const message of batch.messages) {
+ const offset = BigInt(message.offset)
+ if (offset >= end) {
+ break
+ }
+ if (records.length >= limit) {
+ if (position !== undefined) {
+ positions.set(batch.partition, position)
+ }
+ return
+ }
+ records.push(kafkaRecords.toRecord({ topic: batch.topic, partition: batch.partition, message }))
+ resolveOffset(message.offset)
+ position = offset + BigInt(1)
+ }
+ resolveOffset(batch.lastOffset())
+ const fetchedUpTo = BigInt(batch.lastOffset()) + BigInt(1)
+ const reached = fetchedUpTo < end ? fetchedUpTo : end
+ positions.set(batch.partition, position !== undefined && position > reached ? position : reached)
},
})
- const deadline = Date.now() + pollTimeoutMs
- let lastCount = 0
- let quietSince = Date.now()
- while (records.length < limit && Date.now() < deadline) {
- if (records.length > 0 && records.length === lastCount && Date.now() - quietSince >= 500) {
- break
- }
- if (records.length !== lastCount) {
- lastCount = records.length
- quietSince = Date.now()
- }
- await delay(100)
+ const deadline = Date.now() + timeoutMs
+ while (records.length < limit && !ownTargetsDone({ ends, positions, assigned }) && Date.now() < deadline) {
+ await delay(50)
}
collecting = false
const batch = [...records]
- if (commit && batch.length > 0) {
- await consumer.commitOffsets(kafkaRecords.commitPlan({ topic: topicName, records: batch }))
+ const committed = [...positions.entries()].map(([partition, offset]) => ({ topic, partition, offset: offset.toString() }))
+ if (commit && committed.length > 0) {
+ await consumer.commitOffsets(committed)
}
return batch
- } catch (error) {
- throw new Error(kafkaConfig.describeError({ error, brokers: auth.brokers, topic: topicName }))
} finally {
await consumer.stop().catch(() => undefined)
await consumer.disconnect().catch(() => undefined)
}
}
+async function assignPartitions({ kafka, topic, messages }: {
+ kafka: Kafka
+ topic: string
+ messages: OutboundMessage[]
+}): Promise<OutboundMessage[]> {
+ if (messages.length === 1 || messages.every((message) => message.partition !== undefined)) {
+ return messages
+ }
+ const { Partitioners } = await import('kafkajs')
+ const admin = kafka.admin()
+ try {
+ await admin.connect()
+ const metadata = await admin.fetchTopicMetadata({ topics: [topic] })
+ const partitionMetadata = metadata.topics.find((entry) => entry.name === topic)?.partitions ?? []
+ const partitioner = Partitioners.DefaultPartitioner()
+ return messages.map((message) => message.partition !== undefined
+ ? message
+ : { ...message, partition: partitioner({ topic, partitionMetadata, message: kafkaRecords.toKafkaMessage(message) }) })
+ } finally {
+ await admin.disconnect().catch(() => undefined)
+ }
+}
+
+async function createKafka(config: KafkaConfig): Promise<Kafka> {
+ const { Kafka: KafkaClient } = await import('kafkajs')
+ return new KafkaClient(config)
+}
+
+function ownTargetsDone({ ends, positions, assigned }: {
+ ends: Map<number, bigint>
+ positions: Map<number, bigint>
+ assigned: Set<number> | undefined
+}): boolean {
+ if (assigned === undefined) {
+ return false
+ }
+ const own = assigned
+ return [...ends.entries()]
+ .filter(([partition]) => own.has(partition))
+ .every(([partition, end]) => (positions.get(partition) ?? BigInt(0)) >= end)
+}
+
function delay(ms: number): Promise<void> {
return new Promise((resolve) => {
setTimeout(resolve, ms)
})
}
+const SAMPLE_WINDOW_FACTORS = [1, 10, 100]
+
export const kafkaClient = {
validate,
publishMessages,
pinStartOffsets,
- consume,
-}
-
-export type PublishResult = {
- topic: string
- key: string | null
- partition: number
- offset: string
+ poll,
+ sample,
}
diff --git a/packages/pieces/community/apache-kafka/src/lib/common/config.ts b/packages/pieces/community/apache-kafka/src/lib/common/config.ts
index 1ce9809134a..48595c5b803 100644
--- a/packages/pieces/community/apache-kafka/src/lib/common/config.ts
+++ b/packages/pieces/community/apache-kafka/src/lib/common/config.ts
@@ -1,4 +1,7 @@
-import { logLevel, type KafkaConfig, type SASLOptions } from 'kafkajs'
+import type { KafkaConfig, SASLOptions } from 'kafkajs'
+
+const KAFKAJS_LOG_NOTHING = 0
+const VALIDATION_TIMEOUT_MS = 5_000
function parseBrokers(brokers: string): string[] {
const parsed = brokers
@@ -57,7 +60,7 @@ function build(input: KafkaAuthInput): KafkaConfig {
}
: undefined,
sasl: usesSasl(securityProtocol) ? readSasl(input) : undefined,
- logLevel: logLevel.NOTHING,
+ logLevel: KAFKAJS_LOG_NOTHING,
connectionTimeout: 10_000,
authenticationTimeout: 10_000,
requestTimeout: 15_000,
@@ -69,6 +72,16 @@ function build(input: KafkaAuthInput): KafkaConfig {
}
}
+function buildForValidation(input: KafkaAuthInput): KafkaConfig {
+ return {
+ ...build(input),
+ connectionTimeout: VALIDATION_TIMEOUT_MS,
+ authenticationTimeout: VALIDATION_TIMEOUT_MS,
+ requestTimeout: VALIDATION_TIMEOUT_MS,
+ retry: { retries: 0 },
+ }
+}
+
function describeError({ error, brokers, topic }: { error: unknown, brokers: string, topic?: string }): string {
const raw = errorText(error)
if (isPassthrough(raw)) {
@@ -170,12 +183,19 @@ function errorText(error: unknown): string {
if (!(error instanceof Error)) {
return 'Unknown Kafka error'
}
- if (error.cause instanceof Error && error.cause.message.length > 0) {
+ if (error.cause instanceof Error && error.cause.message.length > 0 && !error.message.includes(error.cause.message)) {
return `${error.message} ${error.cause.message}`
}
return error.message
}
+function isGroupAuthorizationError(error: unknown): boolean {
+ if (typeof error === 'object' && error !== null && Reflect.get(error, 'type') === 'GROUP_AUTHORIZATION_FAILED') {
+ return true
+ }
+ return /not authorized to access group/i.test(errorText(error))
+}
+
function isPassthrough(message: string): boolean {
const prefixes = [
'Enter at least one broker',
@@ -208,9 +228,11 @@ function isSaslMechanism(value: string): value is SaslMechanism {
}
export const kafkaConfig = {
+ isGroupAuthorizationError,
parseBrokers,
requirementError,
build,
+ buildForValidation,
describeError,
readTopic,
readConsumerGroup,
diff --git a/packages/pieces/community/apache-kafka/src/lib/common/records.ts b/packages/pieces/community/apache-kafka/src/lib/common/records.ts
index 68f2f1f50b6..8d1584f6fb2 100644
--- a/packages/pieces/community/apache-kafka/src/lib/common/records.ts
+++ b/packages/pieces/community/apache-kafka/src/lib/common/records.ts
@@ -1,4 +1,4 @@
-import { type Message } from 'kafkajs'
+import type { Message, RecordMetadata } from 'kafkajs'
function toRecord({ topic, partition, message }: {
topic: string
@@ -46,7 +46,14 @@ function startOffsetPlan({ topicOffsets, committed, fromBeginning }: {
committed: { partition: number, offset: string }[]
fromBeginning: boolean
}): { partition: number, offset: string }[] {
- const pinned = new Set(committed.filter((entry) => BigInt(entry.offset) >= BigInt(0)).map((entry) => entry.partition))
+ const highByPartition = new Map(topicOffsets.map((entry) => [entry.partition, BigInt(entry.high)]))
+ const pinned = new Set(committed
+ .filter((entry) => {
+ const offset = BigInt(entry.offset)
+ const high = highByPartition.get(entry.partition)
+ return offset >= BigInt(0) && (high === undefined || offset <= high)
+ })
+ .map((entry) => entry.partition))
return topicOffsets
.filter((entry) => !pinned.has(entry.partition))
.map((entry) => ({ partition: entry.partition, offset: fromBeginning ? entry.low : entry.high }))
@@ -56,22 +63,22 @@ function readOutboundMessages(value: unknown): OutboundMessage[] {
if (!Array.isArray(value) || value.length === 0) {
throw new Error('Add at least one message. Each message needs a value.')
}
- return value.map((item, index) => readOutboundMessage(item, index))
+ return value.map((item, index) => readOutboundMessage({ value: item, index }))
}
-function readOutboundMessage(value: unknown, index: number): OutboundMessage {
+function readOutboundMessage({ value, index }: { value: unknown, index: number }): OutboundMessage {
if (typeof value !== 'object' || value === null || Array.isArray(value)) {
throw new Error(`${messageLabel(index)} must include a value.`)
}
- const rawValue = readField(value, 'value')
+ const rawValue = readField({ record: value, key: 'value' })
if (typeof rawValue !== 'string' || rawValue.length === 0) {
throw new Error(`${messageLabel(index)} needs a value.`)
}
return {
value: rawValue,
- key: readOptionalString(readField(value, 'key'), `${messageLabel(index)} key`),
- partition: readOptionalPartition(readField(value, 'partition'), index),
- headers: readHeadersInput(readField(value, 'headers'), index),
+ key: readOptionalString({ value: readField({ record: value, key: 'key' }), label: `${messageLabel(index)} key` }),
+ partition: readOptionalPartition({ value: readField({ record: value, key: 'partition' }), index }),
+ headers: readHeadersInput({ value: readField({ record: value, key: 'headers' }), index }),
}
}
@@ -102,6 +109,86 @@ function toTriggerOutput({ records, topic, batch }: {
}]
}
+function pendingTargets({ topicOffsets, committed }: {
+ topicOffsets: { partition: number, low: string, high: string }[]
+ committed: { partition: number, offset: string }[]
+}): ReadTarget[] {
+ const committedByPartition = new Map(committed.map((entry) => [entry.partition, BigInt(entry.offset)]))
+ return topicOffsets.flatMap((entry) => {
+ const high = BigInt(entry.high)
+ const low = BigInt(entry.low)
+ const current = committedByPartition.get(entry.partition)
+ const start = current === undefined || current < low ? low : current
+ if (start >= high) {
+ return []
+ }
+ return [{ partition: entry.partition, offset: start.toString(), end: entry.high }]
+ })
+}
+
+function reachesLogStart({ topicOffsets, targets }: {
+ topicOffsets: { partition: number, low: string }[]
+ targets: ReadTarget[]
+}): boolean {
+ const lowByPartition = new Map(topicOffsets.map((entry) => [entry.partition, entry.low]))
+ return targets.every((target) => lowByPartition.get(target.partition) === target.offset)
+}
+
+function recentTargets({ topicOffsets, count }: {
+ topicOffsets: { partition: number, low: string, high: string }[]
+ count: number
+}): ReadTarget[] {
+ return topicOffsets.flatMap((entry) => {
+ const high = BigInt(entry.high)
+ const low = BigInt(entry.low)
+ if (high <= low) {
+ return []
+ }
+ const fromCount = high - BigInt(count)
+ const start = fromCount > low ? fromCount : low
+ return [{ partition: entry.partition, offset: start.toString(), end: entry.high }]
+ })
+}
+
+function newestFirst({ records, count }: { records: KafkaRecord[], count: number }): KafkaRecord[] {
+ return [...records]
+ .sort((left, right) => compareBigInt({ left: BigInt(right.timestamp), right: BigInt(left.timestamp) }))
+ .slice(0, count)
+}
+
+function publishResults({ topic, messages, metadata }: {
+ topic: string
+ messages: OutboundMessage[]
+ metadata: RecordMetadata[]
+}): PublishResult[] {
+ const nextIndex = new Map<number, number>()
+ return messages.map((message) => {
+ const record = messages.length === 1 ? metadata[0] : metadata.find((entry) => entry.partition === message.partition)
+ if (record === undefined || record.errorCode !== 0) {
+ throw new Error(`Broker returned error code ${record?.errorCode ?? 'unknown'}.`)
+ }
+ const base = record.baseOffset ?? record.offset
+ if (base === undefined || base.length === 0) {
+ throw new Error('Kafka did not return an offset for the published message.')
+ }
+ const index = nextIndex.get(record.partition) ?? 0
+ nextIndex.set(record.partition, index + 1)
+ return {
+ topic,
+ key: message.key ?? null,
+ partition: record.partition,
+ offset: (BigInt(base) + BigInt(index)).toString(),
+ }
+ })
+}
+
+function compareBigInt({ left, right }: { left: bigint, right: bigint }): number {
+ if (left === right) {
+ return 0
+ }
+ return left < right ? -1 : 1
+}
+
function headerRecord(headers: Record<string, Buffer | string | (Buffer | string)[] | undefined> | undefined): Record<string, string> {
if (headers === undefined) {
return {}
@@ -124,7 +211,7 @@ function headerValue(value: Buffer | string | (Buffer | string)[]): string {
return value.map((entry) => (Buffer.isBuffer(entry) ? entry.toString('utf8') : entry)).join(',')
}
-function readHeadersInput(value: unknown, index: number): Record<string, string> | undefined {
+function readHeadersInput({ value, index }: { value: unknown, index: number }): Record<string, string> | undefined {
if (value === undefined || value === null) {
return undefined
}
@@ -140,7 +227,7 @@ function readHeadersInput(value: unknown, index: number): Record<string, string>
}, {})
}
-function readOptionalString(value: unknown, label: string): string | undefined {
+function readOptionalString({ value, label }: { value: unknown, label: string }): string | undefined {
if (value === undefined || value === null || value === '') {
return undefined
}
@@ -150,7 +237,7 @@ function readOptionalString(value: unknown, label: string): string | undefined {
return value
}
-function readOptionalPartition(value: unknown, index: number): number | undefined {
+function readOptionalPartition({ value, index }: { value: unknown, index: number }): number | undefined {
if (value === undefined || value === null || value === '') {
return undefined
}
@@ -160,7 +247,7 @@ function readOptionalPartition(value: unknown, index: number): number | undefine
return value
}
-function readField(record: object, key: string): unknown {
+function readField({ record, key }: { record: object, key: string }): unknown {
if (!(key in record)) {
return undefined
}
@@ -179,6 +266,11 @@ export const kafkaRecords = {
readOutboundMessages,
toKafkaMessage,
toTriggerOutput,
+ pendingTargets,
+ recentTargets,
+ reachesLogStart,
+ newestFirst,
+ publishResults,
}
export type KafkaRecord = {
@@ -203,3 +295,16 @@ export type OutboundMessage = {
headers?: Record<string, string>
partition?: number
}
+
+export type ReadTarget = {
+ partition: number
+ offset: string
+ end: string
+}
+
+export type PublishResult = {
+ topic: string
+ key: string | null
+ partition: number
+ offset: string
+}
diff --git a/packages/pieces/community/apache-kafka/src/lib/output-schemas.ts b/packages/pieces/community/apache-kafka/src/lib/output-schemas.ts
new file mode 100644
index 00000000000..7f4911a87f8
--- /dev/null
+++ b/packages/pieces/community/apache-kafka/src/lib/output-schemas.ts
@@ -0,0 +1,59 @@
+import { OutputSchema, OutputSchemaField } from '@activepieces/pieces-framework'
+
+const publishResultFields: OutputSchemaField[] = [
+ { key: 'topic', label: 'Topic' },
+ { key: 'key', label: 'Key' },
+ { key: 'partition', label: 'Partition', format: 'number' },
+ { key: 'offset', label: 'Offset' },
+]
+
+const messageFields: OutputSchemaField[] = [
+ { key: 'topic', label: 'Topic' },
+ { key: 'partition', label: 'Partition', format: 'number' },
+ { key: 'offset', label: 'Offset' },
+ { key: 'key', label: 'Key' },
+ { key: 'payload', label: 'Payload' },
+ { key: 'headers', label: 'Headers', dynamicKey: true },
+ { key: 'timestamp', label: 'Timestamp (Unix ms)' },
+]
+
+const publishMessageOutputSchema: OutputSchema = {
+ fields: publishResultFields,
+}
+
+const publishMessagesOutputSchema: OutputSchema = {
+ itemLabel: 'Message {offset}',
+ fields: [
+ {
+ key: 'messages',
+ label: 'Published Messages',
+ value: '',
+ labelKey: 'offset',
+ listItems: publishResultFields,
+ },
+ ],
+}
+
+const newMessageOutputSchema: OutputSchema = {
+ fields: messageFields,
+}
+
+const newMessagesBatchOutputSchema: OutputSchema = {
+ fields: [
+ { key: 'topic', label: 'Topic' },
+ { key: 'message_count', label: 'Message Count', format: 'number' },
+ {
+ key: 'messages',
+ label: 'Messages',
+ labelKey: 'offset',
+ listItems: messageFields,
+ },
+ ],
+}
+
+export const kafkaOutputSchemas = {
+ publishMessage: publishMessageOutputSchema,
+ publishMessages: publishMessagesOutputSchema,
+ newMessage: newMessageOutputSchema,
+ newMessagesBatch: newMessagesBatchOutputSchema,
+}
diff --git a/packages/pieces/community/apache-kafka/src/lib/triggers/new-message.ts b/packages/pieces/community/apache-kafka/src/lib/triggers/new-message.ts
index 52cc5f9c4f6..c3b8bf6334e 100644
--- a/packages/pieces/community/apache-kafka/src/lib/triggers/new-message.ts
+++ b/packages/pieces/community/apache-kafka/src/lib/triggers/new-message.ts
@@ -1,31 +1,8 @@
import { createTrigger, TriggerStrategy } from '@activepieces/pieces-framework'
import { kafkaAuth } from '../auth'
-import { kafkaClient } from '../common/client'
-import { kafkaConfig } from '../common/config'
-import { kafkaRecords } from '../common/records'
+import { kafkaOutputSchemas } from '../output-schemas'
import { kafkaTriggerProps } from './props'
-
-async function readMessages({ auth, topic, consumerGroup, maxMessages, pollTimeoutSeconds, startFrom, commit, batch }: {
- auth: Parameters<typeof kafkaClient.consume>[0]['auth']
- topic: string
- consumerGroup: string
- maxMessages: number
- pollTimeoutSeconds: number
- startFrom: string
- commit: boolean
- batch: boolean
-}) {
- const records = await kafkaClient.consume({
- auth,
- topic,
- consumerGroup,
- maxMessages,
- pollTimeoutSeconds,
- fromBeginning: kafkaConfig.readStartFrom(startFrom),
- commit,
- })
- return kafkaRecords.toTriggerOutput({ records, topic: kafkaConfig.readTopic(topic), batch })
-}
+import { kafkaTriggerRead } from './read'
export const newMessage = createTrigger({
auth: kafkaAuth,
@@ -46,40 +23,18 @@ export const newMessage = createTrigger({
headers: { source: 'checkout' },
timestamp: '1710000000000',
},
+ outputSchema: kafkaOutputSchemas.newMessage,
type: TriggerStrategy.POLLING,
async test(context) {
- return readMessages({
- auth: context.auth.props,
- topic: context.propsValue.topic,
- consumerGroup: context.propsValue.consumerGroup,
- maxMessages: context.propsValue.maxMessages,
- pollTimeoutSeconds: context.propsValue.pollTimeoutSeconds,
- startFrom: context.propsValue.startFrom,
- commit: false,
- batch: false,
- })
+ return kafkaTriggerRead.sample({ auth: context.auth.props, props: context.propsValue, batch: false })
},
async onEnable(context) {
- await kafkaClient.pinStartOffsets({
- auth: context.auth.props,
- topic: context.propsValue.topic,
- consumerGroup: context.propsValue.consumerGroup,
- fromBeginning: kafkaConfig.readStartFrom(context.propsValue.startFrom),
- })
+ await kafkaTriggerRead.enable({ auth: context.auth.props, props: context.propsValue })
},
async onDisable() {
return
},
async run(context) {
- return readMessages({
- auth: context.auth.props,
- topic: context.propsValue.topic,
- consumerGroup: context.propsValue.consumerGroup,
- maxMessages: context.propsValue.maxMessages,
- pollTimeoutSeconds: context.propsValue.pollTimeoutSeconds,
- startFrom: context.propsValue.startFrom,
- commit: true,
- batch: false,
- })
+ return kafkaTriggerRead.poll({ auth: context.auth.props, props: context.propsValue, batch: false })
},
})
diff --git a/packages/pieces/community/apache-kafka/src/lib/triggers/new-messages-batch.ts b/packages/pieces/community/apache-kafka/src/lib/triggers/new-messages-batch.ts
index 9076f85eb2b..f5ae9d82e10 100644
--- a/packages/pieces/community/apache-kafka/src/lib/triggers/new-messages-batch.ts
+++ b/packages/pieces/community/apache-kafka/src/lib/triggers/new-messages-batch.ts
@@ -1,30 +1,8 @@
import { createTrigger, TriggerStrategy } from '@activepieces/pieces-framework'
import { kafkaAuth } from '../auth'
-import { kafkaClient } from '../common/client'
-import { kafkaConfig } from '../common/config'
-import { kafkaRecords } from '../common/records'
+import { kafkaOutputSchemas } from '../output-schemas'
import { kafkaTriggerProps } from './props'
-
-async function readBatch({ auth, topic, consumerGroup, maxMessages, pollTimeoutSeconds, startFrom, commit }: {
- auth: Parameters<typeof kafkaClient.consume>[0]['auth']
- topic: string
- consumerGroup: string
- maxMessages: number
- pollTimeoutSeconds: number
- startFrom: string
- commit: boolean
-}) {
- const records = await kafkaClient.consume({
- auth,
- topic,
- consumerGroup,
- maxMessages,
- pollTimeoutSeconds,
- fromBeginning: kafkaConfig.readStartFrom(startFrom),
- commit,
- })
- return kafkaRecords.toTriggerOutput({ records, topic: kafkaConfig.readTopic(topic), batch: true })
-}
+import { kafkaTriggerRead } from './read'
export const newMessagesBatch = createTrigger({
auth: kafkaAuth,
@@ -49,38 +27,18 @@ export const newMessagesBatch = createTrigger({
timestamp: '1710000000000',
}],
},
+ outputSchema: kafkaOutputSchemas.newMessagesBatch,
type: TriggerStrategy.POLLING,
async test(context) {
- return readBatch({
- auth: context.auth.props,
- topic: context.propsValue.topic,
- consumerGroup: context.propsValue.consumerGroup,
- maxMessages: context.propsValue.maxMessages,
- pollTimeoutSeconds: context.propsValue.pollTimeoutSeconds,
- startFrom: context.propsValue.startFrom,
- commit: false,
- })
+ return kafkaTriggerRead.sample({ auth: context.auth.props, props: context.propsValue, batch: true })
},
async onEnable(context) {
- await kafkaClient.pinStartOffsets({
- auth: context.auth.props,
- topic: context.propsValue.topic,
- consumerGroup: context.propsValue.consumerGroup,
- fromBeginning: kafkaConfig.readStartFrom(context.propsValue.startFrom),
- })
+ await kafkaTriggerRead.enable({ auth: context.auth.props, props: context.propsValue })
},
async onDisable() {
return
},
async run(context) {
- return readBatch({
- auth: context.auth.props,
- topic: context.propsValue.topic,
- consumerGroup: context.propsValue.consumerGroup,
- maxMessages: context.propsValue.maxMessages,
- pollTimeoutSeconds: context.propsValue.pollTimeoutSeconds,
- startFrom: context.propsValue.startFrom,
- commit: true,
- })
+ return kafkaTriggerRead.poll({ auth: context.auth.props, props: context.propsValue, batch: true })
},
})
diff --git a/packages/pieces/community/apache-kafka/src/lib/triggers/props.ts b/packages/pieces/community/apache-kafka/src/lib/triggers/props.ts
index 62f4f45f418..fac0b3e4bbd 100644
--- a/packages/pieces/community/apache-kafka/src/lib/triggers/props.ts
+++ b/packages/pieces/community/apache-kafka/src/lib/triggers/props.ts
@@ -10,7 +10,7 @@ export const kafkaTriggerProps = {
}),
consumerGroup: Property.ShortText({
displayName: 'Consumer group',
- description: 'Kafka consumer group id, for example activepieces-orders. Use a different group for each flow that should receive every message. Flows that share a group split the topic between them.',
+ description: 'Kafka consumer group id, for example activepieces-orders. Use a different group for each flow that should receive every message. Flows that share a group split the topic between them. Testing the trigger uses a temporary group whose name starts with this id plus -activepieces-test-, so a group ACL must be PREFIXED and allow Read and Delete (Delete removes the temporary group after the test).',
required: true,
}),
maxMessages: Property.Number({
@@ -23,7 +23,7 @@ export const kafkaTriggerProps = {
}),
pollTimeoutSeconds: Property.Number({
displayName: 'Poll timeout (seconds)',
- description: 'How long to wait for messages on each check. Whole number from 1 to 20. The flow schedule controls how often checks run.',
+ description: 'Longest time one check spends reading. A check with no new messages ends at once. Whole number from 1 to 20.',
required: true,
defaultValue: 10,
}),
diff --git a/packages/pieces/community/apache-kafka/src/lib/triggers/read.ts b/packages/pieces/community/apache-kafka/src/lib/triggers/read.ts
new file mode 100644
index 00000000000..f61f5dc2687
--- /dev/null
+++ b/packages/pieces/community/apache-kafka/src/lib/triggers/read.ts
@@ -0,0 +1,60 @@
+import { kafkaClient } from '../common/client'
+import { type KafkaAuthInput, kafkaConfig } from '../common/config'
+import { kafkaRecords } from '../common/records'
+
+const SAMPLE_MESSAGE_COUNT = 5
+const SAMPLE_BATCH_SIZE = 10
+
+async function poll({ auth, props, batch }: {
+ auth: KafkaAuthInput
+ props: TriggerProps
+ batch: boolean
+}) {
+ const records = await kafkaClient.poll({
+ auth,
+ topic: props.topic,
+ consumerGroup: props.consumerGroup,
+ maxMessages: props.maxMessages,
+ pollTimeoutSeconds: props.pollTimeoutSeconds,
+ fromBeginning: kafkaConfig.readStartFrom(props.startFrom),
+ })
+ return kafkaRecords.toTriggerOutput({ records, topic: kafkaConfig.readTopic(props.topic), batch })
+}
+
+async function sample({ auth, props, batch }: {
+ auth: KafkaAuthInput
+ props: TriggerProps
+ batch: boolean
+}) {
+ const records = await kafkaClient.sample({
+ auth,
+ topic: props.topic,
+ consumerGroup: props.consumerGroup,
+ count: Math.min(kafkaConfig.readMaxMessages(props.maxMessages), batch ? SAMPLE_BATCH_SIZE : SAMPLE_MESSAGE_COUNT),
+ pollTimeoutSeconds: props.pollTimeoutSeconds,
+ })
+ return kafkaRecords.toTriggerOutput({ records, topic: kafkaConfig.readTopic(props.topic), batch })
+}
+
+async function enable({ auth, props }: { auth: KafkaAuthInput, props: TriggerProps }) {
+ await kafkaClient.pinStartOffsets({
+ auth,
+ topic: props.topic,
+ consumerGroup: props.consumerGroup,
+ fromBeginning: kafkaConfig.readStartFrom(props.startFrom),
+ })
+}
+
+export const kafkaTriggerRead = {
+ poll,
+ sample,
+ enable,
+}
+
+type TriggerProps = {
+ topic: string
+ consumerGroup: string
+ maxMessages: number
+ pollTimeoutSeconds: number
+ startFrom: string
+}