Skip to content

AI Agents & MCPs & AI Workflow Automation • (~400 MCP servers for AI agents) • AI Automation / AI Agent with MCPs • AI Workflows & AI Agents • MCPs for AI Agents

fix(engine): yield between loop iterations and stop re-walking loop output for log size (#16431)

Amr Elmohamadydceb655c8708
diff --git a/packages/server/engine/src/lib/handler/loop-executor.ts b/packages/server/engine/src/lib/handler/loop-executor.ts
index 3f91d1ff391..797f0b27065 100644
--- a/packages/server/engine/src/lib/handler/loop-executor.ts
+++ b/packages/server/engine/src/lib/handler/loop-executor.ts
@@ -1,3 +1,4 @@
+import { setImmediate } from 'timers/promises'
import { isNil } from '@activepieces/core-utils'
import { LATEST_CONTEXT_VERSION } from '@activepieces/pieces-framework'
import { FlowRunStatus, LoopOnItemsAction, LoopStepOutput } from '@activepieces/shared'
@@ -54,6 +55,7 @@ export const loopExecutor: BaseExecutor<LoopOnItemsAction> = {
for (let i = 0; i < resolvedInput.items.length; ++i) {
+ await setImmediate()
const newCurrentPath = newExecutionContext.currentPath.loopIteration({ loopName: action.name, iteration: i })
const testSingleStepMode = !isNil(constants.stepNameToTest)
diff --git a/packages/server/engine/src/lib/helper/sizeof.ts b/packages/server/engine/src/lib/helper/sizeof.ts
index 1ec2fdd76ce..6ce99af1ba5 100644
--- a/packages/server/engine/src/lib/helper/sizeof.ts
+++ b/packages/server/engine/src/lib/helper/sizeof.ts
@@ -47,11 +47,66 @@ function isSliceRefShape(value: unknown): value is { size: number } {
}
function upsertStepDelta({ stepName, previousStep, nextStep }: UpsertStepDeltaParams): number {
- const nextSize = recursiveSizeof(nextStep)
if (isNil(previousStep)) {
- return utils.sizeof(stepName) + 2 + nextSize
+ return utils.sizeof(stepName) + 2 + recursiveSizeof(nextStep)
}
- return nextSize - recursiveSizeof(previousStep)
+ return sizeDelta({ previous: previousStep, next: nextStep })
+}
+
+function sizeDelta({ previous, next }: SizeDeltaParams): number {
+ if (previous === next) {
+ return 0
+ }
+ if (Array.isArray(previous) && Array.isArray(next)) {
+ return arraySizeDelta({ previous, next })
+ }
+ if (isWalkableObject(previous) && isWalkableObject(next)) {
+ return objectSizeDelta({ previous, next })
+ }
+ return recursiveSizeof(next) - recursiveSizeof(previous)
+}
+
+function arraySizeDelta({ previous, next }: { previous: unknown[], next: unknown[] }): number {
+ const shared = Math.min(previous.length, next.length)
+ let delta = 0
+ for (let i = 0; i < shared; i++) {
+ delta += sizeDelta({ previous: previous[i], next: next[i] })
+ }
+ for (let i = shared; i < next.length; i++) {
+ delta += recursiveSizeof(next[i]) + 1
+ }
+ for (let i = shared; i < previous.length; i++) {
+ delta -= recursiveSizeof(previous[i]) + 1
+ }
+ return delta
+}
+
+function objectSizeDelta({ previous, next }: { previous: Record<string, unknown>, next: Record<string, unknown> }): number {
+ const previousEntries = new Map(Object.entries(previous))
+ const nextKeys = new Set(Object.keys(next))
+ let delta = 0
+ for (const [key, child] of Object.entries(next)) {
+ if (previousEntries.has(key)) {
+ delta += sizeDelta({ previous: previousEntries.get(key), next: child })
+ }
+ else {
+ delta += utils.sizeof(key) + 1 + recursiveSizeof(child) + 1
+ }
+ }
+ for (const [key, child] of previousEntries) {
+ if (!nextKeys.has(key)) {
+ delta -= utils.sizeof(key) + 1 + recursiveSizeof(child) + 1
+ }
+ }
+ return delta
+}
+
+function isWalkableObject(value: unknown): value is Record<string, unknown> {
+ if (typeof value !== 'object' || value === null || Array.isArray(value)) {
+ return false
+ }
+ const isSliceStep = 'outputType' in value && value.outputType === StepOutputType.SLICE && 'output' in value && isSliceRefShape(value.output)
+ return !isSliceStep
}
export const sizeofUtils = {
@@ -59,6 +114,11 @@ export const sizeofUtils = {
upsertStepDelta,
}
+type SizeDeltaParams = {
+ previous: unknown
+ next: unknown
+}
+
type UpsertStepDeltaParams = {
stepName: string
previousStep: unknown
diff --git a/packages/server/engine/test/handler/context/flow-execution-context-log-size.test.ts b/packages/server/engine/test/handler/context/flow-execution-context-log-size.test.ts
index dacdbdc563c..cccf38d78c9 100644
--- a/packages/server/engine/test/handler/context/flow-execution-context-log-size.test.ts
+++ b/packages/server/engine/test/handler/context/flow-execution-context-log-size.test.ts
@@ -54,4 +54,35 @@ describe('FlowExecutorContext.logSizeBytes', () => {
expect(ctx.logSizeBytes).toBe(sizeofUtils.recursiveSizeof(ctx.steps))
expect(ctx.logSizeBytes).toBeGreaterThan(4_096)
})
+
+ it('stays exact when a growing loop is upserted the way the loop executor does it', async () => {
+ let ctx = await FlowExecutorContext.empty().upsertStep('loop', LoopStepOutput.init({ input: { items: [] } }))
+
+ for (let iteration = 0; iteration < 200; iteration++) {
+ const current = ctx.getLoopStepOutput({ stepName: 'loop' }) ?? LoopStepOutput.init({ input: { items: [] } })
+ const next = current.setItemAndIndex({ item: { id: iteration }, index: iteration + 1 }).addIteration()
+ ctx = (await ctx.upsertStep('loop', next))
+ .setCurrentPath(ctx.currentPath.loopIteration({ loopName: 'loop', iteration }))
+ ctx = await ctx.upsertStep('router', pieceStep({ branch: iteration % 2 === 0 ? 'a' : 'fallback' }))
+ ctx = ctx.setCurrentPath(ctx.currentPath.removeLast())
+ }
+
+ expect(ctx.logSizeBytes).toBe(sizeofUtils.recursiveSizeof(ctx.steps))
+ })
+
+ it('stays exact when a step flips between a slice ref and an inline output', async () => {
+ const sliced = new GenericStepOutput({
+ type: FlowActionType.CODE,
+ status: StepOutputStatus.SUCCEEDED,
+ input: {},
+ outputType: StepOutputType.SLICE,
+ output: { fileId: 'file-1', size: 8_192, url: 'http://example.com/file-1' },
+ })
+
+ let ctx = await FlowExecutorContext.empty().upsertStep('echo_step', sliced)
+ ctx = await ctx.upsertStep('echo_step', pieceStep({ small: 'inline' }))
+ ctx = await ctx.upsertStep('echo_step', sliced)
+
+ expect(ctx.logSizeBytes).toBe(sizeofUtils.recursiveSizeof(ctx.steps))
+ })
})
diff --git a/packages/server/engine/test/handler/flow-looping.test.ts b/packages/server/engine/test/handler/flow-looping.test.ts
index c12751605e0..25b00dbf26a 100644
--- a/packages/server/engine/test/handler/flow-looping.test.ts
+++ b/packages/server/engine/test/handler/flow-looping.test.ts
@@ -131,4 +131,42 @@ describe('flow with looping', () => {
expect(() => JSON.stringify(result.steps)).not.toThrow()
})
+ it('should yield to the event loop between iterations', async () => {
+ const iterationCount = 50
+ let macrotaskTicks = 0
+ let keepTicking = true
+ const tick = (): void => {
+ if (!keepTicking) {
+ return
+ }
+ macrotaskTicks++
+ setImmediate(tick)
+ }
+ setImmediate(tick)
+ const ticksAtLoopUpserts: number[] = []
+ const originalUpsertStep = FlowExecutorContext.prototype.upsertStep
+ const upsertSpy = vi.spyOn(FlowExecutorContext.prototype, 'upsertStep').mockImplementation(function (this: FlowExecutorContext, stepName, stepOutput) {
+ if (stepName === 'loop') {
+ ticksAtLoopUpserts.push(macrotaskTicks)
+ }
+ return originalUpsertStep.call(this, stepName, stepOutput)
+ })
+
+ const result = await flowExecutor.execute({
+ action: buildSimpleLoopAction({
+ name: 'loop',
+ loopItems: `{{ Array.from({ length: ${iterationCount} }, (_, i) => i) }}`,
+ }),
+ executionState: FlowExecutorContext.empty(),
+ constants: generateMockEngineConstants({ stepNames: ['loop'] }),
+ }).finally(() => {
+ keepTicking = false
+ upsertSpy.mockRestore()
+ })
+
+ const loopOut = result.steps.loop as LoopStepOutput
+ expect(loopOut.output?.iterations.length).toBe(iterationCount)
+ expect(new Set(ticksAtLoopUpserts).size).toBeGreaterThanOrEqual(iterationCount)
+ })
+
})