AI Agents & MCPs & AI Workflow Automation • (~400 MCP servers for AI agents) • AI Automation / AI Agent with MCPs • AI Workflows & AI Agents • MCPs for AI Agents
TypeScript ai-agentai-agent-toolsai-agentsai-agents-frameworkmcpmcp-servermcp-toolsmcpsn8n-alternativeno-code-automationworkflowworkflow-automationworkflows
fix(engine): yield between loop iterations and stop re-walking loop output for log size (#16431)
Amr Elmohamadydceb655c8708
diff --git a/packages/server/engine/src/lib/handler/loop-executor.ts b/packages/server/engine/src/lib/handler/loop-executor.tsindex 3f91d1ff391..797f0b27065 100644--- a/packages/server/engine/src/lib/handler/loop-executor.ts+++ b/packages/server/engine/src/lib/handler/loop-executor.ts@@ -1,3 +1,4 @@+import { setImmediate } from 'timers/promises'import { isNil } from '@activepieces/core-utils'import { LATEST_CONTEXT_VERSION } from '@activepieces/pieces-framework'import { FlowRunStatus, LoopOnItemsAction, LoopStepOutput } from '@activepieces/shared'@@ -54,6 +55,7 @@ export const loopExecutor: BaseExecutor<LoopOnItemsAction> = {for (let i = 0; i < resolvedInput.items.length; ++i) {+ await setImmediate()const newCurrentPath = newExecutionContext.currentPath.loopIteration({ loopName: action.name, iteration: i })const testSingleStepMode = !isNil(constants.stepNameToTest)diff --git a/packages/server/engine/src/lib/helper/sizeof.ts b/packages/server/engine/src/lib/helper/sizeof.tsindex 1ec2fdd76ce..6ce99af1ba5 100644--- a/packages/server/engine/src/lib/helper/sizeof.ts+++ b/packages/server/engine/src/lib/helper/sizeof.ts@@ -47,11 +47,66 @@ function isSliceRefShape(value: unknown): value is { size: number } {}function upsertStepDelta({ stepName, previousStep, nextStep }: UpsertStepDeltaParams): number {- const nextSize = recursiveSizeof(nextStep)if (isNil(previousStep)) {- return utils.sizeof(stepName) + 2 + nextSize+ return utils.sizeof(stepName) + 2 + recursiveSizeof(nextStep)}- return nextSize - recursiveSizeof(previousStep)+ return sizeDelta({ previous: previousStep, next: nextStep })+}++function sizeDelta({ previous, next }: SizeDeltaParams): number {+ if (previous === next) {+ return 0+ }+ if (Array.isArray(previous) && Array.isArray(next)) {+ return arraySizeDelta({ previous, next })+ }+ if (isWalkableObject(previous) && isWalkableObject(next)) {+ return objectSizeDelta({ previous, next })+ }+ return recursiveSizeof(next) - recursiveSizeof(previous)+}++function arraySizeDelta({ previous, next }: { previous: unknown[], next: unknown[] }): number {+ const shared = Math.min(previous.length, next.length)+ let delta = 0+ for (let i = 0; i < shared; i++) {+ delta += sizeDelta({ previous: previous[i], next: next[i] })+ }+ for (let i = shared; i < next.length; i++) {+ delta += recursiveSizeof(next[i]) + 1+ }+ for (let i = shared; i < previous.length; i++) {+ delta -= recursiveSizeof(previous[i]) + 1+ }+ return delta+}++function objectSizeDelta({ previous, next }: { previous: Record<string, unknown>, next: Record<string, unknown> }): number {+ const previousEntries = new Map(Object.entries(previous))+ const nextKeys = new Set(Object.keys(next))+ let delta = 0+ for (const [key, child] of Object.entries(next)) {+ if (previousEntries.has(key)) {+ delta += sizeDelta({ previous: previousEntries.get(key), next: child })+ }+ else {+ delta += utils.sizeof(key) + 1 + recursiveSizeof(child) + 1+ }+ }+ for (const [key, child] of previousEntries) {+ if (!nextKeys.has(key)) {+ delta -= utils.sizeof(key) + 1 + recursiveSizeof(child) + 1+ }+ }+ return delta+}++function isWalkableObject(value: unknown): value is Record<string, unknown> {+ if (typeof value !== 'object' || value === null || Array.isArray(value)) {+ return false+ }+ const isSliceStep = 'outputType' in value && value.outputType === StepOutputType.SLICE && 'output' in value && isSliceRefShape(value.output)+ return !isSliceStep}export const sizeofUtils = {@@ -59,6 +114,11 @@ export const sizeofUtils = {upsertStepDelta,}+type SizeDeltaParams = {+ previous: unknown+ next: unknown+}+type UpsertStepDeltaParams = {stepName: stringpreviousStep: unknowndiff --git a/packages/server/engine/test/handler/context/flow-execution-context-log-size.test.ts b/packages/server/engine/test/handler/context/flow-execution-context-log-size.test.tsindex dacdbdc563c..cccf38d78c9 100644--- a/packages/server/engine/test/handler/context/flow-execution-context-log-size.test.ts+++ b/packages/server/engine/test/handler/context/flow-execution-context-log-size.test.ts@@ -54,4 +54,35 @@ describe('FlowExecutorContext.logSizeBytes', () => {expect(ctx.logSizeBytes).toBe(sizeofUtils.recursiveSizeof(ctx.steps))expect(ctx.logSizeBytes).toBeGreaterThan(4_096)})++ it('stays exact when a growing loop is upserted the way the loop executor does it', async () => {+ let ctx = await FlowExecutorContext.empty().upsertStep('loop', LoopStepOutput.init({ input: { items: [] } }))++ for (let iteration = 0; iteration < 200; iteration++) {+ const current = ctx.getLoopStepOutput({ stepName: 'loop' }) ?? LoopStepOutput.init({ input: { items: [] } })+ const next = current.setItemAndIndex({ item: { id: iteration }, index: iteration + 1 }).addIteration()+ ctx = (await ctx.upsertStep('loop', next))+ .setCurrentPath(ctx.currentPath.loopIteration({ loopName: 'loop', iteration }))+ ctx = await ctx.upsertStep('router', pieceStep({ branch: iteration % 2 === 0 ? 'a' : 'fallback' }))+ ctx = ctx.setCurrentPath(ctx.currentPath.removeLast())+ }++ expect(ctx.logSizeBytes).toBe(sizeofUtils.recursiveSizeof(ctx.steps))+ })++ it('stays exact when a step flips between a slice ref and an inline output', async () => {+ const sliced = new GenericStepOutput({+ type: FlowActionType.CODE,+ status: StepOutputStatus.SUCCEEDED,+ input: {},+ outputType: StepOutputType.SLICE,+ output: { fileId: 'file-1', size: 8_192, url: 'http://example.com/file-1' },+ })++ let ctx = await FlowExecutorContext.empty().upsertStep('echo_step', sliced)+ ctx = await ctx.upsertStep('echo_step', pieceStep({ small: 'inline' }))+ ctx = await ctx.upsertStep('echo_step', sliced)++ expect(ctx.logSizeBytes).toBe(sizeofUtils.recursiveSizeof(ctx.steps))+ })})diff --git a/packages/server/engine/test/handler/flow-looping.test.ts b/packages/server/engine/test/handler/flow-looping.test.tsindex c12751605e0..25b00dbf26a 100644--- a/packages/server/engine/test/handler/flow-looping.test.ts+++ b/packages/server/engine/test/handler/flow-looping.test.ts@@ -131,4 +131,42 @@ describe('flow with looping', () => {expect(() => JSON.stringify(result.steps)).not.toThrow()})+ it('should yield to the event loop between iterations', async () => {+ const iterationCount = 50+ let macrotaskTicks = 0+ let keepTicking = true+ const tick = (): void => {+ if (!keepTicking) {+ return+ }+ macrotaskTicks+++ setImmediate(tick)+ }+ setImmediate(tick)+ const ticksAtLoopUpserts: number[] = []+ const originalUpsertStep = FlowExecutorContext.prototype.upsertStep+ const upsertSpy = vi.spyOn(FlowExecutorContext.prototype, 'upsertStep').mockImplementation(function (this: FlowExecutorContext, stepName, stepOutput) {+ if (stepName === 'loop') {+ ticksAtLoopUpserts.push(macrotaskTicks)+ }+ return originalUpsertStep.call(this, stepName, stepOutput)+ })++ const result = await flowExecutor.execute({+ action: buildSimpleLoopAction({+ name: 'loop',+ loopItems: `{{ Array.from({ length: ${iterationCount} }, (_, i) => i) }}`,+ }),+ executionState: FlowExecutorContext.empty(),+ constants: generateMockEngineConstants({ stepNames: ['loop'] }),+ }).finally(() => {+ keepTicking = false+ upsertSpy.mockRestore()+ })++ const loopOut = result.steps.loop as LoopStepOutput+ expect(loopOut.output?.iterations.length).toBe(iterationCount)+ expect(new Set(ticksAtLoopUpserts).size).toBeGreaterThanOrEqual(iterationCount)+ })+})