Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
25.0 KB
Raw Blame
/**
 * Telegram Hybrid E2E test — fully automated.
 *
 * Suite 1 — HTTP-direct: opt-in POSTs to mcp-host /message and validates LLM response.
 *
 * Suite 2 — Benchmark with TG delivery: POSTs to mcp-host /message, validates
 *   the response, then posts both the question and answer to the TG group via
 *   the test bot so results are visible. Fully automated, no manual trigger.
 *
 * Suite 3 — Advanced Multi-Tool Benchmark: Chains multiple tools (native + MCP)
 *   across 8 sequential tests sharing conversation context to build a Clerum
 *   landing page. Tests file_read/write, MCP tools, system_info, memory_search.
 *
 * Suite 4 — TG manual trigger (optional): A real user sends a message in TG;
 *   test verifies channel-reader picks it up and replies via pod logs.
 *   Set TG_MANUAL_TRIGGER=1 to enable.
 *
 * Required env vars:
 *   MCP_HOST_URL        — mcp-host endpoint (default: http://localhost:8080)
 *
 * For TG delivery:
 *   TG_TEST_BOT_TOKEN   — Test/observer bot token (sends results to group)
 *   TG_GROUP_CHAT_ID    — Group chat ID
 *
 * Optional:
 *   E2E_RUN_TELEGRAM_HTTP_DIRECT — Set to "1" to enable direct HTTP LLM checks
 *   TG_MANUAL_TRIGGER   — Set to "1" to enable manual TG trigger suite
 *   TG_POLL_TIMEOUT_MS  — Max wait for manual TG test (default: 90000)
 */
import { afterAll, beforeAll, describe, expect, it } from 'vitest'
import { MCP_HOST_URL, fetchJson, kubectl, sendMessage, sleep } from '../helpers'

// ── Config ──────────────────────────────────────────────────────────
const GROUP_CHAT_ID = process.env.TG_GROUP_CHAT_ID || 'test-channel'
const TEST_BOT_TOKEN = process.env.TG_TEST_BOT_TOKEN
const MAIN_BOT_ID = parseInt(process.env.TG_MAIN_BOT_ID || '0000000000', 10)
const POLL_TIMEOUT = parseInt(process.env.TG_POLL_TIMEOUT_MS || '90000', 10)
const MANUAL_TRIGGER = process.env.TG_MANUAL_TRIGGER === '1'
const RUN_HTTP_DIRECT = process.env.E2E_RUN_TELEGRAM_HTTP_DIRECT === '1'
const EXPECTED_MODEL_PROVIDER = process.env.E2E_EXPECTED_MODEL_PROVIDER?.toLowerCase()
const EXPECTED_MODEL_NAME = process.env.E2E_EXPECTED_MODEL_NAME?.toLowerCase()

const TG_API = TEST_BOT_TOKEN ? `https://api.telegram.org/bot${TEST_BOT_TOKEN}` : ''

function assertConfiguredModelIdentity(data: any): void {
  const model = String(data.model || '').toLowerCase()
  const text = `${model} ${data.response || ''}`.toLowerCase()

  expect(model.length).toBeGreaterThan(0)

  if (EXPECTED_MODEL_PROVIDER) {
    expect(text).toContain(EXPECTED_MODEL_PROVIDER)
  } else {
    expect(text).toMatch(/openai|gpt|claude|anthropic|zai|z\.ai|glm|bailian|qwen/)
  }

  if (EXPECTED_MODEL_NAME) {
    expect(text).toContain(EXPECTED_MODEL_NAME)
  }
}

function benchmarkModelLabel(): string {
  const provider = EXPECTED_MODEL_PROVIDER || 'configured provider'
  const model = EXPECTED_MODEL_NAME || 'configured model'
  return `Provider: ${provider} | Model: ${model}`
}

// ── TG helpers ──────────────────────────────────────────────────────

/** Post a message to the TG group via the test bot. */
async function postToGroup(text: string): Promise<number> {
  if (!TG_API) throw new Error('TG_TEST_BOT_TOKEN required for TG delivery')
  const res = await fetch(`${TG_API}/sendMessage`, {
    method: 'POST',
    headers: { 'Content-Type': 'application/json' },
    body: JSON.stringify({
      chat_id: GROUP_CHAT_ID,
      text,
      parse_mode: 'Markdown',
    }),
  })
  const data = await res.json()
  if (!data.ok) {
    // Retry without markdown if parse fails
    const retry = await fetch(`${TG_API}/sendMessage`, {
      method: 'POST',
      headers: { 'Content-Type': 'application/json' },
      body: JSON.stringify({ chat_id: GROUP_CHAT_ID, text }),
    })
    const retryData = await retry.json()
    if (!retryData.ok) throw new Error(`TG sendMessage failed: ${JSON.stringify(retryData)}`)
    return retryData.result.message_id
  }
  return data.result.message_id
}

/**
 * Auto-approve ALL pending MCP tool approval requests for a given userId.
 * Keeps polling and approving until the agent returns to idle or timeout.
 */
async function autoApproveLoop(userId: string, timeoutMs = 90_000): Promise<void> {
  const start = Date.now()
  while (Date.now() - start < timeoutMs) {
    const { data: status } = await fetchJson(`${MCP_HOST_URL}/v1/runtime/status`)
    const pending = status.pendingApprovals || []
    const mine = pending.find((a: any) => a.userId === userId)
    if (mine) {
      console.log(`[AutoApprove] Approving ${mine.toolName} for ${userId}`)
      await fetchJson(`${MCP_HOST_URL}/v1/runtime/approvals/approve`, {
        method: 'POST',
        headers: { 'Content-Type': 'application/json' },
        body: JSON.stringify({
          userId,
          requestId: mine.requestId,
          alwaysApprove: true,
        }),
      })
      // Don't return — keep looping to catch more approvals
      await sleep(500)
      continue
    }
    // Check if agent is done (no approval needed)
    if (status.agent?.state === 'idle') return
    await sleep(1000)
  }
}

/** Send a question via HTTP, auto-approve MCP tools, post Q&A to TG. */
async function benchmarkQuestion(
  label: string,
  question: string,
  opts?: { userId?: string; needsApproval?: boolean }
): Promise<{ status: number; data: any }> {
  const nonce = Math.random().toString(36).slice(2, 8)
  const userId = opts?.userId ?? `e2e-bench-${nonce}`
  const fullQuestion = `[${label}] ${question}`

  // Post question to TG
  if (TG_API) {
    await postToGroup(`🔬 *E2E Benchmark — ${label}*\n\n📩 ${question}`)
  }

  // For MCP tools that need approval, send message and auto-approve in parallel
  if (opts?.needsApproval) {
    // Fire the message (may return immediately with waiting_approval or block)
    const resultPromise = sendMessage(fullQuestion, {
      channelId: GROUP_CHAT_ID,
      userId,
      channelType: 'telegram',
    })

    // Start auto-approve loop in parallel — keeps approving until agent is idle
    const approvePromise = autoApproveLoop(userId, 90_000)

    // Wait for the HTTP response first
    const result = await resultPromise

    // If the response came back as waiting_approval, we need to poll for task result
    if (result.data?.status === 'waiting_approval' || !result.data?.response) {
      const taskId = result.data?.taskId || result.data?.approval?.taskId
      // Keep approving while we wait for task completion
      const start = Date.now()
      while (Date.now() - start < 90_000) {
        // Check and approve any pending approvals inline
        const { data: status } = await fetchJson(`${MCP_HOST_URL}/v1/runtime/status`)
        const pending = status.pendingApprovals || []
        const mine = pending.find((a: any) => a.userId === userId)
        if (mine) {
          console.log(`[AutoApprove] Approving ${mine.toolName} for ${userId}`)
          await fetchJson(`${MCP_HOST_URL}/v1/runtime/approvals/approve`, {
            method: 'POST',
            headers: { 'Content-Type': 'application/json' },
            body: JSON.stringify({ userId, requestId: mine.requestId, alwaysApprove: true }),
          })
          await sleep(500)
          continue
        }

        // Check if task completed
        if (taskId) {
          const taskRes = await fetchJson(`${MCP_HOST_URL}/v1/runtime/tasks/${taskId}/result`)
          if (taskRes.data?.status === 'completed' || taskRes.data?.response) {
            result.data = { ...result.data, ...taskRes.data, success: true, status: 'completed' }
            break
          }
        }

        // If agent is idle, the task is done — fetch final result
        if (status.agent?.state === 'idle') {
          if (taskId) {
            const taskRes = await fetchJson(`${MCP_HOST_URL}/v1/runtime/tasks/${taskId}/result`)
            result.data = { ...result.data, ...taskRes.data, success: true, status: 'completed' }
          }
          break
        }

        await sleep(1000)
      }
    }

    // Cancel the background approve loop (it'll stop on idle anyway)
    await Promise.race([approvePromise, sleep(100)])

    // Post response to TG
    if (TG_API && result.data?.response) {
      const response = result.data.response
      const model = result.data.model || 'unknown'
      const truncated = response.length > 3000 ? response.slice(0, 3000) + '...' : response
      await postToGroup(`✅ *${label}* — _${model}_\n\n${truncated}`)
    } else if (TG_API) {
      await postToGroup(`❌ *${label}* — failed: ${JSON.stringify(result.data).slice(0, 200)}`)
    }
    return result
  }

  // Simple path: no approval needed
  const result = await sendMessage(fullQuestion, {
    channelId: GROUP_CHAT_ID,
    userId,
    channelType: 'telegram',
  })

  if (TG_API && result.data?.response) {
    const response = result.data.response
    const model = result.data.model || 'unknown'
    const truncated = response.length > 3000 ? response.slice(0, 3000) + '...' : response
    await postToGroup(`✅ *${label}* — _${model}_\n\n${truncated}`)
  } else if (TG_API) {
    await postToGroup(`❌ *${label}* — failed: ${JSON.stringify(result.data).slice(0, 200)}`)
  }

  return result
}

// ── Channel-reader log helpers ──────────────────────────────────────

const CHANNEL_READER_NS = process.env.CHANNEL_READER_NS || 'channels'
// Per-Host Deployments are named `channel-reader-<host>` (one per Host CRD).
// Default to `channel-reader-chatllm` since chatllm is the long-standing
// test Host; override via env when the test targets a different Host.
// The legacy `clerum-channel-reader` static Deployment was retired in #273.
const CHANNEL_READER_DEPLOY =
  process.env.CHANNEL_READER_DEPLOY || `channel-reader-${process.env.CLERUM_HOST_REF || 'chatllm'}`

function getReaderLogs(): string {
  return kubectl(`logs deploy/${CHANNEL_READER_DEPLOY} -n ${CHANNEL_READER_NS} --tail=500`)
}

// ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
// Suite 1: HTTP-direct (opt-in; no TG dependency)
// ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━

describe.skipIf(!RUN_HTTP_DIRECT)('HTTP-direct E2E (mcp-host + configured provider)', () => {
  it('should identify model and provider', async () => {
    const nonce = Math.random().toString(36).slice(2, 8)
    const { status, data } = await sendMessage(
      `E2E ${nonce}: What model and provider are you running? Answer in one short line.`,
      { channelId: GROUP_CHAT_ID, userId: 'e2e-test', channelType: 'telegram' }
    )

    console.log(`[HTTP] model=${data.model} response=${data.response?.slice(0, 150)}`)

    expect(status).toBe(200)
    expect(data.success).toBe(true)
    expect(data.status).toBe('completed')
    assertConfiguredModelIdentity(data)
  }, 30_000)

  it('should invoke native tools (system_info)', async () => {
    const nonce = Math.random().toString(36).slice(2, 8)
    const { status, data } = await sendMessage(
      `E2E ${nonce}: What is today's date? Use your system_info tool.`,
      { channelId: GROUP_CHAT_ID, userId: 'e2e-test', channelType: 'telegram' }
    )

    console.log(`[HTTP] tool response=${data.response?.slice(0, 200)}`)

    expect(status).toBe(200)
    expect(data.success).toBe(true)
    expect(data.response).toMatch(/2026/)
  }, 60_000)

  it('should list MCP server tools', async () => {
    const nonce = Math.random().toString(36).slice(2, 8)
    const { status, data } = await sendMessage(
      `E2E ${nonce}: List your available MCP server tools (just tool names, grouped by server).`,
      { channelId: GROUP_CHAT_ID, userId: 'e2e-test', channelType: 'telegram' }
    )

    console.log(`[HTTP] tools=${data.response?.slice(0, 300)}`)

    expect(status).toBe(200)
    const text = (data.response || '').toLowerCase()
    expect(text.includes('airtable') || text.includes('mongodb')).toBe(true)
  }, 90_000)

  it('should handle multi-turn context', async () => {
    const nonce = Math.random().toString(36).slice(2, 8)
    const userId = `e2e-ctx-${nonce}`

    await sendMessage(`E2E ${nonce}: Remember this code: ALPHA-${nonce}`, {
      channelId: GROUP_CHAT_ID,
      userId,
      channelType: 'telegram',
    })

    const { data } = await sendMessage(`E2E ${nonce}: What code did I just ask you to remember?`, {
      channelId: GROUP_CHAT_ID,
      userId,
      channelType: 'telegram',
    })

    console.log(`[HTTP] context recall=${data.response?.slice(0, 200)}`)

    expect(data.success).toBe(true)
    expect(data.response?.toUpperCase()).toContain(`ALPHA-${nonce}`.toUpperCase())
  }, 60_000)
})

// ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
// Suite 2: Benchmark with TG delivery (automated, posts to group)
// ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━

describe.skipIf(!TEST_BOT_TOKEN)('Benchmark E2E with TG delivery', () => {
  beforeAll(async () => {
    await postToGroup(
      '🚀 *Clerum E2E Benchmark Started*\n' +
        `${benchmarkModelLabel()}\n` +
        `Time: ${new Date().toISOString()}`
    )
  })

  it('model identity', async () => {
    const { status, data } = await benchmarkQuestion(
      'Model Identity',
      'What model and provider are you running? Answer in one line.'
    )
    expect(status).toBe(200)
    expect(data.success).toBe(true)
    assertConfiguredModelIdentity(data)
  }, 30_000)

  it('native tool — system_info', async () => {
    const { data } = await benchmarkQuestion(
      'Native Tool: system_info',
      "What is today's date and time? Use your system_info tool."
    )
    expect(data.success).toBe(true)
    expect(data.response).toMatch(/2026/)
  }, 60_000)

  it('native tool — json_transform', async () => {
    const { data } = await benchmarkQuestion(
      'Native Tool: json_transform',
      'Use json_transform to extract the "name" field from this JSON: {"name":"Clerum","version":"0.6.0","type":"agent"}. Return just the result.'
    )
    expect(data.success).toBe(true)
    expect(data.response).toContain('Clerum')
  }, 60_000)

  it('MCP tool — airtable list_bases', async () => {
    const { data } = await benchmarkQuestion(
      'MCP Tool: Airtable',
      'Use the airtable-server list_bases tool to show available Airtable bases. Summarize what you find.',
      { needsApproval: true }
    )
    expect(data.success).toBe(true)
    expect(data.response).toBeTruthy()
    expect(data.response!.length).toBeGreaterThan(20)
  }, 120_000)

  it('MCP tool — mongodb list_collections', async () => {
    const { data } = await benchmarkQuestion(
      'MCP Tool: MongoDB',
      'Use the mongodb-server tools to list collections in the database. Summarize what you find.',
      { needsApproval: true }
    )
    expect(data.success).toBe(true)
    expect(data.response).toBeTruthy()
    expect(data.response!.length).toBeGreaterThan(20)
  }, 120_000)

  it('multi-turn memory', async () => {
    const nonce = Math.random().toString(36).slice(2, 8)
    const userId = `e2e-mem-${nonce}`

    await sendMessage(`Remember this secret: BENCHMARK-${nonce}`, {
      channelId: GROUP_CHAT_ID,
      userId,
      channelType: 'telegram',
    })

    const { data } = await benchmarkQuestion(
      'Multi-turn Memory',
      'What secret did I just ask you to remember?',
      { userId }
    )

    expect(data.success).toBe(true)
    expect(data.response?.toUpperCase()).toContain(`BENCHMARK-${nonce}`.toUpperCase())
  }, 60_000)

  it('reasoning — simple math', async () => {
    const { data } = await benchmarkQuestion(
      'Reasoning: Math',
      'What is 17 * 23 + 42? Show just the final number.'
    )
    expect(data.success).toBe(true)
    expect(data.response).toContain('433')
  }, 30_000)

  it('instruction following — format constraint', async () => {
    const { data } = await benchmarkQuestion(
      'Instruction Following',
      'List exactly 3 colors, one per line, with no other text. No bullet points, no numbering.'
    )
    expect(data.success).toBe(true)
    const lines = data
      .response!.trim()
      .split('\n')
      .filter((l: string) => l.trim())
    expect(lines.length).toBe(3)
  }, 30_000)

  afterAll(async () => {
    await postToGroup('✅ *Clerum E2E Benchmark Complete*\n' + `Time: ${new Date().toISOString()}`)
  })
})

// ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
// Suite 3: Advanced Multi-Tool Benchmark — Landing Page Generation
// ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━

describe.skipIf(!TEST_BOT_TOKEN)('Advanced Multi-Tool Benchmark — Landing Page', () => {
  const nonce = Math.random().toString(36).slice(2, 8)
  const LANDING_USER_ID = `e2e-landing-${nonce}`

  beforeAll(async () => {
    await postToGroup(
      '🏗️ *Advanced Multi-Tool Benchmark — Landing Page Generation Started*\n' +
        `User: ${LANDING_USER_ID}\n` +
        `Time: ${new Date().toISOString()}`
    )
  })

  it('1. Research phase — gather architecture info', async () => {
    const { data } = await benchmarkQuestion(
      'LP1: Research',
      "Use system_info to get today's date, then use memory_search to find any existing notes about Clerum's architecture. Summarize what you know about Clerum's services, CRDs, and capabilities.",
      { userId: LANDING_USER_ID }
    )
    expect(data.success).toBe(true)
    const text = (data.response || '').toLowerCase()
    expect(
      text.includes('host') ||
        text.includes('context') ||
        text.includes('mcp') ||
        text.includes('channel')
    ).toBe(true)
  }, 90_000)

  it('2. Data gathering — query connected services', async () => {
    const { data } = await benchmarkQuestion(
      'LP2: Data Gathering',
      'Use airtable-server list_bases to see our Airtable setup, and mongodb-server to check available databases. Summarize the external services Clerum currently connects to.',
      { userId: LANDING_USER_ID, needsApproval: true }
    )
    expect(data.success).toBe(true)
    const text = (data.response || '').toLowerCase()
    expect(text.includes('airtable') || text.includes('mongodb')).toBe(true)
  }, 120_000)

  it('3. Content planning — create landing page outline', async () => {
    const { data } = await benchmarkQuestion(
      'LP3: Content Outline',
      'Based on everything you know about Clerum (Kubernetes-native LLM agent platform, multi-channel Telegram/Email/Slack, MCP tool integration, multi-provider OpenAI/Claude/Bailian, CRD-driven config, approval gates), create a detailed landing page outline with sections: Hero, Features, Architecture, How It Works, Use Cases, Quick Start. Write the outline to a file called landing-page-outline.md',
      { userId: LANDING_USER_ID }
    )
    expect(data.success).toBe(true)
    const text = (data.response || '').toLowerCase()
    expect(text.includes('hero') || text.includes('feature') || text.includes('outline')).toBe(true)
  }, 90_000)

  it('4. Generate hero section HTML', async () => {
    const { data } = await benchmarkQuestion(
      'LP4: Hero Section',
      "Now create the landing page HTML. Start with a modern, dark-themed hero section with: headline 'Clerum — Kubernetes-Native AI Agent Platform', subheadline about multi-channel + MCP tools, and a CTA button. Use inline CSS. Write to workspace/landing-page.html",
      { userId: LANDING_USER_ID }
    )
    expect(data.success).toBe(true)
    const text = (data.response || '').toLowerCase()
    expect(text.includes('html') || text.includes('hero') || text.includes('written')).toBe(true)
  }, 90_000)

  it('5. Generate features section', async () => {
    const { data } = await benchmarkQuestion(
      'LP5: Features Section',
      'Read the landing-page.html file you just created, then append a Features section with 6 feature cards: Multi-Channel (Telegram/Email/Slack), LLM Provider Agnostic, MCP Tool Integration, Kubernetes CRDs, Approval Gates, Dev Mode. Use a responsive grid layout with inline CSS. Write the updated file.',
      { userId: LANDING_USER_ID }
    )
    expect(data.success).toBe(true)
    const text = (data.response || '').toLowerCase()
    expect(text.includes('feature') || text.includes('card') || text.includes('updated')).toBe(true)
  }, 120_000)

  it('6. Generate architecture diagram section', async () => {
    const { data } = await benchmarkQuestion(
      'LP6: Architecture Diagram',
      'Read landing-page.html and append an Architecture section. Include an ASCII/text-based flow diagram showing: User → channel-reader → mcp-host → MCP Servers, with CRDs as config inputs. Style it with a code-block appearance. Write the updated file.',
      { userId: LANDING_USER_ID }
    )
    expect(data.success).toBe(true)
    const text = (data.response || '').toLowerCase()
    expect(
      text.includes('architecture') || text.includes('diagram') || text.includes('updated')
    ).toBe(true)
  }, 120_000)

  it('7. Add dynamic data from MCP tools', async () => {
    const { data } = await benchmarkQuestion(
      'LP7: Dynamic MCP Data',
      "Query airtable-server for available bases and mongodb-server for databases. Then read landing-page.html and append a 'Connected Services' section showing the real services Clerum is connected to right now. Write the updated file.",
      { userId: LANDING_USER_ID, needsApproval: true }
    )
    expect(data.success).toBe(true)
    const text = (data.response || '').toLowerCase()
    expect(text.includes('service') || text.includes('connected') || text.includes('updated')).toBe(
      true
    )
  }, 180_000)

  it('8. Final review — read and summarize', async () => {
    const { data } = await benchmarkQuestion(
      'LP8: Final Review',
      'Read the complete landing-page.html file and give me a summary of what we built: how many sections, total approximate line count, and list each section title. Also use system_info to timestamp when this was generated.',
      { userId: LANDING_USER_ID }
    )
    expect(data.success).toBe(true)
    const text = (data.response || '').toLowerCase()
    expect(text.includes('section') || text.includes('line') || text.includes('landing')).toBe(true)
  }, 90_000)

  afterAll(async () => {
    await postToGroup(
      '✅ *Advanced Multi-Tool Benchmark — Landing Page Generation Complete*\n' +
        `Time: ${new Date().toISOString()}`
    )
  })
})

// ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
// Suite 4: TG manual trigger (optional)
// ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━

describe.skipIf(!MANUAL_TRIGGER)('TG delivery E2E (channel-reader log verification)', () => {
  it(
    'should pick up a TG message, process it, and reply',
    async () => {
      const baselineLogs = getReaderLogs()
      const baselineReplyCount = (baselineLogs.match(/Reply sent successfully/g) || []).length

      console.log('[TG-E2E] Send a message in the TG group NOW')
      console.log(`[TG-E2E] Watching channel-reader logs for up to ${POLL_TIMEOUT / 1000}s...`)
      console.log(`[TG-E2E] Baseline: ${baselineReplyCount} replies already in logs`)

      const start = Date.now()
      while (Date.now() - start < POLL_TIMEOUT) {
        const logs = getReaderLogs()
        const currentReplyCount = (logs.match(/Reply sent successfully/g) || []).length

        if (currentReplyCount > baselineReplyCount) {
          console.log('[TG-E2E] New reply detected in logs!')

          const lastReceiveIdx = logs.lastIndexOf('[Telegram] Message received')
          const tail = logs.slice(lastReceiveIdx)

          const hasReceive = /\[Telegram\] Message received.*sender: \d+/.test(tail)
          const hasForward = tail.includes('Message processed successfully')
          const hasReply = tail.includes('Reply sent successfully')

          console.log(`[TG-E2E] Received: ${hasReceive}`)
          console.log(`[TG-E2E] Forwarded: ${hasForward}`)
          console.log(`[TG-E2E] Reply sent: ${hasReply}`)

          expect(hasReceive).toBe(true)
          expect(hasForward).toBe(true)
          expect(hasReply).toBe(true)
          return
        }

        await sleep(5000)
      }
      throw new Error(`No new reply in channel-reader logs within ${POLL_TIMEOUT}ms`)
    },
    POLL_TIMEOUT + 10_000
  )
})