Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
18.8 KB
Raw Blame
/**
 * E2E: Software Creation — validate the Clerum agent can create, build, and
 * test software end-to-end through multi-step tool-use chains.
 *
 * Tests SC-1 through SC-11 from docs/archive/testing/E2E-SOFTWARE-CREATION-PLAN.md.
 * Requires a REAL LLM API key (OpenAI or Claude) — these tests make actual
 * LLM calls and exercise multi-step tool orchestration.
 *
 * Prerequisites:
 *   1. Minikube running with mcp-host deployed (image with /workspace dir)
 *   2. Real OPENAI_API_KEY (or CLAUDE_API_KEY) in chatllm-api-keys secret
 *   3. ConfigMap tuned for software creation:
 *        CLERUM_ENABLE_APPROVAL=false
 *        CLERUM_AGENT_MAX_TOOL_CALLS=200
 *        CLERUM_AGENT_MAX_TASK_DURATION=1200000
 *        CLERUM_SHELL_TIMEOUT=120000
 *   4. Port-forwarding active:
 *        kubectl port-forward -n mcp-host svc/chatllm 8080:8080
 *   5. Auth-chain synchronized:
 *        make minikube-sync-auth-key
 *        CONTEXT=clerum-test [email protected] scripts/minikube/seed-test-data.sh
 *
 * Run:
 *   cd tests/e2e && npx vitest run software-creation.test.ts
 */
import { afterAll, describe as baseDescribe, beforeAll, expect, it } from 'vitest'
import {
  getRpcHostStatus,
  getRpcTaskResult,
  issueRealRpcToken,
  kubectlExecInChatllm,
  sendRpcHostMessage,
  waitForRpcHostIdle,
} from './runtimeAuth.js'

// ---------------------------------------------------------------------------
// Test-wide state
// ---------------------------------------------------------------------------

/** Track files/directories created during tests for cleanup. */
const cleanupPaths: string[] = []
let authToken = ''
const RUN_SOFTWARE_CREATION = /^(1|true|yes)$/i.test(process.env.E2E_RUN_SOFTWARE_CREATION ?? '')
const describe = baseDescribe.skipIf(!RUN_SOFTWARE_CREATION)

// ---------------------------------------------------------------------------
// Helpers
// ---------------------------------------------------------------------------

/** Auto-incrementing counter to give each test a unique userId (isolated conversation). */
let testCounter = 0

/**
 * Send a message and wait for the agent to finish, handling the hardcoded
 * 5-minute HTTP timeout in the /message endpoint.
 *
 * Each call uses a unique userId to prevent conversation context accumulation
 * from prior tests confusing the LLM.
 *
 * Returns the response text if available, or null if the HTTP timed out
 * (the agent may still be completing in the background).
 */
async function sendAndWait(
  content: string,
  timeoutMs: number = 300_000
): Promise<{ response: string | null; timedOut: boolean }> {
  testCounter++
  const userId = `sc-test-${testCounter}`

  const res = await sendRpcHostMessage(authToken, content, {
    userId,
    async: true,
  })
  expect(res.status).toBe(200)

  if (typeof res.data?.taskId === 'string' && res.data.taskId.length > 0) {
    const result = await getRpcTaskResult(authToken, res.data.taskId, timeoutMs)
    await waitForRpcHostIdle(authToken, 10_000).catch(() => {})
    return {
      response: typeof result.data?.response === 'string' ? result.data.response : null,
      timedOut: false,
    }
  }

  if (typeof res.data?.response === 'string') {
    await waitForRpcHostIdle(authToken, 10_000).catch(() => {})
    return { response: res.data.response, timedOut: false }
  }

  await waitForRpcHostIdle(authToken, timeoutMs)
  return { response: res.data?.response || null, timedOut: true }
}

/**
 * Execute a shell command inside the mcp-host container.
 * Returns stdout. Throws if the command fails.
 */
function kubectlExec(cmd: string): string {
  return kubectlExecInChatllm(cmd)
}

/**
 * Check if a file or directory exists in the container workspace.
 */
function fileExists(path: string): boolean {
  try {
    kubectlExec(`test -e /workspace/${path} && echo EXISTS`)
    return true
  } catch {
    return false
  }
}

/**
 * Read file contents from the container workspace.
 */
function readFileInContainer(path: string): string {
  return kubectlExec(`cat /workspace/${path}`)
}

function isExecutableInContainer(path: string): boolean {
  try {
    kubectlExec(`test -x /workspace/${path} && echo EXECUTABLE`)
    return true
  } catch {
    return false
  }
}

function runNodeFile(path: string): string {
  return kubectlExec(`cd /workspace && node ${path} 2>&1`)
}

function expectWorkspaceEntry(path: string, response: string | null): void {
  expect(
    fileExists(path),
    [
      `Expected agent to create /workspace/${path}, but it was missing.`,
      "This suite validates real artifact creation inside chatllm, not only the model's natural-language claim.",
      `Agent response: ${response ?? '<empty>'}`,
    ].join('\n\n')
  ).toBe(true)
}

beforeAll(async () => {
  if (!RUN_SOFTWARE_CREATION) return

  // These tests are intentionally strict: they verify the real user -> rpc token
  // -> rpc-proxy -> mcp-host path, not a bypass token or a mocked host.
  authToken = await issueRealRpcToken(
    [
      'host:message:invoke',
      'host:status:read',
      'host:task:read',
      'host:activity:read',
      'host:approval:write',
    ],
    ['chatllm']
  )

  const status = await getRpcHostStatus(authToken)
  expect(status).toBeTruthy()
  await waitForRpcHostIdle(authToken, 10_000).catch(() => {})
}, 60_000)

function normalizeLines(output: string): string[] {
  return output
    .split(/\r?\n/)
    .map(line => line.trim())
    .filter(Boolean)
}

function countTextStats(text: string): {
  lines: number
  words: number
  characters: number
} {
  const normalized = text.replace(/\r\n/g, '\n')
  const lines = normalized.length === 0 ? 0 : normalized.split('\n').length
  const words = normalized.trim().length === 0 ? 0 : normalized.trim().split(/\s+/).length
  return {
    lines,
    words,
    characters: normalized.length,
  }
}

// ---------------------------------------------------------------------------
// Phase 1: Single File Creation
// ---------------------------------------------------------------------------

describe('Phase 1: Single File Creation', () => {
  it('SC-1: shell script — create, chmod, execute', async () => {
    cleanupPaths.push('hello.sh')

    const { response } = await sendAndWait(
      "Create a shell script called hello.sh that prints 'Hello Clerum' and the current date. Make it executable and run it. Tell me the output.",
      300_000
    )

    // Verify file was created
    expectWorkspaceEntry('hello.sh', response)
    expect(isExecutableInContainer('hello.sh')).toBe(true)

    const output = kubectlExec('cd /workspace && ./hello.sh')
    expect(output.toLowerCase()).toContain('hello clerum')

    if (response) {
      const lower = response.toLowerCase()
      expect(lower).toContain('hello clerum')
    }

    const status = await getRpcHostStatus(authToken)
    expect(status.agent.state).toBe('idle')
  }, 360_000)

  it('SC-2: Node.js FizzBuzz — code generation and execution', async () => {
    cleanupPaths.push('fizzbuzz.js')

    const { response } = await sendAndWait(
      'Write a Node.js script called fizzbuzz.js that prints FizzBuzz for numbers 1 to 20, then run it with node and tell me the output.',
      300_000
    )

    expectWorkspaceEntry('fizzbuzz.js', response)

    const output = normalizeLines(runNodeFile('fizzbuzz.js'))
    expect(output).toEqual([
      '1',
      '2',
      'Fizz',
      '4',
      'Buzz',
      'Fizz',
      '7',
      '8',
      'Fizz',
      'Buzz',
      '11',
      'Fizz',
      '13',
      '14',
      'FizzBuzz',
      '16',
      '17',
      'Fizz',
      '19',
      'Buzz',
    ])

    if (response) {
      const responseTokens =
        response
          .match(/\d+|FizzBuzz|Fizz|Buzz/gi)
          ?.map(token =>
            /^\d+$/.test(token) ? token : token[0].toUpperCase() + token.slice(1).toLowerCase()
          ) ?? []
      expect(responseTokens.slice(0, output.length)).toEqual(output)
    }

    const status = await getRpcHostStatus(authToken)
    expect(status.agent.state).toBe('idle')
  }, 360_000)

  it('SC-3: Node.js factorial — compute and report', async () => {
    cleanupPaths.push('calc.js')

    const { response } = await sendAndWait(
      'Create a Node.js script called calc.js that computes the factorial of 10 and prints the result. Run it and tell me the answer.',
      300_000
    )

    expectWorkspaceEntry('calc.js', response)

    const output = runNodeFile('calc.js').trim()
    expect(output).toBe('3628800')

    if (response) {
      expect(response).toContain('3628800')
    }

    const status = await getRpcHostStatus(authToken)
    expect(status.agent.state).toBe('idle')
  }, 360_000)
})

// ---------------------------------------------------------------------------
// Phase 2: Multi-File Projects
// ---------------------------------------------------------------------------

describe('Phase 2: Multi-File Projects', () => {
  it('SC-4: Node.js module — multi-file with require', async () => {
    cleanupPaths.push('mathlib')

    const { response } = await sendAndWait(
      "Use the file_write tool to create these files, then use shell_exec to run the code: 1) Use file_write to create mathlib/utils.js that exports functions 'add(a,b)' and 'multiply(a,b)'. 2) Use file_write to create mathlib/index.js that requires both functions from ./utils.js and prints add(3,4) and multiply(5,6). 3) Use shell_exec to run 'node mathlib/index.js' and tell me the results.",
      300_000
    )

    expectWorkspaceEntry('mathlib/utils.js', response)
    expectWorkspaceEntry('mathlib/index.js', response)

    const output = normalizeLines(kubectlExec('cd /workspace && node mathlib/index.js'))
    expect(output).toEqual(['7', '30'])

    if (response) {
      const responseNumbers = response.match(/\b\d+\b/g)?.map(Number) ?? []
      expect(responseNumbers).toEqual(expect.arrayContaining([7, 30]))
    }

    const status = await getRpcHostStatus(authToken)
    expect(status.agent.state).toBe('idle')
  }, 360_000)

  it('SC-5: static site — HTML + CSS + JS in a folder', async () => {
    cleanupPaths.push('site')

    const { response } = await sendAndWait(
      "Use the file_write tool to create all three files: 1) Use file_write to create site/index.html — an HTML page with title 'Clerum Test Site', a link to style.css, a script tag for app.js, and a div with id 'output'. 2) Use file_write to create site/style.css — styles the body with a blue background and white text. 3) Use file_write to create site/app.js — sets document.getElementById('output').innerText to 'Site loaded successfully'. After creating all files, use shell_exec to run 'ls site/' and confirm they exist.",
      300_000
    )

    // Verify all three files were created
    expectWorkspaceEntry('site/index.html', response)
    expectWorkspaceEntry('site/style.css', response)
    expectWorkspaceEntry('site/app.js', response)

    // Verify HTML content references CSS and JS
    const html = readFileInContainer('site/index.html')
    expect(html).toContain('style.css')
    expect(html).toContain('app.js')
    expect(html.toLowerCase()).toContain('clerum test site')

    // Verify CSS has blue-ish background
    const css = readFileInContainer('site/style.css')
    expect(css.toLowerCase()).toMatch(/blue|#0000ff|rgb\(0/)

    const status = await getRpcHostStatus(authToken)
    expect(status.agent.state).toBe('idle')
  }, 360_000)
})

// ---------------------------------------------------------------------------
// Phase 3: Build & Test
// ---------------------------------------------------------------------------

describe('Phase 3: Build & Test', () => {
  it('SC-6: npm project — init, write code, run tests', async () => {
    cleanupPaths.push('string-utils')

    const { response } = await sendAndWait(
      "Do these steps using the tools: 1) Use file_write to create string-utils/package.json with content '{\"name\":\"string-utils\",\"version\":\"1.0.0\"}'. 2) Use file_write to create string-utils/src/index.js that exports a function 'capitalize' which uppercases the first letter of a string (module.exports = { capitalize }). 3) Use file_write to create string-utils/test/test.js that requires('../src/index.js') and tests capitalize with 3 cases using console.assert ('hello' → 'Hello', 'world' → 'World', '' → ''). Print 'All tests passed' at the end if no assertions fail. 4) Use shell_exec with command 'node' and args ['string-utils/test/test.js'] to run the tests. Tell me if they pass.",
      600_000
    )

    // Verify project structure
    expectWorkspaceEntry('string-utils/package.json', response)
    expectWorkspaceEntry('string-utils/src/index.js', response)
    expectWorkspaceEntry('string-utils/test/test.js', response)

    const output = kubectlExec('node /workspace/string-utils/test/test.js 2>&1')
    expect(output).toContain('All tests passed')
    expect(output.toLowerCase()).not.toContain('assertionerror')

    if (response) {
      expect(response).toContain('All tests passed')
      expect(response.toLowerCase()).not.toContain('assertionerror')
    }

    const status = await getRpcHostStatus(authToken)
    expect(status.agent.state).toBe('idle')
  }, 660_000)
})

// ---------------------------------------------------------------------------
// Phase 4: Error Recovery
// ---------------------------------------------------------------------------

describe('Phase 4: Error Recovery', () => {
  it('SC-8: syntax error — detect and fix autonomously', async () => {
    cleanupPaths.push('broken.js')

    const { response } = await sendAndWait(
      'Create a file called broken.js with this exact content: \'function greet(name { return "Hello " + name; } console.log(greet("World"));\'. Run it with node. If there\'s an error, fix the file and run it again. Tell me the final output.',
      600_000
    )

    if (response) {
      const lower = response.toLowerCase()
      expect(lower).toContain('hello world')
    }

    const output = kubectlExec('cd /workspace && node broken.js 2>&1')
    expect(output).toContain('Hello World')

    const status = await getRpcHostStatus(authToken)
    expect(status.agent.state).toBe('idle')
  }, 660_000)

  it('SC-9: fetch API — handle errors and adapt', async () => {
    cleanupPaths.push('fetch-test.js')

    const { response } = await sendAndWait(
      "Use file_write to create a Node.js script called fetch-test.js that uses the built-in fetch API to GET https://httpbin.org/get and prints the response status code. Then use shell_exec to run 'node fetch-test.js' and tell me the result. If there are any errors, fix the file with file_write and run it again.",
      600_000
    )

    if (response) {
      expect(response).toMatch(/\b200\b/)
    }

    expectWorkspaceEntry('fetch-test.js', response)

    const output = runNodeFile('fetch-test.js')
    expect(output).toMatch(/\b200\b/)

    const status = await getRpcHostStatus(authToken)
    expect(status.agent.state).toBe('idle')
  }, 660_000)
})

// ---------------------------------------------------------------------------
// Phase 5: Complex Projects
// ---------------------------------------------------------------------------

describe('Phase 5: Complex Projects', () => {
  it('SC-10: CLI word counter — multi-file with argument parsing', async () => {
    cleanupPaths.push('wordcount', 'sample.txt')

    const { response } = await sendAndWait(
      "Use the tools to build this: 1) Use file_write to create wordcount/index.js — a Node.js script that reads a filename from process.argv[2], reads the file with fs.readFileSync, counts lines, words, and characters, and prints 'Lines: N, Words: N, Characters: N'. 2) Use file_write to create sample.txt with at least 3 lines of text. 3) Use shell_exec to run 'node wordcount/index.js sample.txt' and tell me the counts.",
      600_000
    )

    expectWorkspaceEntry('wordcount/index.js', response)
    expectWorkspaceEntry('sample.txt', response)

    const sample = readFileInContainer('sample.txt')
    const expected = countTextStats(sample)
    const output = kubectlExec('cd /workspace && node wordcount/index.js sample.txt 2>&1')
    const lowerOutput = output.toLowerCase()
    expect(lowerOutput).toMatch(new RegExp(`lines?[:\\s]+${expected.lines}\\b`))
    expect(lowerOutput).toMatch(new RegExp(`words?[:\\s]+${expected.words}\\b`))
    expect(lowerOutput).toMatch(new RegExp(`characters?[:\\s]+${expected.characters}\\b`))

    if (response) {
      const lower = response.toLowerCase()
      expect(lower).toMatch(new RegExp(`lines?[:\\s]+${expected.lines}\\b`))
      expect(lower).toMatch(new RegExp(`words?[:\\s]+${expected.words}\\b`))
      expect(lower).toMatch(new RegExp(`characters?[:\\s]+${expected.characters}\\b`))
    }

    const status = await getRpcHostStatus(authToken)
    expect(status.agent.state).toBe('idle')
  }, 660_000)

  it('SC-11: JSON pipeline — write data, process, verify output', async () => {
    cleanupPaths.push('users.json', 'filter.js', 'active-users.json')

    const { response } = await sendAndWait(
      'Create a data processing pipeline: 1) Write a JSON file called users.json with: {"users":[{"name":"Alice","age":30,"active":true},{"name":"Bob","age":25,"active":false},{"name":"Charlie","age":35,"active":true}]}. 2) Create a Node.js script called filter.js that reads users.json, filters only active users, and writes the result to active-users.json. 3) Run filter.js. 4) Read active-users.json and tell me how many active users there are and their names.',
      600_000
    )

    // Verify files were created
    expectWorkspaceEntry('users.json', response)
    expectWorkspaceEntry('filter.js', response)
    expectWorkspaceEntry('active-users.json', response)

    // Verify active-users.json content directly
    const activeUsersRaw = readFileInContainer('active-users.json')
    const parsed = JSON.parse(activeUsersRaw)
    // Structure may vary: array or {users: [...]} or {activeUsers: [...]}
    const users = Array.isArray(parsed) ? parsed : parsed.users || parsed.activeUsers || []
    expect(users.length).toBe(2)

    const names = users.map((u: any) => u.name?.toLowerCase())
    expect(names).toContain('alice')
    expect(names).toContain('charlie')
    expect(names).not.toContain('bob')

    if (response) {
      const lower = response.toLowerCase()
      expect(lower).toContain('alice')
      expect(lower).toContain('charlie')
      expect(lower).not.toContain('bob')
      expect(lower).toMatch(/\b2\b|two/)
    }

    const status = await getRpcHostStatus(authToken)
    expect(status.agent.state).toBe('idle')
  }, 660_000)
})

// ---------------------------------------------------------------------------
// Cleanup
// ---------------------------------------------------------------------------

afterAll(async () => {
  if (!RUN_SOFTWARE_CREATION) return

  if (cleanupPaths.length > 0) {
    const pathList = cleanupPaths.map(p => `/workspace/${p}`).join(' ')
    try {
      kubectlExec(`rm -rf ${pathList}`)
      console.log(`[Cleanup] Removed: ${pathList}`)
    } catch (e) {
      console.log(`[Cleanup] Warning: could not remove paths: ${e}`)
    }
  }
})