SigNoz is an open-source, OpenTelemetry-native observability platform for your team and their AI agents. Get logs, metrics, and traces in one tool with features like APM, distributed tracing, log management, infra monitoring, etc. Combined with SigNoz MCP and a native AI teammate (in SigNoz Cloud) it helps you build more resilient apps.
TypeScript apmapplication-monitoringdistributed-tracinggogood-first-issuejaegerloglogsmetricsmonitoringnextjsobservabilityopen-sourceopentelemetryprometheusreactreactjsself-hostedtracingtypescript
feat: add e2e for llm o11y (#12235)
Gaurav Tewari35d1869314b1
diff --git a/frontend/src/container/LLMObservability/AttributeMapping/TestTab/__tests__/TestTab.test.tsx b/frontend/src/container/LLMObservability/AttributeMapping/TestTab/__tests__/TestTab.test.tsxnew file mode 100644index 0000000000..03c9e7c512--- /dev/null+++ b/frontend/src/container/LLMObservability/AttributeMapping/TestTab/__tests__/TestTab.test.tsx@@ -0,0 +1,171 @@+import { rest, server } from 'mocks-server/server';+import get from 'api/browser/localstorage/get';+import remove from 'api/browser/localstorage/remove';+import set from 'api/browser/localstorage/set';+import { LOCALSTORAGE } from 'constants/localStorage';+import { useAuthZ } from 'lib/authz/hooks/useAuthZ/useAuthZ';+import { mockUseAuthZGrantAll } from 'lib/authz/utils/authz-test-utils';+import { render, screen, userEvent, waitFor } from 'tests/test-utils';++jest.mock('@monaco-editor/react', () => ({+ __esModule: true,+ default: ({+ value,+ onChange,+ }: {+ value: string;+ onChange: (next?: string) => void;+ }): JSX.Element => (+ <textarea+ aria-label="json-editor"+ data-testid="monaco"+ value={value}+ onChange={(e): void => onChange(e.target.value)}+ />+ ),+}));++jest.mock('lib/authz/hooks/useAuthZ/useAuthZ');+const mockedUseAuthZ = useAuthZ as jest.MockedFunction<typeof useAuthZ>;++import LLMObservabilityAttributeMapping from '../../LLMObservabilityAttributeMapping';+import { SAMPLE_SPAN_JSON } from '../spanInputStorage';+import {+ GROUPS_ENDPOINT,+ makeGroupsResponse,+ makeTestResponse,+ mockGroups,+ TEST_ENDPOINT,+} from '../../__tests__/fixtures';++const RESULT_SPAN = {+ attributes: {+ 'my_company.llm.input': 'What is quantum computing?',+ 'llm.input_messages': 'What is quantum computing?',+ 'gen_ai.request.model': 'gpt-4',+ 'gen_ai.usage.total_tokens': 1250,+ 'gen_ai.content.completion': 'Quantum computing leverages...',+ 'gen_ai.content.prompt': 'What is quantum computing?',+ },+ resource: {+ 'service.name': 'llm-gateway',+ 'deployment.environment': 'production',+ },+};++const EDITED_SPAN_JSON = `{+ "attributes": {+ "gen_ai.request.model": "claude-opus-5"+ },+ "resource": {+ "service.name": "my-edited-gateway"+ }+}`;++const SPAN_INPUT_KEY = LOCALSTORAGE.LLM_ATTRIBUTE_MAPPING_TEST_SPAN;++describe('TestTab — sample-span flow', () => {+ beforeEach(() => {+ window.history.pushState(null, '', '/');+ remove(SPAN_INPUT_KEY);+ server.use(+ rest.get(GROUPS_ENDPOINT, (_req, res, ctx) =>+ res(ctx.status(200), ctx.json(makeGroupsResponse(mockGroups))),+ ),+ );+ mockedUseAuthZ.mockImplementation(mockUseAuthZGrantAll);+ });++ it('runs the sample span through the mappers and renders the populated result', async () => {+ const user = userEvent.setup({ pointerEventsCheck: 0 });+ server.use(+ rest.post(TEST_ENDPOINT, (_req, res, ctx) =>+ res(ctx.status(200), ctx.json(makeTestResponse([RESULT_SPAN]))),+ ),+ );++ render(<LLMObservabilityAttributeMapping />);++ await user.click(screen.getByRole('tab', { name: 'Test' }));+ const runBtn = await screen.findByTestId('run-test-button');+ expect(screen.getByTestId('test-results-placeholder')).toBeInTheDocument();++ await user.click(runBtn);++ await expect(+ screen.findByTestId('test-results'),+ ).resolves.toBeInTheDocument();+ expect(screen.getByTestId('test-result-0')).toBeInTheDocument();+ expect(screen.getByTestId('test-result-0-attributes')).toHaveTextContent(+ 'gen_ai.content.prompt',+ );+ expect(screen.getByText('populated')).toBeInTheDocument();+ expect(screen.queryByTestId('test-error')).not.toBeInTheDocument();+ });++ it('surfaces a backend error and renders no results', async () => {+ const user = userEvent.setup({ pointerEventsCheck: 0 });+ server.use(+ rest.post(TEST_ENDPOINT, (_req, res, ctx) =>+ res(+ ctx.status(500),+ ctx.json({ error: { message: 'span mapper test failed' } }),+ ),+ ),+ );++ render(<LLMObservabilityAttributeMapping />);++ await user.click(screen.getByRole('tab', { name: 'Test' }));+ await user.click(await screen.findByTestId('run-test-button'));++ await expect(screen.findByTestId('test-error')).resolves.toHaveTextContent(+ 'span mapper test failed',+ );+ expect(screen.queryByTestId('test-results')).not.toBeInTheDocument();+ });++ it('persists an edited span to local storage and restores it on remount', async () => {+ const user = userEvent.setup({ pointerEventsCheck: 0 });+ const { unmount } = render(<LLMObservabilityAttributeMapping />);++ await user.click(screen.getByRole('tab', { name: 'Test' }));+ await screen.findByTestId('run-test-button');++ const editor = screen.getByTestId('monaco');+ expect(editor).toHaveValue(SAMPLE_SPAN_JSON);++ await user.clear(editor);+ await user.paste(EDITED_SPAN_JSON);++ await waitFor(() => expect(get(SPAN_INPUT_KEY)).toBe(EDITED_SPAN_JSON), {+ timeout: 2000,+ });++ unmount();+ render(<LLMObservabilityAttributeMapping />);++ await user.click(screen.getByRole('tab', { name: 'Test' }));+ await screen.findByTestId('run-test-button');+ expect(screen.getByTestId('monaco')).toHaveValue(EDITED_SPAN_JSON);+ });++ it('resets to the sample span and clears the persisted input', async () => {+ const user = userEvent.setup({ pointerEventsCheck: 0 });+ set(SPAN_INPUT_KEY, EDITED_SPAN_JSON);++ render(<LLMObservabilityAttributeMapping />);++ await user.click(screen.getByRole('tab', { name: 'Test' }));+ const resetBtn = await screen.findByTestId('reset-template-button');++ expect(screen.getByTestId('monaco')).toHaveValue(EDITED_SPAN_JSON);+ expect(resetBtn).toBeEnabled();++ await user.click(resetBtn);++ expect(screen.getByTestId('monaco')).toHaveValue(SAMPLE_SPAN_JSON);+ expect(get(SPAN_INPUT_KEY)).toBeFalsy();+ expect(resetBtn).toBeDisabled();+ });+});diff --git a/frontend/src/container/LLMObservability/AttributeMapping/__tests__/fixtures.ts b/frontend/src/container/LLMObservability/AttributeMapping/__tests__/fixtures.tsindex ff71b11ae0..cc17c14cc2 100644--- a/frontend/src/container/LLMObservability/AttributeMapping/__tests__/fixtures.ts+++ b/frontend/src/container/LLMObservability/AttributeMapping/__tests__/fixtures.ts@@ -3,6 +3,7 @@ import {SpantypesSpanMapperDTO as Mapper,SpantypesSpanMapperGroupDTO as MapperGroup,SpantypesSpanMapperOperationDTO as MapperOperation,+ SpantypesSpanMapperTestSpanDTO as TestSpan,} from 'api/generated/services/sigNoz.schemas';// Endpoint globs used by MSW handlers. The generated client hits relative@@ -12,6 +13,7 @@ export const GROUPS_ENDPOINT = '*/api/v1/span_mapper_groups';export function mappersEndpoint(groupId: string): string {return `*/api/v1/span_mapper_groups/${groupId}/span_mappers`;}+export const TEST_ENDPOINT = '*/api/v1/span_mapper_groups/test';export function makeGroup(overrides: Partial<MapperGroup> = {}): MapperGroup {return {@@ -71,6 +73,13 @@ export function makeMappersResponse(mappers: Mapper[]): {return { status: 'ok', data: { items: mappers } };}+export function makeTestResponse(spans: TestSpan[]): {+ status: string;+ data: { spans: TestSpan[] };+} {+ return { status: 'ok', data: { spans } };+}+export const mockGroups: MapperGroup[] = [makeGroup({id: 'group-1',diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.pynew file mode 100644index 0000000000..2e1028573e--- /dev/null+++ b/tests/e2e/conftest.py@@ -0,0 +1,39 @@+# TODO: remove this file once enable_ai_observability flag is removed or defaulted to True+import pytest+from testcontainers.core.container import Network++from fixtures import types+from fixtures.signoz import create_signoz++[email protected](name="signoz", scope="package")+def signoz_e2e( # pylint: disable=too-many-arguments,too-many-positional-arguments+ network: Network,+ zeus: types.TestContainerDocker,+ gateway: types.TestContainerDocker,+ sqlstore: types.TestContainerSQL,+ clickhouse: types.TestContainerClickhouse,+ request: pytest.FixtureRequest,+ pytestconfig: pytest.Config,+) -> types.SigNoz:+ """+ E2E-scoped SigNoz override. Enables the experimental AI/LLM Observability+ module (disabled by default in pkg/flagger/registry.go) so its routes render+ instead of redirecting to /home — required by the llm-o11y e2e specs. Scoped+ to the e2e package via this conftest so normal integration tests keep the+ stock feature set. Follows the same pattern as+ tests/integration/tests/metricreduction/conftest.py.+ """+ return create_signoz(+ network=network,+ zeus=zeus,+ gateway=gateway,+ sqlstore=sqlstore,+ clickhouse=clickhouse,+ request=request,+ pytestconfig=pytestconfig,+ cache_key="signoz_e2e",+ env_overrides={+ "SIGNOZ_FLAGGER_CONFIG_BOOLEAN_ENABLE__AI__OBSERVABILITY": True,+ },+ )diff --git a/tests/e2e/helpers/llm-o11y/attribute-mapping.ts b/tests/e2e/helpers/llm-o11y/attribute-mapping.tsnew file mode 100644index 0000000000..e91c07f716--- /dev/null+++ b/tests/e2e/helpers/llm-o11y/attribute-mapping.ts@@ -0,0 +1,53 @@+import type { APIRequestContext, Page } from '@playwright/test';++export const ATTRIBUTE_MAPPING_PATH = '/ai-observability/attribute-mapping';+export const ATTRIBUTE_MAPPING_PAGE_TESTID =+ 'llm-observability-attribute-mapping-page';++/** Navigate to the Attribute Mapping page and wait for its root container. */+export async function gotoAttributeMapping(page: Page): Promise<void> {+ await page.goto(ATTRIBUTE_MAPPING_PATH);+ await page+ .getByTestId(ATTRIBUTE_MAPPING_PAGE_TESTID)+ .waitFor({ state: 'visible' });+}++interface SpanMapperGroup {+ id: string;+ name: string;+}++/** List span-mapper groups via API (`GET /api/v1/span_mapper_groups`). */+async function listSpanMapperGroups(+ request: APIRequestContext,+ token: string,+): Promise<SpanMapperGroup[]> {+ const res = await request.get('/api/v1/span_mapper_groups', {+ headers: { Authorization: `Bearer ${token}` },+ });+ if (!res.ok()) {+ return [];+ }+ const body = (await res.json()) as { data?: { items?: SpanMapperGroup[] } };+ return body.data?.items ?? [];+}++export async function deleteSpanMapperGroupsByName(+ request: APIRequestContext,+ token: string,+ names: string[],+): Promise<void> {+ const wanted = new Set(names);+ const groups = await listSpanMapperGroups(request, token).catch(() => []);+ await Promise.all(+ groups+ .filter((group) => wanted.has(group.name))+ .map((group) =>+ request+ .delete(`/api/v1/span_mapper_groups/${group.id}`, {+ headers: { Authorization: `Bearer ${token}` },+ })+ .catch(() => undefined),+ ),+ );+}diff --git a/tests/e2e/helpers/llm-o11y/llm-pricing.ts b/tests/e2e/helpers/llm-o11y/llm-pricing.tsnew file mode 100644index 0000000000..961d79eca8--- /dev/null+++ b/tests/e2e/helpers/llm-o11y/llm-pricing.ts@@ -0,0 +1,52 @@+import type { APIRequestContext, Page } from '@playwright/test';++export const LLM_PRICING_PATH = '/ai-observability/configuration';+export const LLM_PRICING_PAGE_TESTID = 'llm-observability-model-pricing-page';++export async function gotoLlmPricing(page: Page): Promise<void> {+ await page.goto(LLM_PRICING_PATH);+ await page.getByTestId(LLM_PRICING_PAGE_TESTID).waitFor({ state: 'visible' });+}++interface PricingRule {+ id: string;+ modelName: string;+}++/** List pricing rules via API (`GET /api/v1/llm_pricing_rules`). */+async function listPricingRules(+ request: APIRequestContext,+ token: string,+): Promise<PricingRule[]> {+ const res = await request.get('/api/v1/llm_pricing_rules', {+ params: { offset: 0, limit: 100 },+ headers: { Authorization: `Bearer ${token}` },+ });+ if (!res.ok()) {+ return [];+ }+ const body = (await res.json()) as {+ data?: { items?: PricingRule[] | null };+ };+ return body.data?.items ?? [];+}++export async function deletePricingRulesByModelName(+ request: APIRequestContext,+ token: string,+ modelNames: string[],+): Promise<void> {+ const wanted = new Set(modelNames);+ const rules = await listPricingRules(request, token).catch(() => []);+ await Promise.all(+ rules+ .filter((rule) => wanted.has(rule.modelName))+ .map((rule) =>+ request+ .delete(`/api/v1/llm_pricing_rules/${rule.id}`, {+ headers: { Authorization: `Bearer ${token}` },+ })+ .catch(() => undefined),+ ),+ );+}diff --git a/tests/e2e/tests/llm-o11y/attribute-mapping.spec.ts b/tests/e2e/tests/llm-o11y/attribute-mapping.spec.tsnew file mode 100644index 0000000000..fa212a02d6--- /dev/null+++ b/tests/e2e/tests/llm-o11y/attribute-mapping.spec.ts@@ -0,0 +1,94 @@+import { expect, test } from '../../fixtures/auth';+import { newAdminContext } from '../../helpers/auth';+import {+ deleteSpanMapperGroupsByName,+ gotoAttributeMapping,+} from '../../helpers/llm-o11y/attribute-mapping';+import { authToken } from '../../helpers/common';++test.describe.configure({ mode: 'serial' });++const GROUP_NAME = 'e2e-attr-mapping-happy';++const TARGET_ATTR = 'gen_ai.content.prompt';+const SOURCE_ATTR = 'my_company.llm.input';++test.afterAll(async ({ browser }) => {+ const ctx = await newAdminContext(browser);+ const page = await ctx.newPage();+ try {+ const token = await authToken(page);+ await deleteSpanMapperGroupsByName(ctx.request, token, [GROUP_NAME]);+ } finally {+ await ctx.close();+ }+});++test.describe('LLM Observability — Attribute Mapping', () => {+ test('basic flow: add a mapping, test it, then save it to the backend', async ({+ authedPage: page,+ }) => {+ await gotoAttributeMapping(page);++ await page.getByTestId('add-group-row').click();+ const groupDrawer = page.getByTestId('group-form-drawer');+ await expect(groupDrawer).toBeVisible();+ await page.getByTestId('group-form-name').fill(GROUP_NAME);+ await page.getByTestId('group-form-save').click();+ await expect(groupDrawer).toBeHidden();++ const groupHeader = page+ .locator('[data-testid^="group-expand-"]')+ .filter({ hasText: GROUP_NAME });+ await expect(groupHeader).toBeVisible();+ await groupHeader.click();++ await page.locator('[data-testid^="add-mapper-"]').click();+ const mapperDrawer = page.getByTestId('mapper-form-drawer');+ await expect(mapperDrawer).toBeVisible();++ await page.getByTestId('mapper-form-target').fill(TARGET_ATTR);+ await page.getByTestId('mapper-form-source-0').fill(SOURCE_ATTR);++ const mapperSave = page.getByTestId('mapper-form-save');+ await expect(mapperSave).toBeEnabled();+ await mapperSave.click();+ await expect(mapperDrawer).toBeHidden();++ await expect(page.getByTestId('unsaved-changes')).toBeVisible();++ await page.getByRole('tab', { name: 'Test' }).click();+ await expect(page.getByTestId('test-tab')).toBeVisible();++ const testResponse = page.waitForResponse(+ (r) =>+ new URL(r.url()).pathname.endsWith('/span_mapper_groups/test') &&+ r.request().method() === 'POST',+ );+ await page.getByTestId('run-test-button').click();+ expect((await testResponse).ok()).toBeTruthy();++ await expect(page.getByTestId('test-error')).toHaveCount(0);+ await expect(page.getByTestId('test-results')).toBeVisible();++ const createGroup = page.waitForResponse(+ (r) =>+ new URL(r.url()).pathname.endsWith('/span_mapper_groups') &&+ r.request().method() === 'POST',+ );+ const createMapper = page.waitForResponse(+ (r) =>+ /\/span_mapper_groups\/[^/]+\/span_mappers$/.test(+ new URL(r.url()).pathname,+ ) && r.request().method() === 'POST',+ );++ await page.getByTestId('save-changes-btn').click();++ expect((await createGroup).ok()).toBeTruthy();+ expect((await createMapper).ok()).toBeTruthy();++ await expect(page.getByText('Attribute mapping changes saved')).toBeVisible();+ await expect(page.getByTestId('unsaved-changes')).toHaveCount(0);+ });+});diff --git a/tests/e2e/tests/llm-o11y/llm-pricing.spec.ts b/tests/e2e/tests/llm-o11y/llm-pricing.spec.tsnew file mode 100644index 0000000000..17566ce6c8--- /dev/null+++ b/tests/e2e/tests/llm-o11y/llm-pricing.spec.ts@@ -0,0 +1,58 @@+import { expect, test } from '../../fixtures/auth';+import { newAdminContext } from '../../helpers/auth';+import { authToken } from '../../helpers/common';+import {+ deletePricingRulesByModelName,+ gotoLlmPricing,+} from '../../helpers/llm-o11y/llm-pricing';++test.describe.configure({ mode: 'serial' });++const MODEL_NAME = 'e2e-pricing-happy-model';+const INPUT_COST = '5.50';+const OUTPUT_COST = '17.00';++test.afterAll(async ({ browser }) => {+ const ctx = await newAdminContext(browser);+ const page = await ctx.newPage();+ try {+ const token = await authToken(page);+ await deletePricingRulesByModelName(ctx.request, token, [MODEL_NAME]);+ } finally {+ await ctx.close();+ }+});++test.describe('LLM Observability — Model Pricing', () => {+ test('basic flow: add a user-override model cost and save it to the backend', async ({+ authedPage: page,+ }) => {+ await gotoLlmPricing(page);++ await page.getByTestId('add-model-cost-btn').click();+ await expect(page.getByTestId('drawer-model-id-input')).toBeVisible();++ await page.getByTestId('drawer-model-id-input').fill(MODEL_NAME);+ await page.getByTestId('drawer-input-cost').fill(INPUT_COST);+ await page.getByTestId('drawer-output-cost').fill(OUTPUT_COST);++ const putResponse = page.waitForResponse(+ (r) =>+ new URL(r.url()).pathname.endsWith('/llm_pricing_rules') &&+ r.request().method() === 'PUT',+ );++ const saveBtn = page.getByTestId('drawer-save-btn');+ await expect(saveBtn).toBeEnabled();+ await saveBtn.click();+ expect((await putResponse).ok()).toBeTruthy();++ await expect(page.getByTestId('drawer-model-id-input')).toHaveCount(0);+ await expect(page.getByTestId('model-costs-table')).toBeVisible();+ await expect(+ page+ .locator('[data-testid^="model-cell-name-"]')+ .filter({ hasText: MODEL_NAME }),+ ).toBeVisible();+ });+});