Skip to content

SigNoz is an open-source, OpenTelemetry-native observability platform for your team and their AI agents. Get logs, metrics, and traces in one tool with features like APM, distributed tracing, log management, infra monitoring, etc. Combined with SigNoz MCP and a native AI teammate (in SigNoz Cloud) it helps you build more resilient apps.

feat(promote): refactor path promotion API to support traces as well (#12839)

Nikhil Soniba3ca5645e9b
diff --git a/docs/api/openapi.yml b/docs/api/openapi.yml
index e38b0ebf2b..c728c2447c 100644
--- a/docs/api/openapi.yml
+++ b/docs/api/openapi.yml
@@ -7756,6 +7756,8 @@ components:
type: object
PromotetypesPromotePath:
properties:
+ context:
+ type: string
indexes:
items:
$ref: '#/components/schemas/PromotetypesWrappedIndex'
@@ -7764,6 +7766,12 @@ components:
type: string
promote:
type: boolean
+ signal:
+ type: string
+ required:
+ - signal
+ - context
+ - path
type: object
PromotetypesWrappedIndex:
properties:
@@ -13432,11 +13440,11 @@ paths:
tags:
- llmpricingrules
x-signoz-stability: alpha
- /api/v1/logs/promote_paths:
+ /api/v1/org/preferences:
get:
deprecated: false
- description: This endpoints promotes and indexes paths
- operationId: ListPromotedAndIndexedPaths
+ description: This endpoint lists all org preferences
+ operationId: ListOrgPreferences
responses:
"200":
content:
@@ -13445,8 +13453,7 @@ paths:
properties:
data:
items:
- $ref: '#/components/schemas/PromotetypesPromotePath'
- nullable: true
+ $ref: '#/components/schemas/PreferencetypesPreference'
type: array
status:
type: string
@@ -13455,12 +13462,6 @@ paths:
- data
type: object
description: OK
- "400":
- content:
- application/json:
- schema:
- $ref: '#/components/schemas/RenderErrorResponse'
- description: Bad Request
"401":
content:
application/json:
@@ -13481,28 +13482,39 @@ paths:
description: Internal Server Error
security:
- api_key:
- - VIEWER
+ - ADMIN
- tokenizer:
- - VIEWER
- summary: Promote and index paths
+ - ADMIN
+ summary: List org preferences
tags:
- - logs
+ - preferences
x-signoz-stability: alpha
- post:
+ /api/v1/org/preferences/{name}:
+ get:
deprecated: false
- description: This endpoints promotes and indexes paths
- operationId: HandlePromoteAndIndexPaths
- requestBody:
- content:
- application/json:
- schema:
- items:
- $ref: '#/components/schemas/PromotetypesPromotePath'
- nullable: true
- type: array
+ description: This endpoint returns the org preference by name
+ operationId: GetOrgPreference
+ parameters:
+ - in: path
+ name: name
+ required: true
+ schema:
+ type: string
responses:
- "201":
- description: Created
+ "200":
+ content:
+ application/json:
+ schema:
+ properties:
+ data:
+ $ref: '#/components/schemas/PreferencetypesPreference'
+ status:
+ type: string
+ required:
+ - status
+ - data
+ type: object
+ description: OK
"400":
content:
application/json:
@@ -13521,6 +13533,12 @@ paths:
schema:
$ref: '#/components/schemas/RenderErrorResponse'
description: Forbidden
+ "404":
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/RenderErrorResponse'
+ description: Not Found
"500":
content:
application/json:
@@ -13529,35 +13547,37 @@ paths:
description: Internal Server Error
security:
- api_key:
- - EDITOR
+ - ADMIN
- tokenizer:
- - EDITOR
- summary: Promote and index paths
+ - ADMIN
+ summary: Get org preference
tags:
- - logs
+ - preferences
x-signoz-stability: alpha
- /api/v1/org/preferences:
- get:
+ put:
deprecated: false
- description: This endpoint lists all org preferences
- operationId: ListOrgPreferences
+ description: This endpoint updates the org preference by name
+ operationId: UpdateOrgPreference
+ parameters:
+ - in: path
+ name: name
+ required: true
+ schema:
+ type: string
+ requestBody:
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/PreferencetypesUpdatablePreference'
responses:
- "200":
+ "204":
+ description: No Content
+ "400":
content:
application/json:
schema:
- properties:
- data:
- items:
- $ref: '#/components/schemas/PreferencetypesPreference'
- type: array
- status:
- type: string
- required:
- - status
- - data
- type: object
- description: OK
+ $ref: '#/components/schemas/RenderErrorResponse'
+ description: Bad Request
"401":
content:
application/json:
@@ -13570,6 +13590,12 @@ paths:
schema:
$ref: '#/components/schemas/RenderErrorResponse'
description: Forbidden
+ "404":
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/RenderErrorResponse'
+ description: Not Found
"500":
content:
application/json:
@@ -13581,21 +13607,36 @@ paths:
- ADMIN
- tokenizer:
- ADMIN
- summary: List org preferences
+ summary: Update org preference
tags:
- preferences
x-signoz-stability: alpha
- /api/v1/org/preferences/{name}:
+ /api/v1/promoted_paths:
get:
deprecated: false
- description: This endpoint returns the org preference by name
- operationId: GetOrgPreference
+ description: This endpoint lists the promoted paths of every JSON column, each
+ annotated with its signal and context. The signal, context, promoted and indexes
+ query parameters filter the listing.
+ operationId: ListPromotedPaths
parameters:
- - in: path
- name: name
- required: true
+ - in: query
+ name: signal
+ schema:
+ type: string
+ - in: query
+ name: context
schema:
type: string
+ - in: query
+ name: promoted
+ schema:
+ nullable: true
+ type: boolean
+ - in: query
+ name: indexes
+ schema:
+ nullable: true
+ type: boolean
responses:
"200":
content:
@@ -13603,7 +13644,10 @@ paths:
schema:
properties:
data:
- $ref: '#/components/schemas/PreferencetypesPreference'
+ items:
+ $ref: '#/components/schemas/PromotetypesPromotePath'
+ nullable: true
+ type: array
status:
type: string
required:
@@ -13629,12 +13673,6 @@ paths:
schema:
$ref: '#/components/schemas/RenderErrorResponse'
description: Forbidden
- "404":
- content:
- application/json:
- schema:
- $ref: '#/components/schemas/RenderErrorResponse'
- description: Not Found
"500":
content:
application/json:
@@ -13643,31 +13681,49 @@ paths:
description: Internal Server Error
security:
- api_key:
- - ADMIN
+ - VIEWER
- tokenizer:
- - ADMIN
- summary: Get org preference
+ - VIEWER
+ summary: List promoted paths
tags:
- - preferences
+ - promote
x-signoz-stability: alpha
- put:
+ post:
deprecated: false
- description: This endpoint updates the org preference by name
- operationId: UpdateOrgPreference
- parameters:
- - in: path
- name: name
- required: true
- schema:
- type: string
+ description: This endpoint promotes paths of JSON columns to their promoted
+ columns. Each path names its promotion domain with its signal and context,
+ e.g. traces/attribute.
+ operationId: PromotePaths
requestBody:
content:
application/json:
+ examples:
+ logs_body:
+ summary: 'Logs body: promote a path and index it'
+ value:
+ - context: body
+ indexes:
+ - fieldDataType: string
+ granularity: 1
+ type: ngrambf_v1(4, 1024, 2, 0)
+ path: body.user.name
+ promote: true
+ signal: logs
+ traces_attribute:
+ summary: 'Traces attribute: promote a path'
+ value:
+ - context: attribute
+ path: http.method
+ promote: true
+ signal: traces
schema:
- $ref: '#/components/schemas/PreferencetypesUpdatablePreference'
+ items:
+ $ref: '#/components/schemas/PromotetypesPromotePath'
+ nullable: true
+ type: array
responses:
- "204":
- description: No Content
+ "201":
+ description: Created
"400":
content:
application/json:
@@ -13686,12 +13742,6 @@ paths:
schema:
$ref: '#/components/schemas/RenderErrorResponse'
description: Forbidden
- "404":
- content:
- application/json:
- schema:
- $ref: '#/components/schemas/RenderErrorResponse'
- description: Not Found
"500":
content:
application/json:
@@ -13700,12 +13750,12 @@ paths:
description: Internal Server Error
security:
- api_key:
- - ADMIN
+ - EDITOR
- tokenizer:
- - ADMIN
- summary: Update org preference
+ - EDITOR
+ summary: Promote paths
tags:
- - preferences
+ - promote
x-signoz-stability: alpha
/api/v1/roles:
get:
diff --git a/frontend/src/api/generated/services/logs/index.ts b/frontend/src/api/generated/services/logs/index.ts
index 5da46696f9..5ebf6740ee 100644
--- a/frontend/src/api/generated/services/logs/index.ts
+++ b/frontend/src/api/generated/services/logs/index.ts
@@ -4,23 +4,15 @@
* * regenerate with 'pnpm generate:api'
* SigNoz
*/
-import { useMutation, useQuery } from 'react-query';
+import { useMutation } from 'react-query';
import type {
- InvalidateOptions,
MutationFunction,
- QueryClient,
- QueryFunction,
- QueryKey,
UseMutationOptions,
UseMutationResult,
- UseQueryOptions,
- UseQueryResult,
} from 'react-query';
import type {
HandleExportRawDataPOSTParams,
- ListPromotedAndIndexedPaths200,
- PromotetypesPromotePathDTO,
Querybuildertypesv5QueryRangeRequestDTO,
RenderErrorResponseDTO,
} from '../sigNoz.schemas';
@@ -28,26 +20,6 @@ import type {
import { GeneratedAPIInstance } from '../../../generatedAPIInstance';
import type { ErrorType, BodyType } from '../../../generatedAPIInstance';
-const withQueryKey = <T extends object, K>(
- query: T,
- queryKey: K,
-): T & { queryKey: K } => {
- const result = { queryKey } as T & { queryKey: K };
- for (const key of Object.keys(query)) {
- // The explicit queryKey always wins, matching the previous
- // `{ ...query, queryKey }` spread where it was set last.
- if (key === 'queryKey') {
- continue;
- }
- Object.defineProperty(result, key, {
- enumerable: true,
- configurable: true,
- get: () => (query as Record<string, unknown>)[key],
- });
- }
- return result;
-};
-
/**
* This endpoints allows complex query exporting raw data for traces and logs
* @summary Export raw data
@@ -149,175 +121,3 @@ export const useHandleExportRawDataPOST = <
> => {
return useMutation(getHandleExportRawDataPOSTMutationOptions(options));
};
-/**
- * This endpoints promotes and indexes paths
- * @summary Promote and index paths
- */
-export const listPromotedAndIndexedPaths = (signal?: AbortSignal) => {
- return GeneratedAPIInstance<ListPromotedAndIndexedPaths200>({
- url: `/api/v1/logs/promote_paths`,
- method: 'GET',
- signal,
- });
-};
-
-export const getListPromotedAndIndexedPathsQueryKey = () => {
- return [`/api/v1/logs/promote_paths`] as const;
-};
-
-export const getListPromotedAndIndexedPathsQueryOptions = <
- TData = Awaited<ReturnType<typeof listPromotedAndIndexedPaths>>,
- TError = ErrorType<RenderErrorResponseDTO>,
->(options?: {
- query?: UseQueryOptions<
- Awaited<ReturnType<typeof listPromotedAndIndexedPaths>>,
- TError,
- TData
- >;
-}) => {
- const { query: queryOptions } = options ?? {};
-
- const queryKey =
- queryOptions?.queryKey ?? getListPromotedAndIndexedPathsQueryKey();
-
- const queryFn: QueryFunction<
- Awaited<ReturnType<typeof listPromotedAndIndexedPaths>>
- > = ({ signal }) => listPromotedAndIndexedPaths(signal);
-
- return { queryKey, queryFn, ...queryOptions } as UseQueryOptions<
- Awaited<ReturnType<typeof listPromotedAndIndexedPaths>>,
- TError,
- TData
- > & { queryKey: QueryKey };
-};
-
-export type ListPromotedAndIndexedPathsQueryResult = NonNullable<
- Awaited<ReturnType<typeof listPromotedAndIndexedPaths>>
->;
-export type ListPromotedAndIndexedPathsQueryError =
- ErrorType<RenderErrorResponseDTO>;
-
-/**
- * @summary Promote and index paths
- */
-
-export function useListPromotedAndIndexedPaths<
- TData = Awaited<ReturnType<typeof listPromotedAndIndexedPaths>>,
- TError = ErrorType<RenderErrorResponseDTO>,
->(options?: {
- query?: UseQueryOptions<
- Awaited<ReturnType<typeof listPromotedAndIndexedPaths>>,
- TError,
- TData
- >;
-}): UseQueryResult<TData, TError> & { queryKey: QueryKey } {
- const queryOptions = getListPromotedAndIndexedPathsQueryOptions(options);
-
- const query = useQuery(queryOptions) as UseQueryResult<TData, TError> & {
- queryKey: QueryKey;
- };
-
- return withQueryKey(query, queryOptions.queryKey);
-}
-
-/**
- * @summary Promote and index paths
- */
-export const invalidateListPromotedAndIndexedPaths = async (
- queryClient: QueryClient,
- options?: InvalidateOptions,
-): Promise<QueryClient> => {
- await queryClient.invalidateQueries(
- { queryKey: getListPromotedAndIndexedPathsQueryKey() },
- options,
- );
-
- return queryClient;
-};
-
-/**
- * This endpoints promotes and indexes paths
- * @summary Promote and index paths
- */
-export const handlePromoteAndIndexPaths = (
- promotetypesPromotePathDTONull?: BodyType<
- PromotetypesPromotePathDTO[] | null
- > | null,
- signal?: AbortSignal,
-) => {
- return GeneratedAPIInstance<void>({
- url: `/api/v1/logs/promote_paths`,
- method: 'POST',
- headers: { 'Content-Type': 'application/json' },
- data: promotetypesPromotePathDTONull,
- signal,
- });
-};
-
-export const getHandlePromoteAndIndexPathsMutationOptions = <
- TError = ErrorType<RenderErrorResponseDTO>,
- TContext = unknown,
->(options?: {
- mutation?: UseMutationOptions<
- Awaited<ReturnType<typeof handlePromoteAndIndexPaths>>,
- TError,
- { data?: BodyType<PromotetypesPromotePathDTO[] | null> },
- TContext
- >;
-}): UseMutationOptions<
- Awaited<ReturnType<typeof handlePromoteAndIndexPaths>>,
- TError,
- { data?: BodyType<PromotetypesPromotePathDTO[] | null> },
- TContext
-> => {
- const mutationKey = ['handlePromoteAndIndexPaths'];
- const { mutation: mutationOptions } = options
- ? options.mutation &&
- 'mutationKey' in options.mutation &&
- options.mutation.mutationKey
- ? options
- : { ...options, mutation: { ...options.mutation, mutationKey } }
- : { mutation: { mutationKey } };
-
- const mutationFn: MutationFunction<
- Awaited<ReturnType<typeof handlePromoteAndIndexPaths>>,
- { data?: BodyType<PromotetypesPromotePathDTO[] | null> }
- > = (props) => {
- const { data } = props ?? {};
-
- return handlePromoteAndIndexPaths(data);
- };
-
- return { mutationFn, ...mutationOptions };
-};
-
-export type HandlePromoteAndIndexPathsMutationResult = NonNullable<
- Awaited<ReturnType<typeof handlePromoteAndIndexPaths>>
->;
-export type HandlePromoteAndIndexPathsMutationBody =
- | BodyType<PromotetypesPromotePathDTO[] | null>
- | undefined;
-export type HandlePromoteAndIndexPathsMutationError =
- ErrorType<RenderErrorResponseDTO>;
-
-/**
- * @summary Promote and index paths
- */
-export const useHandlePromoteAndIndexPaths = <
- TError = ErrorType<RenderErrorResponseDTO>,
- TContext = unknown,
->(options?: {
- mutation?: UseMutationOptions<
- Awaited<ReturnType<typeof handlePromoteAndIndexPaths>>,
- TError,
- { data?: BodyType<PromotetypesPromotePathDTO[] | null> },
- TContext
- >;
-}): UseMutationResult<
- Awaited<ReturnType<typeof handlePromoteAndIndexPaths>>,
- TError,
- { data?: BodyType<PromotetypesPromotePathDTO[] | null> },
- TContext
-> => {
- return useMutation(getHandlePromoteAndIndexPathsMutationOptions(options));
-};
diff --git a/frontend/src/api/generated/services/promote/index.ts b/frontend/src/api/generated/services/promote/index.ts
new file mode 100644
index 0000000000..b934e94b69
--- /dev/null
+++ b/frontend/src/api/generated/services/promote/index.ts
@@ -0,0 +1,232 @@
+/**
+ * ! Do not edit manually
+ * * The file has been auto-generated using Orval for SigNoz
+ * * regenerate with 'pnpm generate:api'
+ * SigNoz
+ */
+import { useMutation, useQuery } from 'react-query';
+import type {
+ InvalidateOptions,
+ MutationFunction,
+ QueryClient,
+ QueryFunction,
+ QueryKey,
+ UseMutationOptions,
+ UseMutationResult,
+ UseQueryOptions,
+ UseQueryResult,
+} from 'react-query';
+
+import type {
+ ListPromotedPaths200,
+ ListPromotedPathsParams,
+ PromotetypesPromotePathDTO,
+ RenderErrorResponseDTO,
+} from '../sigNoz.schemas';
+
+import { GeneratedAPIInstance } from '../../../generatedAPIInstance';
+import type { ErrorType, BodyType } from '../../../generatedAPIInstance';
+
+const withQueryKey = <T extends object, K>(
+ query: T,
+ queryKey: K,
+): T & { queryKey: K } => {
+ const result = { queryKey } as T & { queryKey: K };
+ for (const key of Object.keys(query)) {
+ // The explicit queryKey always wins, matching the previous
+ // `{ ...query, queryKey }` spread where it was set last.
+ if (key === 'queryKey') {
+ continue;
+ }
+ Object.defineProperty(result, key, {
+ enumerable: true,
+ configurable: true,
+ get: () => (query as Record<string, unknown>)[key],
+ });
+ }
+ return result;
+};
+
+/**
+ * This endpoint lists the promoted paths of every JSON column, each annotated with its signal and context. The signal, context, promoted and indexes query parameters filter the listing.
+ * @summary List promoted paths
+ */
+export const listPromotedPaths = (
+ params?: ListPromotedPathsParams,
+ signal?: AbortSignal,
+) => {
+ return GeneratedAPIInstance<ListPromotedPaths200>({
+ url: `/api/v1/promoted_paths`,
+ method: 'GET',
+ params,
+ signal,
+ });
+};
+
+export const getListPromotedPathsQueryKey = (
+ params?: ListPromotedPathsParams,
+) => {
+ return [`/api/v1/promoted_paths`, ...(params ? [params] : [])] as const;
+};
+
+export const getListPromotedPathsQueryOptions = <
+ TData = Awaited<ReturnType<typeof listPromotedPaths>>,
+ TError = ErrorType<RenderErrorResponseDTO>,
+>(
+ params?: ListPromotedPathsParams,
+ options?: {
+ query?: UseQueryOptions<
+ Awaited<ReturnType<typeof listPromotedPaths>>,
+ TError,
+ TData
+ >;
+ },
+) => {
+ const { query: queryOptions } = options ?? {};
+
+ const queryKey =
+ queryOptions?.queryKey ?? getListPromotedPathsQueryKey(params);
+
+ const queryFn: QueryFunction<
+ Awaited<ReturnType<typeof listPromotedPaths>>
+ > = ({ signal }) => listPromotedPaths(params, signal);
+
+ return { queryKey, queryFn, ...queryOptions } as UseQueryOptions<
+ Awaited<ReturnType<typeof listPromotedPaths>>,
+ TError,
+ TData
+ > & { queryKey: QueryKey };
+};
+
+export type ListPromotedPathsQueryResult = NonNullable<
+ Awaited<ReturnType<typeof listPromotedPaths>>
+>;
+export type ListPromotedPathsQueryError = ErrorType<RenderErrorResponseDTO>;
+
+/**
+ * @summary List promoted paths
+ */
+
+export function useListPromotedPaths<
+ TData = Awaited<ReturnType<typeof listPromotedPaths>>,
+ TError = ErrorType<RenderErrorResponseDTO>,
+>(
+ params?: ListPromotedPathsParams,
+ options?: {
+ query?: UseQueryOptions<
+ Awaited<ReturnType<typeof listPromotedPaths>>,
+ TError,
+ TData
+ >;
+ },
+): UseQueryResult<TData, TError> & { queryKey: QueryKey } {
+ const queryOptions = getListPromotedPathsQueryOptions(params, options);
+
+ const query = useQuery(queryOptions) as UseQueryResult<TData, TError> & {
+ queryKey: QueryKey;
+ };
+
+ return withQueryKey(query, queryOptions.queryKey);
+}
+
+/**
+ * @summary List promoted paths
+ */
+export const invalidateListPromotedPaths = async (
+ queryClient: QueryClient,
+ params?: ListPromotedPathsParams,
+ options?: InvalidateOptions,
+): Promise<QueryClient> => {
+ await queryClient.invalidateQueries(
+ { queryKey: getListPromotedPathsQueryKey(params) },
+ options,
+ );
+
+ return queryClient;
+};
+
+/**
+ * This endpoint promotes paths of JSON columns to their promoted columns. Each path names its promotion domain with its signal and context, e.g. traces/attribute.
+ * @summary Promote paths
+ */
+export const promotePaths = (
+ promotetypesPromotePathDTONull?: BodyType<
+ PromotetypesPromotePathDTO[] | null
+ > | null,
+ signal?: AbortSignal,
+) => {
+ return GeneratedAPIInstance<void>({
+ url: `/api/v1/promoted_paths`,
+ method: 'POST',
+ headers: { 'Content-Type': 'application/json' },
+ data: promotetypesPromotePathDTONull,
+ signal,
+ });
+};
+
+export const getPromotePathsMutationOptions = <
+ TError = ErrorType<RenderErrorResponseDTO>,
+ TContext = unknown,
+>(options?: {
+ mutation?: UseMutationOptions<
+ Awaited<ReturnType<typeof promotePaths>>,
+ TError,
+ { data?: BodyType<PromotetypesPromotePathDTO[] | null> },
+ TContext
+ >;
+}): UseMutationOptions<
+ Awaited<ReturnType<typeof promotePaths>>,
+ TError,
+ { data?: BodyType<PromotetypesPromotePathDTO[] | null> },
+ TContext
+> => {
+ const mutationKey = ['promotePaths'];
+ const { mutation: mutationOptions } = options
+ ? options.mutation &&
+ 'mutationKey' in options.mutation &&
+ options.mutation.mutationKey
+ ? options
+ : { ...options, mutation: { ...options.mutation, mutationKey } }
+ : { mutation: { mutationKey } };
+
+ const mutationFn: MutationFunction<
+ Awaited<ReturnType<typeof promotePaths>>,
+ { data?: BodyType<PromotetypesPromotePathDTO[] | null> }
+ > = (props) => {
+ const { data } = props ?? {};
+
+ return promotePaths(data);
+ };
+
+ return { mutationFn, ...mutationOptions };
+};
+
+export type PromotePathsMutationResult = NonNullable<
+ Awaited<ReturnType<typeof promotePaths>>
+>;
+export type PromotePathsMutationBody =
+ | BodyType<PromotetypesPromotePathDTO[] | null>
+ | undefined;
+export type PromotePathsMutationError = ErrorType<RenderErrorResponseDTO>;
+
+/**
+ * @summary Promote paths
+ */
+export const usePromotePaths = <
+ TError = ErrorType<RenderErrorResponseDTO>,
+ TContext = unknown,
+>(options?: {
+ mutation?: UseMutationOptions<
+ Awaited<ReturnType<typeof promotePaths>>,
+ TError,
+ { data?: BodyType<PromotetypesPromotePathDTO[] | null> },
+ TContext
+ >;
+}): UseMutationResult<
+ Awaited<ReturnType<typeof promotePaths>>,
+ TError,
+ { data?: BodyType<PromotetypesPromotePathDTO[] | null> },
+ TContext
+> => {
+ return useMutation(getPromotePathsMutationOptions(options));
+};
diff --git a/frontend/src/api/generated/services/sigNoz.schemas.ts b/frontend/src/api/generated/services/sigNoz.schemas.ts
index 113516cece..fdd95b9335 100644
--- a/frontend/src/api/generated/services/sigNoz.schemas.ts
+++ b/frontend/src/api/generated/services/sigNoz.schemas.ts
@@ -9519,6 +9519,10 @@ export interface PromotetypesWrappedIndexDTO {
}
export interface PromotetypesPromotePathDTO {
+ /**
+ * @type string
+ */
+ context: string;
/**
* @type array
*/
@@ -9526,11 +9530,15 @@ export interface PromotetypesPromotePathDTO {
/**
* @type string
*/
- path?: string;
+ path: string;
/**
* @type boolean
*/
promote?: boolean;
+ /**
+ * @type string
+ */
+ signal: string;
}
export interface Querybuildertypesv5AggregationMetaDTO {
@@ -12834,17 +12842,6 @@ export type ListUnmappedLLMModels200 = {
status: string;
};
-export type ListPromotedAndIndexedPaths200 = {
- /**
- * @type array,null
- */
- data: PromotetypesPromotePathDTO[] | null;
- /**
- * @type string
- */
- status: string;
-};
-
export type ListOrgPreferences200 = {
/**
* @type array
@@ -12870,6 +12867,40 @@ export type GetOrgPreference200 = {
export type UpdateOrgPreferencePathParameters = {
name: string;
};
+export type ListPromotedPathsParams = {
+ /**
+ * @type string
+ * @description undefined
+ */
+ signal?: string;
+ /**
+ * @type string
+ * @description undefined
+ */
+ context?: string;
+ /**
+ * @type boolean,null
+ * @description undefined
+ */
+ promoted?: boolean | null;
+ /**
+ * @type boolean,null
+ * @description undefined
+ */
+ indexes?: boolean | null;
+};
+
+export type ListPromotedPaths200 = {
+ /**
+ * @type array,null
+ */
+ data: PromotetypesPromotePathDTO[] | null;
+ /**
+ * @type string
+ */
+ status: string;
+};
+
export type ListRoles200 = {
/**
* @type array
diff --git a/pkg/apiserver/signozapiserver/promote.go b/pkg/apiserver/signozapiserver/promote.go
index e4b364b272..d94732d89f 100644
--- a/pkg/apiserver/signozapiserver/promote.go
+++ b/pkg/apiserver/signozapiserver/promote.go
@@ -10,13 +10,42 @@ import (
)
func (provider *provider) addPromoteRoutes(router *mux.Router) error {
- if err := router.Handle("/api/v1/logs/promote_paths", handler.New(provider.authzMiddleware.EditAccess(provider.promoteHandler.HandlePromoteAndIndexPaths), handler.OpenAPIDef{
- ID: "HandlePromoteAndIndexPaths",
- Tags: []string{"logs"},
- Summary: "Promote and index paths",
- Description: "This endpoints promotes and indexes paths",
- Request: new([]*promotetypes.PromotePath),
- RequestContentType: "application/json",
+ if err := router.Handle("/api/v1/promoted_paths", handler.New(provider.authzMiddleware.EditAccess(provider.promoteHandler.PromotePaths), handler.OpenAPIDef{
+ ID: "PromotePaths",
+ Tags: []string{"promote"},
+ Summary: "Promote paths",
+ Description: "This endpoint promotes paths of JSON columns to their promoted columns. Each path names its promotion domain with its signal and context, e.g. traces/attribute.",
+ Request: new([]*promotetypes.PromotePath),
+ RequestContentType: "application/json",
+ RequestExamples: []handler.OpenAPIExample{
+ {
+ Name: "logs_body",
+ Summary: "Logs body: promote a path and index it",
+ Value: []map[string]any{
+ {
+ "signal": "logs",
+ "context": "body",
+ "path": "body.user.name",
+ "promote": true,
+ "indexes": []map[string]any{
+ {"fieldDataType": "string", "type": "ngrambf_v1(4, 1024, 2, 0)", "granularity": 1},
+ },
+ },
+ },
+ },
+ {
+ Name: "traces_attribute",
+ Summary: "Traces attribute: promote a path",
+ Value: []map[string]any{
+ {
+ "signal": "traces",
+ "context": "attribute",
+ "path": "http.method",
+ "promote": true,
+ },
+ },
+ },
+ },
Response: nil,
ResponseContentType: "",
SuccessStatusCode: http.StatusCreated,
@@ -26,12 +55,13 @@ func (provider *provider) addPromoteRoutes(router *mux.Router) error {
return err
}
- if err := router.Handle("/api/v1/logs/promote_paths", handler.New(provider.authzMiddleware.ViewAccess(provider.promoteHandler.ListPromotedAndIndexedPaths), handler.OpenAPIDef{
- ID: "ListPromotedAndIndexedPaths",
- Tags: []string{"logs"},
- Summary: "Promote and index paths",
- Description: "This endpoints promotes and indexes paths",
+ if err := router.Handle("/api/v1/promoted_paths", handler.New(provider.authzMiddleware.ViewAccess(provider.promoteHandler.ListPromotedPaths), handler.OpenAPIDef{
+ ID: "ListPromotedPaths",
+ Tags: []string{"promote"},
+ Summary: "List promoted paths",
+ Description: "This endpoint lists the promoted paths of every JSON column, each annotated with its signal and context. The signal, context, promoted and indexes query parameters filter the listing.",
Request: nil,
+ RequestQuery: new(promotetypes.ListPromotedPathsFilters),
RequestContentType: "",
Response: new([]*promotetypes.PromotePath),
ResponseContentType: "",
diff --git a/pkg/modules/promote/implpromote/handler.go b/pkg/modules/promote/implpromote/handler.go
index 19efd78780..ecba7903de 100644
--- a/pkg/modules/promote/implpromote/handler.go
+++ b/pkg/modules/promote/implpromote/handler.go
@@ -19,7 +19,7 @@ func NewHandler(module promote.Module) promote.Handler {
return &handler{module: module}
}
-func (h *handler) HandlePromoteAndIndexPaths(w http.ResponseWriter, r *http.Request) {
+func (h *handler) PromotePaths(w http.ResponseWriter, r *http.Request) {
// TODO(Nitya): Use in multi tenant setup
_, err := authtypes.ClaimsFromContext(r.Context())
if err != nil {
@@ -33,7 +33,7 @@ func (h *handler) HandlePromoteAndIndexPaths(w http.ResponseWriter, r *http.Requ
return
}
- err = h.module.PromoteAndIndexPaths(r.Context(), req...)
+ err = h.module.PromotePaths(r.Context(), req...)
if err != nil {
render.Error(w, err)
return
@@ -42,7 +42,7 @@ func (h *handler) HandlePromoteAndIndexPaths(w http.ResponseWriter, r *http.Requ
render.Success(w, http.StatusCreated, nil)
}
-func (h *handler) ListPromotedAndIndexedPaths(w http.ResponseWriter, r *http.Request) {
+func (h *handler) ListPromotedPaths(w http.ResponseWriter, r *http.Request) {
// TODO(Nitya): Use in multi tenant setup
_, err := authtypes.ClaimsFromContext(r.Context())
if err != nil {
@@ -50,7 +50,17 @@ func (h *handler) ListPromotedAndIndexedPaths(w http.ResponseWriter, r *http.Req
return
}
- paths, err := h.module.ListPromotedAndIndexedPaths(r.Context())
+ var filters promotetypes.ListPromotedPathsFilters
+ if err := binding.Query.BindQuery(r.URL.Query(), &filters); err != nil {
+ render.Error(w, err)
+ return
+ }
+ if err := filters.Validate(); err != nil {
+ render.Error(w, err)
+ return
+ }
+
+ paths, err := h.module.ListPromotedPaths(r.Context(), filters)
if err != nil {
render.Error(w, err)
return
diff --git a/pkg/modules/promote/implpromote/module.go b/pkg/modules/promote/implpromote/module.go
index 4d7590a610..437babf7ad 100644
--- a/pkg/modules/promote/implpromote/module.go
+++ b/pkg/modules/promote/implpromote/module.go
@@ -2,14 +2,11 @@ package implpromote
import (
"context"
- "maps"
- "slices"
"strings"
schemamigrator "github.com/SigNoz/signoz-otel-collector/cmd/signozschemamigrator/schema_migrator"
"github.com/SigNoz/signoz/pkg/errors"
"github.com/SigNoz/signoz/pkg/modules/promote"
- "github.com/SigNoz/signoz/pkg/telemetryschema/logstelemetryschema"
"github.com/SigNoz/signoz/pkg/telemetrystore"
"github.com/SigNoz/signoz/pkg/types/ctxtypes"
"github.com/SigNoz/signoz/pkg/types/instrumentationtypes"
@@ -31,46 +28,77 @@ func NewModule(metadataStore telemetrytypes.MetadataStore, telemetrystore teleme
return &module{metadataStore: metadataStore, telemetryStore: telemetrystore}
}
-func (m *module) ListPromotedAndIndexedPaths(ctx context.Context) ([]promotetypes.PromotePath, error) {
+func (m *module) ListPromotedPaths(ctx context.Context, filters promotetypes.ListPromotedPathsFilters) ([]promotetypes.PromotePath, error) {
+ response := make([]promotetypes.PromotePath, 0)
+ for _, target := range promotetypes.Targets() {
+ if !filters.MatchesTarget(target) {
+ continue
+ }
+ paths, err := m.listPromotedPaths(ctx, target)
+ if err != nil {
+ return nil, err
+ }
+ for _, path := range paths {
+ if filters.MatchesPath(path) {
+ response = append(response, path)
+ }
+ }
+ }
+ return response, nil
+}
+
+func (m *module) listPromotedPaths(ctx context.Context, target promotetypes.Target) ([]promotetypes.PromotePath, error) {
+ promotedPaths, err := m.metadataStore.GetPromotedPaths(ctx, target.Entry)
+ if err != nil {
+ return nil, err
+ }
+
+ response := make([]promotetypes.PromotePath, 0, len(promotedPaths))
+ for path := range promotedPaths {
+ response = append(response, promotetypes.PromotePath{
+ Signal: target.Entry.Signal.StringValue(),
+ Context: target.Entry.FieldContext.StringValue(),
+ Path: target.RequiredPathPrefix + path,
+ Promote: true,
+ })
+ }
+
+ if !target.IndexesSupported {
+ return response, nil
+ }
+
indexes, err := m.metadataStore.ListLogsJSONIndexes(ctx)
if err != nil {
return nil, err
}
+ // aggr keys are full sub-column paths: index.BaseColumn carries the
+ // column prefix and index.Name the bare path.
aggr := map[string][]promotetypes.WrappedIndex{}
for _, index := range indexes {
- aggr[index.Name] = append(aggr[index.Name], promotetypes.WrappedIndex{
+ fullPath := index.BaseColumn + index.Name
+ aggr[fullPath] = append(aggr[fullPath], promotetypes.WrappedIndex{
FieldDataType: index.FieldDataType,
Type: index.IndexType,
Granularity: index.Granularity,
})
}
- promotedPaths, err := m.listPromotedPaths(ctx)
- if err != nil {
- return nil, err
- }
- response := []promotetypes.PromotePath{}
- for _, path := range promotedPaths {
- fullPath := logstelemetryschema.BodyPromotedColumnPrefix + path
- path = telemetrytypes.BodyJSONStringSearchPrefix + path
- item := promotetypes.PromotePath{
- Path: path,
- Promote: true,
- }
- indexes, ok := aggr[fullPath]
- if ok {
- item.Indexes = indexes
+ for i := range response {
+ fullPath := target.PromotedColumnPrefix() + strings.TrimPrefix(response[i].Path, target.RequiredPathPrefix)
+ if indexes, ok := aggr[fullPath]; ok {
+ response[i].Indexes = indexes
delete(aggr, fullPath)
}
- response = append(response, item)
}
- // add the paths that are not promoted but have indexes
- for path, indexes := range aggr {
- path := strings.TrimPrefix(path, logstelemetryschema.BodyV2ColumnPrefix)
- path = telemetrytypes.BodyJSONStringSearchPrefix + path
+ for fullPath, indexes := range aggr {
+ path := strings.TrimPrefix(fullPath, target.BaseColumnPrefix())
+ path = strings.TrimPrefix(path, target.PromotedColumnPrefix())
+ path = target.RequiredPathPrefix + path
response = append(response, promotetypes.PromotePath{
+ Signal: target.Entry.Signal.StringValue(),
+ Context: target.Entry.FieldContext.StringValue(),
Path: path,
Indexes: indexes,
})
@@ -78,27 +106,39 @@ func (m *module) ListPromotedAndIndexedPaths(ctx context.Context) ([]promotetype
return response, nil
}
-func (m *module) listPromotedPaths(ctx context.Context) ([]string, error) {
- paths, err := m.metadataStore.GetPromotedPaths(ctx)
- if err != nil {
- return nil, err
- }
- return slices.Collect(maps.Keys(paths)), nil
-}
-
-// PromotePaths inserts provided JSON paths into the promoted paths table for logs queries.
-func (m *module) PromotePaths(ctx context.Context, paths []string) error {
+func (m *module) PromotePaths(ctx context.Context, paths ...*promotetypes.PromotePath) error {
if len(paths) == 0 {
return errors.NewInvalidInputf(errors.CodeInvalidInput, "paths cannot be empty")
}
- return m.metadataStore.PromotePaths(ctx, paths...)
+ byTarget := map[promotetypes.Target][]*promotetypes.PromotePath{}
+ targets := []promotetypes.Target{}
+ for _, path := range paths {
+ target, err := path.Target()
+ if err != nil {
+ return err
+ }
+ if err := path.ValidateAndSetDefaults(target); err != nil {
+ return err
+ }
+ if _, ok := byTarget[target]; !ok {
+ targets = append(targets, target)
+ }
+ byTarget[target] = append(byTarget[target], path)
+ }
+
+ for _, target := range targets {
+ if err := m.promotePaths(ctx, target, byTarget[target]...); err != nil {
+ return err
+ }
+ }
+ return nil
}
// createIndexes creates string ngram + token filter indexes on JSON path subcolumns for LIKE queries.
-func (m *module) createIndexes(ctx context.Context, indexes []schemamigrator.Index) error {
+func (m *module) createIndexes(ctx context.Context, target promotetypes.Target, indexes []schemamigrator.Index) error {
ctx = ctxtypes.NewContextWithCommentVals(ctx, map[string]string{
- instrumentationtypes.TelemetrySignal: telemetrytypes.SignalLogs.StringValue(),
+ instrumentationtypes.TelemetrySignal: target.Entry.Signal.StringValue(),
instrumentationtypes.CodeNamespace: "promote",
instrumentationtypes.CodeFunctionName: "createIndexes",
})
@@ -108,8 +148,8 @@ func (m *module) createIndexes(ctx context.Context, indexes []schemamigrator.Ind
for _, index := range indexes {
alterStmt := schemamigrator.AlterTableAddIndex{
- Database: logstelemetryschema.DBName,
- Table: logstelemetryschema.LogsV2LocalTableName,
+ Database: target.DBName,
+ Table: target.LocalTableName,
Index: index,
}
op := alterStmt.OnCluster(m.telemetryStore.Cluster())
@@ -121,25 +161,13 @@ func (m *module) createIndexes(ctx context.Context, indexes []schemamigrator.Ind
return nil
}
-// PromoteAndIndexPaths handles promoting paths and creating indexes in one call.
-func (m *module) PromoteAndIndexPaths(
- ctx context.Context,
- paths ...*promotetypes.PromotePath,
-) error {
- if len(paths) == 0 {
- return errors.NewInvalidInputf(errors.CodeInvalidInput, "paths cannot be empty")
- }
-
+func (m *module) promotePaths(ctx context.Context, target promotetypes.Target, paths ...*promotetypes.PromotePath) error {
pathsStr := []string{}
- // validate the paths
for _, path := range paths {
- if err := path.ValidateAndSetDefaults(); err != nil {
- return err
- }
pathsStr = append(pathsStr, path.Path)
}
- existingPromotedPaths, err := m.metadataStore.GetPromotedPaths(ctx, pathsStr...)
+ existingPromotedPaths, err := m.metadataStore.GetPromotedPaths(ctx, target.Entry, pathsStr...)
if err != nil {
return err
}
@@ -153,23 +181,16 @@ func (m *module) PromoteAndIndexPaths(
}
}
if len(it.Indexes) > 0 {
- parentColumn := logstelemetryschema.LogsV2BodyV2Column
+ parentColumn := target.BaseColumn
// if the path is already promoted or is being promoted, add it to the promoted column
if _, promoted := existingPromotedPaths[it.Path]; promoted || it.Promote {
- parentColumn = logstelemetryschema.LogsV2BodyPromotedColumn
+ parentColumn = target.PromotedColumn()
}
for _, index := range it.Indexes {
- var typeIndex schemamigrator.IndexType
- switch {
- case strings.HasPrefix(index.Type, string(schemamigrator.IndexTypeNGramBF)):
- typeIndex = schemamigrator.IndexTypeNGramBF
- case strings.HasPrefix(index.Type, string(schemamigrator.IndexTypeTokenBF)):
- typeIndex = schemamigrator.IndexTypeTokenBF
- case strings.HasPrefix(index.Type, string(schemamigrator.IndexTypeMinMax)):
- typeIndex = schemamigrator.IndexTypeMinMax
- default:
- return errors.NewInvalidInputf(errors.CodeInvalidInput, "invalid index type: %s", index.Type)
+ typeIndex, err := index.SkipIndexType()
+ if err != nil {
+ return err
}
indexes = append(indexes, schemamigrator.Index{
Name: schemamigrator.JSONSubColumnIndexName(parentColumn, it.Path, index.JSONDataType.StringValue(), typeIndex),
@@ -182,14 +203,14 @@ func (m *module) PromoteAndIndexPaths(
}
if len(toInsert) > 0 {
- err := m.PromotePaths(ctx, toInsert)
+ err := m.metadataStore.PromotePaths(ctx, target.Entry, toInsert...)
if err != nil {
return err
}
}
if len(indexes) > 0 {
- if err := m.createIndexes(ctx, indexes); err != nil {
+ if err := m.createIndexes(ctx, target, indexes); err != nil {
return err
}
}
diff --git a/pkg/modules/promote/implpromote/module_test.go b/pkg/modules/promote/implpromote/module_test.go
new file mode 100644
index 0000000000..f74ebf91eb
--- /dev/null
+++ b/pkg/modules/promote/implpromote/module_test.go
@@ -0,0 +1,342 @@
+package implpromote
+
+import (
+ "context"
+ "regexp"
+ "testing"
+
+ sqlmock "github.com/DATA-DOG/go-sqlmock"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+
+ "github.com/SigNoz/signoz/pkg/telemetrystore"
+ "github.com/SigNoz/signoz/pkg/telemetrystore/telemetrystoretest"
+ "github.com/SigNoz/signoz/pkg/types/promotetypes"
+ "github.com/SigNoz/signoz/pkg/types/telemetrytypes"
+ "github.com/SigNoz/signoz/pkg/types/telemetrytypes/telemetrytypestest"
+)
+
+func TestPromotePaths(t *testing.T) {
+ ctx := context.Background()
+
+ testCases := []struct {
+ name string
+ paths []*promotetypes.PromotePath
+ promoteTwice bool
+ wantErr bool
+ wantPromoted map[telemetrytypes.Signal]map[string]bool
+ }{
+ {
+ name: "PromotesNewAttributes_Idempotent",
+ paths: []*promotetypes.PromotePath{
+ {Signal: "traces", Context: "attribute", Path: "http.method", Promote: true},
+ {Signal: "traces", Context: "attribute", Path: "span.operation", Promote: true},
+ },
+ promoteTwice: true,
+ wantPromoted: map[telemetrytypes.Signal]map[string]bool{telemetrytypes.SignalTraces: {"http.method": true, "span.operation": true}},
+ },
+ {
+ name: "MixedDomains_RecordedPerTarget",
+ paths: []*promotetypes.PromotePath{
+ {Signal: "traces", Context: "attribute", Path: "http.method", Promote: true},
+ {Signal: "logs", Context: "body", Path: "body.user.name", Promote: true},
+ },
+ wantPromoted: map[telemetrytypes.Signal]map[string]bool{
+ telemetrytypes.SignalTraces: {"http.method": true},
+ telemetrytypes.SignalLogs: {"user.name": true},
+ },
+ },
+ {
+ name: "NonPromoteEntries_NotRecorded",
+ paths: []*promotetypes.PromotePath{{Signal: "traces", Context: "attribute", Path: "http.method"}},
+ wantPromoted: map[telemetrytypes.Signal]map[string]bool{},
+ },
+ {
+ name: "InvalidIndexTypeInLaterDomain_NothingRecorded",
+ paths: []*promotetypes.PromotePath{
+ {Signal: "traces", Context: "attribute", Path: "http.method", Promote: true},
+ {Signal: "logs", Context: "body", Path: "body.user.name", Promote: true, Indexes: []promotetypes.WrappedIndex{{FieldDataType: telemetrytypes.FieldDataTypeString, Type: "unsupported", Granularity: 1}}},
+ },
+ wantErr: true,
+ },
+ {
+ name: "InvalidSignal_Rejected",
+ paths: []*promotetypes.PromotePath{{Signal: "events", Context: "attribute", Path: "http.method", Promote: true}},
+ wantErr: true,
+ },
+ {
+ name: "UnsupportedDomain_Rejected",
+ paths: []*promotetypes.PromotePath{{Signal: "metrics", Context: "attribute", Path: "http.method", Promote: true}},
+ wantErr: true,
+ },
+ {
+ name: "ColumnPrefixedPath_Rejected",
+ paths: []*promotetypes.PromotePath{{Signal: "traces", Context: "attribute", Path: "attributes.http.method", Promote: true}},
+ wantErr: true,
+ },
+ {
+ name: "EmptyPath_Rejected",
+ paths: []*promotetypes.PromotePath{{Signal: "traces", Context: "attribute", Path: "", Promote: true}},
+ wantErr: true,
+ },
+ {
+ name: "EmptyRequest_Rejected",
+ wantErr: true,
+ },
+ {
+ name: "PromotesBodyPath_PrefixStripped",
+ paths: []*promotetypes.PromotePath{{Signal: "logs", Context: "body", Path: "body.user.name", Promote: true}},
+ wantPromoted: map[telemetrytypes.Signal]map[string]bool{telemetrytypes.SignalLogs: {"user.name": true}},
+ },
+ }
+
+ for _, testCase := range testCases {
+ t.Run(testCase.name, func(t *testing.T) {
+ store := telemetrytypestest.NewMockMetadataStore()
+ m := NewModule(store, nil)
+
+ err := m.PromotePaths(ctx, testCase.paths...)
+ if testCase.wantErr {
+ assert.Error(t, err)
+ assert.Empty(t, store.PromotedPathsMap)
+ return
+ }
+ require.NoError(t, err)
+
+ assert.Equal(t, testCase.wantPromoted, store.PromotedPathsMap)
+
+ if testCase.promoteTwice {
+ require.NoError(t, m.PromotePaths(ctx, testCase.paths...))
+ assert.Equal(t, testCase.wantPromoted, store.PromotedPathsMap)
+ }
+ })
+ }
+}
+
+func TestPromotePathsCreatesIndexes(t *testing.T) {
+ ctx := context.Background()
+
+ testCases := []struct {
+ name string
+ promoted map[string]bool
+ path *promotetypes.PromotePath
+ wantDDLColumn string
+ }{
+ {
+ name: "NewPromotion_IndexesPromotedColumn",
+ path: &promotetypes.PromotePath{
+ Signal: "logs",
+ Context: "body",
+ Path: "body.user.name",
+ Promote: true,
+ Indexes: []promotetypes.WrappedIndex{
+ {FieldDataType: telemetrytypes.FieldDataTypeString, Type: "ngrambf_v1(4, 1024, 2, 0)", Granularity: 1},
+ },
+ },
+ wantDDLColumn: "dynamicElement(body_promoted.user.name",
+ },
+ {
+ name: "AlreadyPromoted_IndexesPromotedColumn",
+ promoted: map[string]bool{"user.name": true},
+ path: &promotetypes.PromotePath{
+ Signal: "logs",
+ Context: "body",
+ Path: "body.user.name",
+ Indexes: []promotetypes.WrappedIndex{
+ {FieldDataType: telemetrytypes.FieldDataTypeString, Type: "ngrambf_v1(4, 1024, 2, 0)", Granularity: 1},
+ },
+ },
+ wantDDLColumn: "dynamicElement(body_promoted.user.name",
+ },
+ {
+ name: "UnpromotedPath_IndexesBaseColumn",
+ path: &promotetypes.PromotePath{
+ Signal: "logs",
+ Context: "body",
+ Path: "body.user.name",
+ Indexes: []promotetypes.WrappedIndex{
+ {FieldDataType: telemetrytypes.FieldDataTypeString, Type: "ngrambf_v1(4, 1024, 2, 0)", Granularity: 1},
+ },
+ },
+ wantDDLColumn: "dynamicElement(body_v2.user.name",
+ },
+ }
+
+ for _, testCase := range testCases {
+ t.Run(testCase.name, func(t *testing.T) {
+ ts := telemetrystoretest.New(telemetrystore.Config{}, sqlmock.QueryMatcherRegexp)
+ store := telemetrytypestest.NewMockMetadataStore()
+ store.PromotedPathsMap[telemetrytypes.SignalLogs] = testCase.promoted
+ m := NewModule(store, ts)
+
+ ts.Mock().ExpectExec("ADD INDEX (.+)" + regexp.QuoteMeta(testCase.wantDDLColumn)).WillReturnError(nil)
+ require.NoError(t, m.PromotePaths(ctx, testCase.path))
+ assert.NoError(t, ts.Mock().ExpectationsWereMet())
+ })
+ }
+}
+
+func TestListPromotedPaths(t *testing.T) {
+ ctx := context.Background()
+ trueValue := true
+ falseValue := false
+
+ testCases := []struct {
+ name string
+ filters promotetypes.ListPromotedPathsFilters
+ promoted map[telemetrytypes.Signal]map[string]bool
+ indexes []telemetrytypes.TelemetryFieldKeySkipIndex
+ wantPaths []promotetypes.PromotePath
+ }{
+ {
+ name: "PromotedPaths_ListedPerDomain",
+ promoted: map[telemetrytypes.Signal]map[string]bool{
+ telemetrytypes.SignalLogs: {"user.name": true},
+ telemetrytypes.SignalTraces: {"http.method": true},
+ },
+ wantPaths: []promotetypes.PromotePath{
+ {Signal: "logs", Context: "body", Path: "body.user.name", Promote: true},
+ {Signal: "traces", Context: "attribute", Path: "http.method", Promote: true},
+ },
+ },
+ {
+ name: "SignalFilter_SkipsOtherDomains",
+ filters: promotetypes.ListPromotedPathsFilters{Signal: "traces"},
+ promoted: map[telemetrytypes.Signal]map[string]bool{
+ telemetrytypes.SignalLogs: {"user.name": true},
+ telemetrytypes.SignalTraces: {"http.method": true},
+ },
+ wantPaths: []promotetypes.PromotePath{{Signal: "traces", Context: "attribute", Path: "http.method", Promote: true}},
+ },
+ {
+ name: "ContextFilter_SkipsOtherDomains",
+ filters: promotetypes.ListPromotedPathsFilters{Context: "body"},
+ promoted: map[telemetrytypes.Signal]map[string]bool{
+ telemetrytypes.SignalLogs: {"user.name": true},
+ telemetrytypes.SignalTraces: {"http.method": true},
+ },
+ wantPaths: []promotetypes.PromotePath{{Signal: "logs", Context: "body", Path: "body.user.name", Promote: true}},
+ },
+ {
+ name: "ContextAliasFilter_MatchesDomain",
+ filters: promotetypes.ListPromotedPathsFilters{Signal: "traces", Context: "tag"},
+ promoted: map[telemetrytypes.Signal]map[string]bool{telemetrytypes.SignalTraces: {"http.method": true}},
+ wantPaths: []promotetypes.PromotePath{{Signal: "traces", Context: "attribute", Path: "http.method", Promote: true}},
+ },
+ {
+ name: "IndexedPaths_MergedForSupportingDomains",
+ promoted: map[telemetrytypes.Signal]map[string]bool{telemetrytypes.SignalLogs: {"user.name": true}},
+ indexes: []telemetrytypes.TelemetryFieldKeySkipIndex{
+ {
+ Name: "user.name",
+ FieldContext: telemetrytypes.FieldContextBody,
+ FieldDataType: telemetrytypes.FieldDataTypeString,
+ BaseColumn: "body_promoted.",
+ IndexType: "ngrambf_v1(4, 1024, 2, 0)",
+ Granularity: 1,
+ },
+ {
+ Name: "request.duration",
+ FieldContext: telemetrytypes.FieldContextBody,
+ FieldDataType: telemetrytypes.FieldDataTypeFloat64,
+ BaseColumn: "body_v2.",
+ IndexType: "minmax",
+ Granularity: 1,
+ },
+ },
+ wantPaths: []promotetypes.PromotePath{
+ {
+ Signal: "logs",
+ Context: "body",
+ Path: "body.user.name",
+ Promote: true,
+ Indexes: []promotetypes.WrappedIndex{
+ {FieldDataType: telemetrytypes.FieldDataTypeString, Type: "ngrambf_v1(4, 1024, 2, 0)", Granularity: 1},
+ },
+ },
+ {
+ Signal: "logs",
+ Context: "body",
+ Path: "body.request.duration",
+ Indexes: []promotetypes.WrappedIndex{
+ {FieldDataType: telemetrytypes.FieldDataTypeFloat64, Type: "minmax", Granularity: 1},
+ },
+ },
+ },
+ },
+ {
+ name: "PromotedFalseFilter_IndexOnlyPaths",
+ filters: promotetypes.ListPromotedPathsFilters{Promoted: &falseValue},
+ promoted: map[telemetrytypes.Signal]map[string]bool{telemetrytypes.SignalLogs: {"user.name": true}},
+ indexes: []telemetrytypes.TelemetryFieldKeySkipIndex{
+ {
+ Name: "request.duration",
+ FieldContext: telemetrytypes.FieldContextBody,
+ FieldDataType: telemetrytypes.FieldDataTypeFloat64,
+ BaseColumn: "body_v2.",
+ IndexType: "minmax",
+ Granularity: 1,
+ },
+ },
+ wantPaths: []promotetypes.PromotePath{
+ {
+ Signal: "logs",
+ Context: "body",
+ Path: "body.request.duration",
+ Indexes: []promotetypes.WrappedIndex{
+ {FieldDataType: telemetrytypes.FieldDataTypeFloat64, Type: "minmax", Granularity: 1},
+ },
+ },
+ },
+ },
+ {
+ name: "IndexesTrueFilter_PathsWithIndexes",
+ filters: promotetypes.ListPromotedPathsFilters{Indexes: &trueValue},
+ promoted: map[telemetrytypes.Signal]map[string]bool{telemetrytypes.SignalLogs: {"user.name": true}},
+ indexes: []telemetrytypes.TelemetryFieldKeySkipIndex{
+ {
+ Name: "user.name",
+ FieldContext: telemetrytypes.FieldContextBody,
+ FieldDataType: telemetrytypes.FieldDataTypeString,
+ BaseColumn: "body_promoted.",
+ IndexType: "ngrambf_v1(4, 1024, 2, 0)",
+ Granularity: 1,
+ },
+ },
+ wantPaths: []promotetypes.PromotePath{
+ {
+ Signal: "logs",
+ Context: "body",
+ Path: "body.user.name",
+ Promote: true,
+ Indexes: []promotetypes.WrappedIndex{
+ {FieldDataType: telemetrytypes.FieldDataTypeString, Type: "ngrambf_v1(4, 1024, 2, 0)", Granularity: 1},
+ },
+ },
+ },
+ },
+ }
+
+ for _, testCase := range testCases {
+ t.Run(testCase.name, func(t *testing.T) {
+ store := telemetrytypestest.NewMockMetadataStore()
+ store.PromotedPathsMap = testCase.promoted
+ store.LogsJSONIndexes = testCase.indexes
+ m := NewModule(store, nil)
+
+ paths, err := m.ListPromotedPaths(ctx, testCase.filters)
+ require.NoError(t, err)
+ require.Len(t, paths, len(testCase.wantPaths))
+
+ byDomainPath := map[string]promotetypes.PromotePath{}
+ for _, path := range paths {
+ byDomainPath[path.Signal+"/"+path.Context+"/"+path.Path] = path
+ }
+ for _, want := range testCase.wantPaths {
+ key := want.Signal + "/" + want.Context + "/" + want.Path
+ require.Contains(t, byDomainPath, key)
+ assert.Equal(t, want, byDomainPath[key])
+ }
+ })
+ }
+}
diff --git a/pkg/modules/promote/promote.go b/pkg/modules/promote/promote.go
index 3146121714..e7d9533de4 100644
--- a/pkg/modules/promote/promote.go
+++ b/pkg/modules/promote/promote.go
@@ -8,11 +8,11 @@ import (
)
type Module interface {
- ListPromotedAndIndexedPaths(ctx context.Context) ([]promotetypes.PromotePath, error)
- PromoteAndIndexPaths(ctx context.Context, paths ...*promotetypes.PromotePath) error
+ ListPromotedPaths(ctx context.Context, filters promotetypes.ListPromotedPathsFilters) ([]promotetypes.PromotePath, error)
+ PromotePaths(ctx context.Context, paths ...*promotetypes.PromotePath) error
}
type Handler interface {
- HandlePromoteAndIndexPaths(w http.ResponseWriter, r *http.Request)
- ListPromotedAndIndexedPaths(w http.ResponseWriter, r *http.Request)
+ PromotePaths(w http.ResponseWriter, r *http.Request)
+ ListPromotedPaths(w http.ResponseWriter, r *http.Request)
}
diff --git a/pkg/telemetrymetadata/body_json_metadata.go b/pkg/telemetrymetadata/body_json_metadata.go
index ff97aa0841..3e82beda16 100644
--- a/pkg/telemetrymetadata/body_json_metadata.go
+++ b/pkg/telemetrymetadata/body_json_metadata.go
@@ -16,6 +16,7 @@ import (
"github.com/SigNoz/signoz/pkg/telemetryschema/logstelemetryschema"
"github.com/SigNoz/signoz/pkg/types/ctxtypes"
"github.com/SigNoz/signoz/pkg/types/instrumentationtypes"
+ "github.com/SigNoz/signoz/pkg/types/promotetypes"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
"github.com/huandu/go-sqlbuilder"
)
@@ -33,6 +34,8 @@ var (
CodeFailedToAppendPath = errors.MustNewCode("failed_to_append_path_promoted_paths")
)
+var logsBodyPromotedEntry = promotetypes.NewLogsBodyTarget().Entry
+
// enrichJSONKeys enriches body-context keys with promoted path info, indexes,
// and JSON access plans. parentTypeCache contains parent array types (ArrayJSON/ArrayDynamic)
// pre-fetched in the main UNION query.
@@ -67,7 +70,7 @@ func (t *telemetryMetaStore) enrichJSONKeys(ctx context.Context, selectors []*te
}
// fetch promoted paths
- promoted, err := t.GetPromotedPaths(ctx, paths...)
+ promoted, err := t.GetPromotedPaths(ctx, logsBodyPromotedEntry, paths...)
if err != nil {
return err
}
@@ -157,7 +160,7 @@ func buildListLogsJSONIndexesQuery(cluster string, filters ...string) (string, [
}
func (t *telemetryMetaStore) ListLogsJSONIndexes(ctx context.Context, filters ...string) ([]telemetrytypes.TelemetryFieldKeySkipIndex, error) {
- ctx = withTelemetryContext(ctx, "ListLogsJSONIndexes")
+ ctx = withTelemetryContext(ctx, telemetrytypes.SignalLogs, "ListLogsJSONIndexes")
query, args := buildListLogsJSONIndexesQuery(t.telemetrystore.Cluster(), filters...)
rows, err := t.telemetrystore.ClickhouseDB().Query(ctx, query, args...)
if err != nil {
@@ -215,14 +218,14 @@ func (t *telemetryMetaStore) ListLogsJSONIndexes(ctx context.Context, filters ..
// TODO(Piyush): Remove this if not used in future.
func (t *telemetryMetaStore) ListJSONValues(ctx context.Context, path string, limit int) (*telemetrytypes.TelemetryFieldValues, bool, error) {
- ctx = withTelemetryContext(ctx, "ListJSONValues")
+ ctx = withTelemetryContext(ctx, telemetrytypes.SignalLogs, "ListJSONValues")
path = CleanPathPrefixes(path)
if strings.Contains(path, telemetrytypes.ArraySep) || strings.Contains(path, telemetrytypes.ArrayAnyIndex) {
return nil, false, errors.NewInvalidInputf(errors.CodeInvalidInput, "array paths are not supported")
}
- promoted, err := t.IsPathPromoted(ctx, path)
+ promoted, err := t.isPathPromoted(ctx, logsBodyPromotedEntry, path)
if err != nil {
return nil, false, err
}
@@ -376,13 +379,12 @@ func derefValue(v any) any {
return val.Interface()
}
-// IsPathPromoted checks if a specific path is promoted (Column Evolution table: field_name for logs body).
-func (t *telemetryMetaStore) IsPathPromoted(ctx context.Context, path string) (bool, error) {
- ctx = withTelemetryContext(ctx, "IsPathPromoted")
+func (t *telemetryMetaStore) isPathPromoted(ctx context.Context, entry telemetrytypes.EvolutionEntry, path string) (bool, error) {
+ ctx = withTelemetryContext(ctx, entry.Signal, "isPathPromoted")
split := strings.Split(path, telemetrytypes.ArraySep)
pathSegment := split[0]
query := fmt.Sprintf("SELECT 1 FROM %s.%s WHERE signal = ? AND column_name = ? AND field_context = ? AND field_name = ? LIMIT 1", DBName, PromotedPathsTableName)
- rows, err := t.telemetrystore.ClickhouseDB().Query(ctx, query, telemetrytypes.SignalLogs, logstelemetryschema.LogsV2BodyPromotedColumn, telemetrytypes.FieldContextBody, pathSegment)
+ rows, err := t.telemetrystore.ClickhouseDB().Query(ctx, query, entry.Signal, entry.ColumnName, entry.FieldContext, pathSegment)
if err != nil {
return false, errors.WrapInternalf(err, CodeFailCheckPathPromoted, "failed to check if path %s is promoted", path)
}
@@ -391,14 +393,13 @@ func (t *telemetryMetaStore) IsPathPromoted(ctx context.Context, path string) (b
return rows.Next(), nil
}
-// GetPromotedPaths returns promoted paths from the Column Evolution table (field_name for logs body).
-func (t *telemetryMetaStore) GetPromotedPaths(ctx context.Context, paths ...string) (map[string]bool, error) {
- ctx = withTelemetryContext(ctx, "GetPromotedPaths")
+func (t *telemetryMetaStore) GetPromotedPaths(ctx context.Context, entry telemetrytypes.EvolutionEntry, paths ...string) (map[string]bool, error) {
+ ctx = withTelemetryContext(ctx, entry.Signal, "GetPromotedPaths")
sb := sqlbuilder.Select("field_name").From(fmt.Sprintf("%s.%s", DBName, PromotedPathsTableName))
conditions := []string{
- sb.Equal("signal", telemetrytypes.SignalLogs),
- sb.Equal("column_name", logstelemetryschema.LogsV2BodyPromotedColumn),
- sb.Equal("field_context", telemetrytypes.FieldContextBody),
+ sb.Equal("signal", entry.Signal),
+ sb.Equal("column_name", entry.ColumnName),
+ sb.Equal("field_context", entry.FieldContext),
sb.NotEqual("field_name", "__all__"),
}
if len(paths) > 0 {
@@ -438,9 +439,8 @@ func CleanPathPrefixes(path string) string {
return path
}
-// PromotePaths inserts promoted paths into the Column Evolution table (same schema as signoz-otel-collector metadata_migrations).
-func (t *telemetryMetaStore) PromotePaths(ctx context.Context, paths ...string) error {
- ctx = withTelemetryContext(ctx, "PromotePaths")
+func (t *telemetryMetaStore) PromotePaths(ctx context.Context, entry telemetrytypes.EvolutionEntry, paths ...string) error {
+ ctx = withTelemetryContext(ctx, entry.Signal, "PromotePaths")
batch, err := t.telemetrystore.ClickhouseDB().PrepareBatch(ctx,
fmt.Sprintf("INSERT INTO %s.%s (signal, column_name, column_type, field_context, field_name, version, release_time) VALUES", DBName,
PromotedPathsTableName))
@@ -454,7 +454,7 @@ func (t *telemetryMetaStore) PromotePaths(ctx context.Context, paths ...string)
if trimmed == "" {
continue
}
- if err := batch.Append(telemetrytypes.SignalLogs, logstelemetryschema.LogsV2BodyPromotedColumn, "JSON()", telemetrytypes.FieldContextBody, trimmed, 0, releaseTime); err != nil {
+ if err := batch.Append(entry.Signal, entry.ColumnName, entry.ColumnType, entry.FieldContext, trimmed, entry.Version, releaseTime); err != nil {
_ = batch.Abort()
return errors.WrapInternalf(err, CodeFailedToAppendPath, "failed to append path")
}
@@ -466,9 +466,9 @@ func (t *telemetryMetaStore) PromotePaths(ctx context.Context, paths ...string)
return nil
}
-func withTelemetryContext(ctx context.Context, functionName string) context.Context {
+func withTelemetryContext(ctx context.Context, signal telemetrytypes.Signal, functionName string) context.Context {
return ctxtypes.NewContextWithCommentVals(ctx, map[string]string{
- instrumentationtypes.TelemetrySignal: telemetrytypes.SignalLogs.StringValue(),
+ instrumentationtypes.TelemetrySignal: signal.StringValue(),
instrumentationtypes.CodeNamespace: "metadata",
instrumentationtypes.CodeFunctionName: functionName,
})
diff --git a/pkg/types/promotetypes/target.go b/pkg/types/promotetypes/target.go
new file mode 100644
index 0000000000..c38b06cd7b
--- /dev/null
+++ b/pkg/types/promotetypes/target.go
@@ -0,0 +1,103 @@
+package promotetypes
+
+import (
+ "github.com/SigNoz/signoz/pkg/errors"
+ "github.com/SigNoz/signoz/pkg/telemetryschema/logstelemetryschema"
+ "github.com/SigNoz/signoz/pkg/telemetryschema/tracestelemetryschema"
+ "github.com/SigNoz/signoz/pkg/types/telemetrytypes"
+)
+
+// Target identifies a promotion domain.
+type Target struct {
+ Entry telemetrytypes.EvolutionEntry // evolution row template; FieldName and ReleaseTime are set per write
+ DBName string // index DDL database, used only when IndexesSupported
+ LocalTableName string // index DDL local table, used only when IndexesSupported
+ BaseColumn string // column holding every path; indexes for unpromoted paths are created on it
+ RequiredPathPrefix string // prefix API paths must carry, stripped before storing; empty for bare names
+ IndexesSupported bool
+}
+
+func (t Target) PromotedColumn() string { return t.Entry.ColumnName }
+
+func (t Target) BaseColumnPrefix() string { return t.BaseColumn + "." }
+
+func (t Target) PromotedColumnPrefix() string { return t.PromotedColumn() + "." }
+
+func NewTarget(entry telemetrytypes.EvolutionEntry, dbName, localTableName, baseColumn, requiredPathPrefix string, indexesSupported bool) Target {
+ return Target{
+ Entry: entry,
+ DBName: dbName,
+ LocalTableName: localTableName,
+ BaseColumn: baseColumn,
+ RequiredPathPrefix: requiredPathPrefix,
+ IndexesSupported: indexesSupported,
+ }
+}
+
+// NewLogsBodyTarget returns the logs body domain (body_v2 -> body_promoted).
+func NewLogsBodyTarget() Target {
+ return NewTarget(
+ telemetrytypes.EvolutionEntry{
+ Signal: telemetrytypes.SignalLogs,
+ ColumnName: logstelemetryschema.LogsV2BodyPromotedColumn,
+ ColumnType: "JSON()",
+ FieldContext: telemetrytypes.FieldContextBody,
+ },
+ logstelemetryschema.DBName,
+ logstelemetryschema.LogsV2LocalTableName,
+ logstelemetryschema.LogsV2BodyV2Column,
+ telemetrytypes.BodyJSONStringSearchPrefix,
+ true,
+ )
+}
+
+// NewTracesAttributesTarget returns the spans attributes domain (attributes
+// -> attributes_promoted).
+func NewTracesAttributesTarget() Target {
+ return NewTarget(
+ telemetrytypes.EvolutionEntry{
+ Signal: telemetrytypes.SignalTraces,
+ ColumnName: tracestelemetryschema.SpanAttributesPromotedColumn,
+ ColumnType: "JSON()",
+ FieldContext: telemetrytypes.FieldContextAttribute,
+ },
+ tracestelemetryschema.DBName,
+ tracestelemetryschema.SpanIndexV3LocalTableName,
+ tracestelemetryschema.SpanAttributesColumn,
+ "",
+ false,
+ )
+}
+
+func NewTargetFromText(signal, context string) (Target, error) {
+ parsedSignal, ok := telemetrytypes.SignalFromText(signal)
+ if !ok {
+ return Target{}, errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "invalid signal: %s", signal)
+ }
+ parsedContext, ok := telemetrytypes.FieldContextFromText(context)
+ if !ok {
+ return Target{}, errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "invalid context: %s", context)
+ }
+ target, ok := TargetFor(parsedSignal, parsedContext)
+ if !ok {
+ return Target{}, errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "promotion is not supported for %s %s", parsedSignal.StringValue(), parsedContext.StringValue())
+ }
+ return target, nil
+}
+
+func Targets() []Target {
+ return []Target{
+ NewLogsBodyTarget(),
+ NewTracesAttributesTarget(),
+ }
+}
+
+func TargetFor(signal telemetrytypes.Signal, context telemetrytypes.FieldContext) (Target, bool) {
+ for _, target := range Targets() {
+ if target.Entry.Signal.StringValue() == signal.StringValue() &&
+ target.Entry.FieldContext.StringValue() == context.StringValue() {
+ return target, true
+ }
+ }
+ return Target{}, false
+}
diff --git a/pkg/types/promotetypes/types.go b/pkg/types/promotetypes/types.go
index e2dca2e439..ad79e0f4d3 100644
--- a/pkg/types/promotetypes/types.go
+++ b/pkg/types/promotetypes/types.go
@@ -3,7 +3,7 @@ package promotetypes
import (
"strings"
- "github.com/SigNoz/signoz-otel-collector/constants"
+ schemamigrator "github.com/SigNoz/signoz-otel-collector/cmd/signozschemamigrator/schema_migrator"
"github.com/SigNoz/signoz-otel-collector/pkg/keycheck"
"github.com/SigNoz/signoz/pkg/errors"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
@@ -17,13 +17,66 @@ type WrappedIndex struct {
}
type PromotePath struct {
- Path string `json:"path"`
+ Signal string `json:"signal" required:"true"`
+ Context string `json:"context" required:"true"`
+ Path string `json:"path" required:"true"`
Promote bool `json:"promote,omitempty"`
Indexes []WrappedIndex `json:"indexes,omitempty"`
}
-func (i *PromotePath) ValidateAndSetDefaults() error {
+func (i *PromotePath) Target() (Target, error) {
+ return NewTargetFromText(i.Signal, i.Context)
+}
+
+type ListPromotedPathsFilters struct {
+ Signal string `query:"signal" json:"signal"`
+ Context string `query:"context" json:"context"`
+ Promoted *bool `query:"promoted" json:"promoted"`
+ Indexes *bool `query:"indexes" json:"indexes"`
+}
+
+// Validate checks the signal and context words are known; the pair need not
+// name a supported domain.
+func (f *ListPromotedPathsFilters) Validate() error {
+ if f.Signal != "" {
+ if _, ok := telemetrytypes.SignalFromText(f.Signal); !ok {
+ return errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "invalid signal: %s", f.Signal)
+ }
+ }
+ if f.Context != "" {
+ if _, ok := telemetrytypes.FieldContextFromText(f.Context); !ok {
+ return errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "invalid context: %s", f.Context)
+ }
+ }
+ return nil
+}
+
+func (f *ListPromotedPathsFilters) MatchesTarget(target Target) bool {
+ if f.Signal != "" {
+ if signal, ok := telemetrytypes.SignalFromText(f.Signal); !ok || signal != target.Entry.Signal {
+ return false
+ }
+ }
+ if f.Context != "" {
+ if fieldContext, ok := telemetrytypes.FieldContextFromText(f.Context); !ok || fieldContext != target.Entry.FieldContext {
+ return false
+ }
+ }
+ return true
+}
+
+func (f *ListPromotedPathsFilters) MatchesPath(path PromotePath) bool {
+ if f.Promoted != nil && *f.Promoted != path.Promote {
+ return false
+ }
+ if f.Indexes != nil && *f.Indexes != (len(path.Indexes) > 0) {
+ return false
+ }
+ return true
+}
+
+func (i *PromotePath) ValidateAndSetDefaults(target Target) error {
if i.Path == "" {
return errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "path is required")
}
@@ -36,26 +89,33 @@ func (i *PromotePath) ValidateAndSetDefaults() error {
return errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "array paths can not be promoted or indexed")
}
- if strings.HasPrefix(i.Path, constants.BodyV2ColumnPrefix) || strings.HasPrefix(i.Path, constants.BodyPromotedColumnPrefix) {
- return errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "`%s`, `%s` don't add these prefixes to the path", constants.BodyV2ColumnPrefix, constants.BodyPromotedColumnPrefix)
+ if strings.HasPrefix(i.Path, target.BaseColumnPrefix()) || strings.HasPrefix(i.Path, target.PromotedColumnPrefix()) {
+ return errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "`%s`, `%s` don't add these prefixes to the path", target.BaseColumnPrefix(), target.PromotedColumnPrefix())
}
- if !strings.HasPrefix(i.Path, telemetrytypes.BodyJSONStringSearchPrefix) {
- return errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "path must start with `body.`")
+ if target.RequiredPathPrefix != "" {
+ if !strings.HasPrefix(i.Path, target.RequiredPathPrefix) {
+ return errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "path must start with `%s`", target.RequiredPathPrefix)
+ }
+ i.Path = strings.TrimPrefix(i.Path, target.RequiredPathPrefix)
}
- // remove the "body." prefix from the path
- i.Path = strings.TrimPrefix(i.Path, telemetrytypes.BodyJSONStringSearchPrefix)
-
isCardinal := keycheck.IsCardinal(i.Path)
if isCardinal {
return errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "cardinal paths can not be promoted or indexed")
}
+ if len(i.Indexes) > 0 && !target.IndexesSupported {
+ return errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "indexes are not supported for %s %s", target.Entry.Signal.StringValue(), target.Entry.FieldContext.StringValue())
+ }
+
for idx, index := range i.Indexes {
if index.Type == "" {
return errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "index type is required")
}
+ if _, err := index.SkipIndexType(); err != nil {
+ return err
+ }
if index.Granularity <= 0 {
return errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "index granularity must be greater than 0")
}
@@ -73,3 +133,16 @@ func (i *PromotePath) ValidateAndSetDefaults() error {
return nil
}
+
+func (index WrappedIndex) SkipIndexType() (schemamigrator.IndexType, error) {
+ switch {
+ case strings.HasPrefix(index.Type, string(schemamigrator.IndexTypeNGramBF)):
+ return schemamigrator.IndexTypeNGramBF, nil
+ case strings.HasPrefix(index.Type, string(schemamigrator.IndexTypeTokenBF)):
+ return schemamigrator.IndexTypeTokenBF, nil
+ case strings.HasPrefix(index.Type, string(schemamigrator.IndexTypeMinMax)):
+ return schemamigrator.IndexTypeMinMax, nil
+ default:
+ return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "invalid index type: %s", index.Type)
+ }
+}
diff --git a/pkg/types/promotetypes/types_test.go b/pkg/types/promotetypes/types_test.go
new file mode 100644
index 0000000000..ca5c815bbd
--- /dev/null
+++ b/pkg/types/promotetypes/types_test.go
@@ -0,0 +1,354 @@
+package promotetypes
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+
+ "github.com/SigNoz/signoz/pkg/types/telemetrytypes"
+)
+
+func TestPromotePathTarget(t *testing.T) {
+ testCases := []struct {
+ name string
+ path *PromotePath
+ want Target
+ wantErr bool
+ }{
+ {
+ name: "LogsBody_Resolved",
+ path: &PromotePath{Signal: "logs", Context: "body", Path: "body.user.name"},
+ want: NewLogsBodyTarget(),
+ },
+ {
+ name: "TracesAttribute_Resolved",
+ path: &PromotePath{Signal: "traces", Context: "attribute", Path: "http.method"},
+ want: NewTracesAttributesTarget(),
+ },
+ {
+ name: "InvalidSignal_Rejected",
+ path: &PromotePath{Signal: "events", Context: "attribute", Path: "http.method"},
+ wantErr: true,
+ },
+ {
+ name: "InvalidContext_Rejected",
+ path: &PromotePath{Signal: "logs", Context: "span", Path: "user.name"},
+ wantErr: true,
+ },
+ {
+ name: "UnsupportedDomain_Rejected",
+ path: &PromotePath{Signal: "metrics", Context: "attribute", Path: "http.method"},
+ wantErr: true,
+ },
+ }
+
+ for _, testCase := range testCases {
+ t.Run(testCase.name, func(t *testing.T) {
+ target, err := testCase.path.Target()
+ if testCase.wantErr {
+ assert.Error(t, err)
+ return
+ }
+ require.NoError(t, err)
+ assert.Equal(t, testCase.want, target)
+ })
+ }
+}
+
+func TestValidateAndSetDefaultsLogsBody(t *testing.T) {
+ target := NewLogsBodyTarget()
+
+ testCases := []struct {
+ name string
+ path *PromotePath
+ wantErr bool
+ wantPath string
+ wantJSONDataType telemetrytypes.JSONDataType
+ }{
+ {
+ name: "ValidPath_BodyPrefixStripped",
+ path: &PromotePath{Path: "body.user.name", Promote: true},
+ wantPath: "user.name",
+ },
+ {
+ name: "PathWithoutBodyPrefix_Rejected",
+ path: &PromotePath{Path: "user.name", Promote: true},
+ wantErr: true,
+ },
+ {
+ name: "BodyV2PrefixedPath_Rejected",
+ path: &PromotePath{Path: "body_v2.user.name", Promote: true},
+ wantErr: true,
+ },
+ {
+ name: "BodyPromotedPrefixedPath_Rejected",
+ path: &PromotePath{Path: "body_promoted.user.name", Promote: true},
+ wantErr: true,
+ },
+ {
+ name: "EmptyPath_Rejected",
+ path: &PromotePath{Path: "", Promote: true},
+ wantErr: true,
+ },
+ {
+ name: "SpacedPath_Rejected",
+ path: &PromotePath{Path: "body.my path", Promote: true},
+ wantErr: true,
+ },
+ {
+ name: "ArrayIndexPath_Rejected",
+ path: &PromotePath{Path: "body.users[].id", Promote: true},
+ wantErr: true,
+ },
+ {
+ name: "ArrayWildcardPath_Rejected",
+ path: &PromotePath{Path: "body.users[*].id", Promote: true},
+ wantErr: true,
+ },
+ {
+ name: "CardinalPath_Rejected",
+ path: &PromotePath{Path: "body.request.550e8400-e29b-41d4-a716-446655440000", Promote: true},
+ wantErr: true,
+ },
+ {
+ name: "ValidIndex_JSONDataTypeDefaulted",
+ path: &PromotePath{
+ Path: "body.user.name",
+ Indexes: []WrappedIndex{
+ {FieldDataType: telemetrytypes.FieldDataTypeString, Type: "ngrambf_v1(4, 1024, 2, 0)", Granularity: 1},
+ },
+ },
+ wantPath: "user.name",
+ wantJSONDataType: telemetrytypes.String,
+ },
+ {
+ name: "UnsupportedColumnTypeIndex_Rejected",
+ path: &PromotePath{
+ Path: "body.user.active",
+ Indexes: []WrappedIndex{{FieldDataType: telemetrytypes.FieldDataTypeBool, Type: "minmax", Granularity: 1}},
+ },
+ wantErr: true,
+ },
+ {
+ name: "IndexWithoutType_Rejected",
+ path: &PromotePath{
+ Path: "body.user.name",
+ Indexes: []WrappedIndex{{FieldDataType: telemetrytypes.FieldDataTypeString, Granularity: 1}},
+ },
+ wantErr: true,
+ },
+ {
+ name: "UnknownIndexType_Rejected",
+ path: &PromotePath{
+ Path: "body.user.name",
+ Indexes: []WrappedIndex{{FieldDataType: telemetrytypes.FieldDataTypeString, Type: "unsupported", Granularity: 1}},
+ },
+ wantErr: true,
+ },
+ {
+ name: "IndexWithoutGranularity_Rejected",
+ path: &PromotePath{
+ Path: "body.user.name",
+ Indexes: []WrappedIndex{{FieldDataType: telemetrytypes.FieldDataTypeString, Type: "minmax"}},
+ },
+ wantErr: true,
+ },
+ }
+
+ for _, testCase := range testCases {
+ t.Run(testCase.name, func(t *testing.T) {
+ err := testCase.path.ValidateAndSetDefaults(target)
+ if testCase.wantErr {
+ assert.Error(t, err)
+ return
+ }
+ require.NoError(t, err)
+ assert.Equal(t, testCase.wantPath, testCase.path.Path)
+ if testCase.wantJSONDataType != (telemetrytypes.JSONDataType{}) {
+ require.Len(t, testCase.path.Indexes, 1)
+ assert.Equal(t, testCase.wantJSONDataType, testCase.path.Indexes[0].JSONDataType)
+ }
+ })
+ }
+}
+
+func TestValidateAndSetDefaultsTracesAttributes(t *testing.T) {
+ target := NewTracesAttributesTarget()
+
+ testCases := []struct {
+ name string
+ path *PromotePath
+ wantErr bool
+ wantPath string
+ }{
+ {
+ name: "BareAttributeName_KeptAsIs",
+ path: &PromotePath{Path: "http.method", Promote: true},
+ wantPath: "http.method",
+ },
+ {
+ name: "AttributesPrefixedPath_Rejected",
+ path: &PromotePath{Path: "attributes.http.method", Promote: true},
+ wantErr: true,
+ },
+ {
+ name: "AttributesPromotedPrefixedPath_Rejected",
+ path: &PromotePath{Path: "attributes_promoted.http.method", Promote: true},
+ wantErr: true,
+ },
+ {
+ name: "EmptyPath_Rejected",
+ path: &PromotePath{Path: "", Promote: true},
+ wantErr: true,
+ },
+ {
+ name: "SpacedPath_Rejected",
+ path: &PromotePath{Path: "my attr", Promote: true},
+ wantErr: true,
+ },
+ {
+ name: "ArrayIndexPath_Rejected",
+ path: &PromotePath{Path: "tags[].id", Promote: true},
+ wantErr: true,
+ },
+ }
+
+ for _, testCase := range testCases {
+ t.Run(testCase.name, func(t *testing.T) {
+ err := testCase.path.ValidateAndSetDefaults(target)
+ if testCase.wantErr {
+ assert.Error(t, err)
+ return
+ }
+ require.NoError(t, err)
+ assert.Equal(t, testCase.wantPath, testCase.path.Path)
+ })
+ }
+}
+
+func TestListPromotedPathsFiltersValidate(t *testing.T) {
+ testCases := []struct {
+ name string
+ filters ListPromotedPathsFilters
+ wantErr bool
+ }{
+ {
+ name: "EmptyFilters_Valid",
+ filters: ListPromotedPathsFilters{},
+ },
+ {
+ name: "KnownSignalAndContext_Valid",
+ filters: ListPromotedPathsFilters{Signal: "traces", Context: "attribute"},
+ },
+ {
+ name: "InvalidSignal_Rejected",
+ filters: ListPromotedPathsFilters{Signal: "events"},
+ wantErr: true,
+ },
+ {
+ name: "InvalidContext_Rejected",
+ filters: ListPromotedPathsFilters{Context: "json"},
+ wantErr: true,
+ },
+ }
+
+ for _, testCase := range testCases {
+ t.Run(testCase.name, func(t *testing.T) {
+ err := testCase.filters.Validate()
+ if testCase.wantErr {
+ assert.Error(t, err)
+ return
+ }
+ require.NoError(t, err)
+ })
+ }
+}
+
+func TestListPromotedPathsFiltersMatch(t *testing.T) {
+ trueValue := true
+ falseValue := false
+
+ testCases := []struct {
+ name string
+ filters ListPromotedPathsFilters
+ target Target
+ path PromotePath
+ wantTarget bool
+ wantPath bool
+ }{
+ {
+ name: "EmptyFilters_MatchEverything",
+ filters: ListPromotedPathsFilters{},
+ target: NewTracesAttributesTarget(),
+ path: PromotePath{Path: "http.method", Promote: true},
+ wantTarget: true,
+ wantPath: true,
+ },
+ {
+ name: "SignalFilter_MatchesSameSignal",
+ filters: ListPromotedPathsFilters{Signal: "traces"},
+ target: NewTracesAttributesTarget(),
+ wantTarget: true,
+ },
+ {
+ name: "SignalFilter_SkipsOtherSignals",
+ filters: ListPromotedPathsFilters{Signal: "traces"},
+ target: NewLogsBodyTarget(),
+ wantTarget: false,
+ },
+ {
+ name: "ContextAliasFilter_MatchesResolvedContext",
+ filters: ListPromotedPathsFilters{Context: "tag"},
+ target: NewTracesAttributesTarget(),
+ wantTarget: true,
+ },
+ {
+ name: "UppercaseContextFilter_MatchesResolvedContext",
+ filters: ListPromotedPathsFilters{Context: "ATTRIBUTE"},
+ target: NewTracesAttributesTarget(),
+ wantTarget: true,
+ },
+ {
+ name: "ContextFilter_SkipsOtherContexts",
+ filters: ListPromotedPathsFilters{Context: "body"},
+ target: NewTracesAttributesTarget(),
+ wantTarget: false,
+ },
+ {
+ name: "PromotedFalseFilter_MatchesUnpromotedPath",
+ filters: ListPromotedPathsFilters{Promoted: &falseValue},
+ path: PromotePath{Path: "request.duration"},
+ wantPath: true,
+ },
+ {
+ name: "PromotedFalseFilter_SkipsPromotedPath",
+ filters: ListPromotedPathsFilters{Promoted: &falseValue},
+ path: PromotePath{Path: "http.method", Promote: true},
+ wantPath: false,
+ },
+ {
+ name: "IndexesTrueFilter_MatchesIndexedPath",
+ filters: ListPromotedPathsFilters{Indexes: &trueValue},
+ path: PromotePath{Path: "user.name", Indexes: []WrappedIndex{{Type: "minmax"}}},
+ wantPath: true,
+ },
+ {
+ name: "IndexesTrueFilter_SkipsUnindexedPath",
+ filters: ListPromotedPathsFilters{Indexes: &trueValue},
+ path: PromotePath{Path: "http.method", Promote: true},
+ wantPath: false,
+ },
+ }
+
+ for _, testCase := range testCases {
+ t.Run(testCase.name, func(t *testing.T) {
+ if testCase.target.Entry.Signal.StringValue() != "" {
+ assert.Equal(t, testCase.wantTarget, testCase.filters.MatchesTarget(testCase.target))
+ }
+ if testCase.path.Path != "" {
+ assert.Equal(t, testCase.wantPath, testCase.filters.MatchesPath(testCase.path))
+ }
+ })
+ }
+}
diff --git a/pkg/types/telemetrytypes/signal.go b/pkg/types/telemetrytypes/signal.go
index 6150a31a44..9f40baae0a 100644
--- a/pkg/types/telemetrytypes/signal.go
+++ b/pkg/types/telemetrytypes/signal.go
@@ -22,3 +22,18 @@ func (Signal) Enum() []any {
SignalUnspecified,
}
}
+
+// SignalFromText resolves a signal word to its Signal; ok is false for an
+// unknown word.
+func SignalFromText(text string) (Signal, bool) {
+ s := Signal{valuer.NewString(text)}
+ switch s {
+ case SignalTraces:
+ return SignalTraces, true
+ case SignalLogs:
+ return SignalLogs, true
+ case SignalMetrics:
+ return SignalMetrics, true
+ }
+ return Signal{}, false
+}
diff --git a/pkg/types/telemetrytypes/store.go b/pkg/types/telemetrytypes/store.go
index fbc3a43e21..b74752b279 100644
--- a/pkg/types/telemetrytypes/store.go
+++ b/pkg/types/telemetrytypes/store.go
@@ -37,11 +37,13 @@ type MetadataStore interface {
// ListLogsJSONIndexes lists the JSON indexes for the logs table.
ListLogsJSONIndexes(ctx context.Context, filters ...string) ([]TelemetryFieldKeySkipIndex, error)
- // ListPromotedPaths lists the promoted paths.
- GetPromotedPaths(ctx context.Context, paths ...string) (map[string]bool, error)
+ // GetPromotedPaths lists the promoted paths recorded in the column
+ // evolution table for the entry's signal, column and field context.
+ GetPromotedPaths(ctx context.Context, entry EvolutionEntry, paths ...string) (map[string]bool, error)
- // PromotePaths promotes the paths.
- PromotePaths(ctx context.Context, paths ...string) error
+ // PromotePaths records promoted paths in the column evolution table as
+ // rows templated by entry; FieldName and ReleaseTime are set per path.
+ PromotePaths(ctx context.Context, entry EvolutionEntry, paths ...string) error
// GetFirstSeenFromMetricMetadata gets the first seen timestamp for a metric metadata lookup key.
GetFirstSeenFromMetricMetadata(ctx context.Context, lookupKeys []MetricMetadataLookupKey) (map[MetricMetadataLookupKey]int64, error)
diff --git a/pkg/types/telemetrytypes/telemetrytypestest/metadata_store.go b/pkg/types/telemetrytypes/telemetrytypestest/metadata_store.go
index c317f9e8d4..69ebe72eec 100644
--- a/pkg/types/telemetrytypes/telemetrytypestest/metadata_store.go
+++ b/pkg/types/telemetrytypes/telemetrytypestest/metadata_store.go
@@ -18,7 +18,7 @@ type MockMetadataStore struct {
TemporalityMap map[string]metrictypes.Temporality
TypeMap map[string]metrictypes.Type
ReducedMap map[string]bool
- PromotedPathsMap map[string]bool
+ PromotedPathsMap map[telemetrytypes.Signal]map[string]bool
LogsJSONIndexes []telemetrytypes.TelemetryFieldKeySkipIndex
ColumnEvolutionMetadataMap map[string][]*telemetrytypes.EvolutionEntry
LookupKeysMap map[telemetrytypes.MetricMetadataLookupKey]int64
@@ -34,7 +34,7 @@ func NewMockMetadataStore() *MockMetadataStore {
AllValuesMap: make(map[string]*telemetrytypes.TelemetryFieldValues),
TemporalityMap: make(map[string]metrictypes.Temporality),
TypeMap: make(map[string]metrictypes.Type),
- PromotedPathsMap: make(map[string]bool),
+ PromotedPathsMap: make(map[telemetrytypes.Signal]map[string]bool),
LogsJSONIndexes: []telemetrytypes.TelemetryFieldKeySkipIndex{},
ColumnEvolutionMetadataMap: make(map[string][]*telemetrytypes.EvolutionEntry),
LookupKeysMap: make(map[telemetrytypes.MetricMetadataLookupKey]int64),
@@ -361,16 +361,19 @@ func (m *MockMetadataStore) SetTemporality(metricName string, temporality metric
}
// PromotePaths promotes the paths.
-func (m *MockMetadataStore) PromotePaths(ctx context.Context, paths ...string) error {
+func (m *MockMetadataStore) PromotePaths(_ context.Context, entry telemetrytypes.EvolutionEntry, paths ...string) error {
+ if m.PromotedPathsMap[entry.Signal] == nil {
+ m.PromotedPathsMap[entry.Signal] = make(map[string]bool)
+ }
for _, path := range paths {
- m.PromotedPathsMap[path] = true
+ m.PromotedPathsMap[entry.Signal][path] = true
}
return nil
}
// GetPromotedPaths returns the promoted paths.
-func (m *MockMetadataStore) GetPromotedPaths(ctx context.Context, paths ...string) (map[string]bool, error) {
- return m.PromotedPathsMap, nil
+func (m *MockMetadataStore) GetPromotedPaths(_ context.Context, entry telemetrytypes.EvolutionEntry, _ ...string) (map[string]bool, error) {
+ return m.PromotedPathsMap[entry.Signal], nil
}
// ListLogsJSONIndexes lists the JSON indexes for the logs table.