SigNoz is an open-source, OpenTelemetry-native observability platform for your team and their AI agents. Get logs, metrics, and traces in one tool with features like APM, distributed tracing, log management, infra monitoring, etc. Combined with SigNoz MCP and a native AI teammate (in SigNoz Cloud) it helps you build more resilient apps.
TypeScript apmapplication-monitoringdistributed-tracinggogood-first-issuejaegerloglogsmetricsmonitoringnextjsobservabilityopen-sourceopentelemetryprometheusreactreactjsself-hostedtracingtypescript
feat: enabled logs and added collection strategy for gcp integration (#13184)
Swapnil Nakade574bfa62b82e
diff --git a/pkg/modules/cloudintegration/implcloudintegration/fs/definitions/gcp/cloudsql_mysql/integration.json b/pkg/modules/cloudintegration/implcloudintegration/fs/definitions/gcp/cloudsql_mysql/integration.jsonindex 59f339525b..297d5e4188 100644--- a/pkg/modules/cloudintegration/implcloudintegration/fs/definitions/gcp/cloudsql_mysql/integration.json+++ b/pkg/modules/cloudintegration/implcloudintegration/fs/definitions/gcp/cloudsql_mysql/integration.json@@ -118,10 +118,47 @@"description": ""}],- "logs": []+ "logs": [+ {+ "name": "Project",+ "path": "gcp.project",+ "type": "string"+ },+ {+ "name": "Resource Type",+ "path": "gcp.resource_type",+ "type": "string"+ }+ ]},"telemetryCollectionStrategy": {- "gcp": {}+ "gcp": {+ "metrics": {+ "metricNames": [+ "cloudsql.googleapis.com/database/up",+ "cloudsql.googleapis.com/database/instance_state",+ "cloudsql.googleapis.com/database/replication/replica_lag",+ "cloudsql.googleapis.com/database/cpu/utilization",+ "cloudsql.googleapis.com/database/memory/utilization",+ "cloudsql.googleapis.com/database/disk/utilization",+ "cloudsql.googleapis.com/database/network/connections",+ "cloudsql.googleapis.com/database/mysql/queries",+ "cloudsql.googleapis.com/database/mysql/dml_operations_count",+ "cloudsql.googleapis.com/database/mysql/threads",+ "cloudsql.googleapis.com/database/mysql/innodb/buffer_pool_reads_count",+ "cloudsql.googleapis.com/database/mysql/innodb/buffer_pool_read_requests_count",+ "cloudsql.googleapis.com/database/mysql/slow_queries_count",+ "cloudsql.googleapis.com/database/mysql/aborted_connects_count",+ "cloudsql.googleapis.com/database/mysql/innodb/deadlocks_count",+ "cloudsql.googleapis.com/database/mysql/innodb/row_lock_waits_count",+ "cloudsql.googleapis.com/database/disk/read_ops_count",+ "cloudsql.googleapis.com/database/disk/write_ops_count"+ ]+ },+ "logs": {+ "sinkFilter": "resource.type=\"cloudsql_database\" AND (logName:\"/logs/cloudsql.googleapis.com%2Fmysql\" OR (log_id(\"cloudaudit.googleapis.com/data_access\") AND protoPayload.request.@type=\"type.googleapis.com/google.cloud.sql.audit.v1.MysqlAuditEntry\"))"+ }+ }},"assets": {"dashboards": [diff --git a/pkg/modules/cloudintegration/implcloudintegration/fs/definitions/gcp/cloudsql_postgres/integration.json b/pkg/modules/cloudintegration/implcloudintegration/fs/definitions/gcp/cloudsql_postgres/integration.jsonindex d2c0ec80ae..516cdbccd9 100644--- a/pkg/modules/cloudintegration/implcloudintegration/fs/definitions/gcp/cloudsql_postgres/integration.json+++ b/pkg/modules/cloudintegration/implcloudintegration/fs/definitions/gcp/cloudsql_postgres/integration.json@@ -5,7 +5,7 @@"overview": "file://overview.md","supportedSignals": {"metrics": true,- "logs": false+ "logs": true},"dataCollected": {"metrics": [@@ -75,6 +75,12 @@"type": "Gauge","description": ""},+ {+ "name": "cloudsql.googleapis.com/database/postgresql/insights/aggregate/io_time",+ "unit": "Microseconds",+ "type": "Sum",+ "description": ""+ },{"name": "cloudsql.googleapis.com/database/postgresql/insights/perquery/execution_time","unit": "Microseconds",@@ -88,10 +94,43 @@"description": ""}],- "logs": []+ "logs": [+ {+ "name": "Project",+ "path": "gcp.project",+ "type": "string"+ },+ {+ "name": "Resource Type",+ "path": "gcp.resource_type",+ "type": "string"+ }+ ]},"telemetryCollectionStrategy": {- "gcp": {}+ "gcp": {+ "metrics": {+ "metricNames": [+ "cloudsql.googleapis.com/database/up",+ "cloudsql.googleapis.com/database/cpu/utilization",+ "cloudsql.googleapis.com/database/memory/utilization",+ "cloudsql.googleapis.com/database/memory/usage",+ "cloudsql.googleapis.com/database/disk/bytes_used",+ "cloudsql.googleapis.com/database/postgresql/num_backends",+ "cloudsql.googleapis.com/database/postgresql/num_backends_by_state",+ "cloudsql.googleapis.com/database/postgresql/transaction_count",+ "cloudsql.googleapis.com/database/postgresql/deadlock_count",+ "cloudsql.googleapis.com/database/postgresql/vacuum/oldest_transaction_age",+ "cloudsql.googleapis.com/database/postgresql/insights/aggregate/execution_time",+ "cloudsql.googleapis.com/database/postgresql/insights/aggregate/io_time",+ "cloudsql.googleapis.com/database/postgresql/insights/perquery/execution_time",+ "cloudsql.googleapis.com/database/postgresql/replication/replica_byte_lag"+ ]+ },+ "logs": {+ "sinkFilter": "resource.type=\"cloudsql_database\" AND (logName:\"/logs/cloudsql.googleapis.com%2Fpostgres\" OR (log_id(\"cloudaudit.googleapis.com/data_access\") AND protoPayload.request.@type=\"type.googleapis.com/google.cloud.sql.audit.v1.PgAuditEntry\"))"+ }+ }},"assets": {"dashboards": [diff --git a/pkg/modules/cloudintegration/implcloudintegration/fs/definitions/gcp/cloudstorage/integration.json b/pkg/modules/cloudintegration/implcloudintegration/fs/definitions/gcp/cloudstorage/integration.jsonindex 462970807d..a432e82500 100644--- a/pkg/modules/cloudintegration/implcloudintegration/fs/definitions/gcp/cloudstorage/integration.json+++ b/pkg/modules/cloudintegration/implcloudintegration/fs/definitions/gcp/cloudstorage/integration.json@@ -5,7 +5,7 @@"overview": "file://overview.md","supportedSignals": {"metrics": true,- "logs": false+ "logs": true},"dataCollected": {"metrics": [@@ -52,10 +52,36 @@"description": ""}],- "logs": []+ "logs": [+ {+ "name": "Project",+ "path": "gcp.project",+ "type": "string"+ },+ {+ "name": "Resource Type",+ "path": "gcp.resource_type",+ "type": "string"+ }+ ]},"telemetryCollectionStrategy": {- "gcp": {}+ "gcp": {+ "metrics": {+ "metricNames": [+ "storage.googleapis.com/storage/v2/total_bytes",+ "storage.googleapis.com/storage/v2/total_count",+ "storage.googleapis.com/storage/v2/deleted_bytes",+ "storage.googleapis.com/api/request_count",+ "storage.googleapis.com/network/received_bytes_count",+ "storage.googleapis.com/network/sent_bytes_count",+ "storage.googleapis.com/authn/authentication_count"+ ]+ },+ "logs": {+ "sinkFilter": "resource.type=\"gcs_bucket\""+ }+ }},"assets": {"dashboards": [diff --git a/pkg/modules/cloudintegration/implcloudintegration/fs/definitions/gcp/computeengine/integration.json b/pkg/modules/cloudintegration/implcloudintegration/fs/definitions/gcp/computeengine/integration.jsonindex 8faadc3718..d391d6a7c8 100644--- a/pkg/modules/cloudintegration/implcloudintegration/fs/definitions/gcp/computeengine/integration.json+++ b/pkg/modules/cloudintegration/implcloudintegration/fs/definitions/gcp/computeengine/integration.json@@ -5,7 +5,7 @@"overview": "file://overview.md","supportedSignals": {"metrics": true,- "logs": false+ "logs": true},"dataCollected": {"metrics": [@@ -106,10 +106,45 @@"description": ""}],- "logs": []+ "logs": [+ {+ "name": "Project",+ "path": "gcp.project",+ "type": "string"+ },+ {+ "name": "Resource Type",+ "path": "gcp.resource_type",+ "type": "string"+ }+ ]},"telemetryCollectionStrategy": {- "gcp": {}+ "gcp": {+ "metrics": {+ "metricNames": [+ "compute.googleapis.com/instance/uptime_total",+ "compute.googleapis.com/instance/cpu/utilization",+ "compute.googleapis.com/guest/memory/bytes_used",+ "compute.googleapis.com/guest/memory/percent_used",+ "agent.googleapis.com/memory/percent_used",+ "compute.googleapis.com/instance/disk/average_io_latency",+ "compute.googleapis.com/instance/disk/read_bytes_count",+ "compute.googleapis.com/instance/disk/write_bytes_count",+ "compute.googleapis.com/instance/disk/read_ops_count",+ "compute.googleapis.com/instance/disk/write_ops_count",+ "compute.googleapis.com/instance/disk/performance_status",+ "compute.googleapis.com/instance/network/received_bytes_count",+ "compute.googleapis.com/instance/network/sent_bytes_count",+ "compute.googleapis.com/instance/network/received_packets_count",+ "compute.googleapis.com/instance/network/sent_packets_count",+ "compute.googleapis.com/firewall/dropped_packets_count"+ ]+ },+ "logs": {+ "sinkFilter": "resource.type=\"gce_instance\" OR protoPayload.serviceName=\"compute.googleapis.com\""+ }+ }},"assets": {"dashboards": [diff --git a/pkg/modules/cloudintegration/implcloudintegration/fs/definitions/gcp/gke/integration.json b/pkg/modules/cloudintegration/implcloudintegration/fs/definitions/gcp/gke/integration.jsonindex 277df67473..534903d0a9 100644--- a/pkg/modules/cloudintegration/implcloudintegration/fs/definitions/gcp/gke/integration.json+++ b/pkg/modules/cloudintegration/implcloudintegration/fs/definitions/gcp/gke/integration.json@@ -5,7 +5,7 @@"overview": "file://overview.md","supportedSignals": {"metrics": true,- "logs": false+ "logs": true},"dataCollected": {"metrics": [@@ -106,10 +106,45 @@"description": ""}],- "logs": []+ "logs": [+ {+ "name": "Project",+ "path": "gcp.project",+ "type": "string"+ },+ {+ "name": "Resource Type",+ "path": "gcp.resource_type",+ "type": "string"+ }+ ]},"telemetryCollectionStrategy": {- "gcp": {}+ "gcp": {+ "metrics": {+ "metricNames": [+ "kubernetes.io/container/cpu/limit_utilization",+ "kubernetes.io/container/cpu/request_utilization",+ "kubernetes.io/container/memory/limit_utilization",+ "kubernetes.io/container/memory/request_utilization",+ "kubernetes.io/container/restart_count",+ "kubernetes.io/pod/latencies/pod_first_ready",+ "kubernetes.io/pod/network/received_bytes_count",+ "kubernetes.io/pod/network/sent_bytes_count",+ "kubernetes.io/pod/volume/utilization",+ "kubernetes.io/node/cpu/allocatable_utilization",+ "kubernetes.io/node/memory/allocatable_utilization",+ "kubernetes.io/node/ephemeral_storage/used_bytes",+ "kubernetes.io/node/ephemeral_storage/allocatable_bytes",+ "kubernetes.io/node/network/received_bytes_count",+ "kubernetes.io/node/network/sent_bytes_count",+ "kubernetes.io/node/status_condition"+ ]+ },+ "logs": {+ "sinkFilter": "resource.type=(\"k8s_container\" OR \"k8s_pod\" OR \"k8s_node\" OR \"k8s_cluster\" OR \"k8s_control_plane_component\" OR \"gke_cluster\" OR \"gke_nodepool\")"+ }+ }},"assets": {"dashboards": [diff --git a/pkg/modules/cloudintegration/implcloudintegration/fs/definitions/gcp/memorystore_redis/integration.json b/pkg/modules/cloudintegration/implcloudintegration/fs/definitions/gcp/memorystore_redis/integration.jsonindex b0d76c2add..138c1b99e1 100644--- a/pkg/modules/cloudintegration/implcloudintegration/fs/definitions/gcp/memorystore_redis/integration.json+++ b/pkg/modules/cloudintegration/implcloudintegration/fs/definitions/gcp/memorystore_redis/integration.json@@ -5,7 +5,7 @@"overview": "file://overview.md","supportedSignals": {"metrics": true,- "logs": false+ "logs": true},"dataCollected": {"metrics": [@@ -76,10 +76,40 @@"description": ""}],- "logs": []+ "logs": [+ {+ "name": "Project",+ "path": "gcp.project",+ "type": "string"+ },+ {+ "name": "Resource Type",+ "path": "gcp.resource_type",+ "type": "string"+ }+ ]},"telemetryCollectionStrategy": {- "gcp": {}+ "gcp": {+ "metrics": {+ "metricNames": [+ "redis.googleapis.com/server/uptime",+ "redis.googleapis.com/stats/memory/usage_ratio",+ "redis.googleapis.com/stats/memory/system_memory_usage_ratio",+ "redis.googleapis.com/stats/cpu_utilization_main_thread",+ "redis.googleapis.com/clients/connected",+ "redis.googleapis.com/stats/cache_hit_ratio",+ "redis.googleapis.com/keyspace/keys",+ "redis.googleapis.com/commands/usec_per_call",+ "redis.googleapis.com/commands/calls",+ "redis.googleapis.com/clients/blocked",+ "redis.googleapis.com/stats/reject_connections_count"+ ]+ },+ "logs": {+ "sinkFilter": "resource.type=\"redis_instance\" OR (protoPayload.serviceName=\"redis.googleapis.com\" AND protoPayload.methodName:\".CloudRedis.\") OR (log_id(\"maintenance.googleapis.com/activity_log\") AND labels.activity_type_name:\"CloudRedisRollout\")"+ }+ }},"assets": {"dashboards": [diff --git a/pkg/types/cloudintegrationtypes/cloudprovider_gcp.go b/pkg/types/cloudintegrationtypes/cloudprovider_gcp.goindex 0095db65ed..afde8c615b 100644--- a/pkg/types/cloudintegrationtypes/cloudprovider_gcp.go+++ b/pkg/types/cloudintegrationtypes/cloudprovider_gcp.go@@ -24,7 +24,19 @@ type GCPConnectionArtifact struct{}type GCPIntegrationConfig struct{}-type GCPTelemetryCollectionStrategy struct{}+type GCPTelemetryCollectionStrategy struct {+ Metrics *GCPMetricsCollectionStrategy `json:"metrics" required:"false" nullable:"false"`+ Logs *GCPLogsCollectionStrategy `json:"logs" required:"false" nullable:"false"`+}++type GCPMetricsCollectionStrategy struct {+ MetricNames []string `json:"metricNames" required:"true" nullable:"false"`+}++type GCPLogsCollectionStrategy struct {+ // Cloud Logging filter for the log sink, https://cloud.google.com/logging/docs/view/logging-query-language+ SinkFilter string `json:"sinkFilter" required:"true"`+}type GCPServiceConfig struct {Logs *GCPServiceLogsConfig `json:"logs,omitempty" required:"false"`diff --git a/tests/fixtures/cloudintegrations.py b/tests/fixtures/cloudintegrations.pyindex ffc752c427..ee0caaced2 100644--- a/tests/fixtures/cloudintegrations.py+++ b/tests/fixtures/cloudintegrations.py@@ -49,9 +49,6 @@ class ProviderAccountSpec:class ProviderServiceSpec:provider: strservice_id: str- # GCP ships every service with supportedSignals.logs false, so a logs block- # is neither required on write nor persisted.- supports_logs: boolaccount_config: dict# id shown in parametrized test names; defaults to the provider slug.id: str = field(default="")@@ -61,10 +58,12 @@ class ProviderServiceSpec:object.__setattr__(self, "id", self.provider)def build_service_config(self, metrics_enabled: bool, logs_enabled: bool | None = None) -> dict:- config: dict = {"metrics": {"enabled": metrics_enabled}}- if self.supports_logs:- config["logs"] = {"enabled": metrics_enabled if logs_enabled is None else logs_enabled}- return {self.provider: config}+ return {+ self.provider: {+ "metrics": {"enabled": metrics_enabled},+ "logs": {"enabled": metrics_enabled if logs_enabled is None else logs_enabled},+ }+ }@pytest.fixture(scope="function")diff --git a/tests/integration/tests/cloudintegrations/05_services.py b/tests/integration/tests/cloudintegrations/05_services.pyindex cfbbd8dfeb..c7407e982d 100644--- a/tests/integration/tests/cloudintegrations/05_services.py+++ b/tests/integration/tests/cloudintegrations/05_services.py@@ -19,14 +19,12 @@ logger = setup_logger(__name__)AWS_SERVICE_SPEC = ProviderServiceSpec(provider="aws",service_id="rds",- supports_logs=True,account_config={"aws": {"deploymentRegion": "us-east-1", "regions": ["us-east-1"]}},)GCP_SERVICE_SPEC = ProviderServiceSpec(provider="gcp",service_id="cloudsql_postgres",- supports_logs=False,account_config={"gcp": {"deploymentProjectId": "signoz-test-project",@@ -162,7 +160,7 @@ def test_get_service_details_without_account(assert data["cloudIntegrationService"] is None, "cloudIntegrationService should be null without account context"assert data["supportedSignals"]["metrics"] is True, "metrics should be a supported signal"- assert data["supportedSignals"]["logs"] is spec.supports_logs, f"logs support should be {spec.supports_logs} for {spec.provider}"+ assert data["supportedSignals"]["logs"] is True, "logs should be a supported signal"@provider_spec@@ -253,10 +251,7 @@ def test_update_service_config(assert svc["config"][spec.provider]["metrics"]["enabled"] is True, "metrics should be enabled"assert svc["cloudIntegrationId"] == account_id, "cloudIntegrationId should match the account"- if spec.supports_logs:- assert svc["config"][spec.provider]["logs"]["enabled"] is True, "logs should be enabled"- else:- assert svc["config"][spec.provider].get("logs") is None, f"logs should not be stored for {spec.provider}, got: {svc['config'][spec.provider]}"+ assert svc["config"][spec.provider]["logs"]["enabled"] is True, "logs should be enabled"@provider_spec@@ -307,8 +302,7 @@ def test_update_service_config_disable(assert svc is not None, "cloudIntegrationService should still be present after disable"assert svc["config"][spec.provider]["metrics"]["enabled"] is False, "metrics should be disabled"- if spec.supports_logs:- assert svc["config"][spec.provider]["logs"]["enabled"] is False, "logs should be disabled"+ assert svc["config"][spec.provider]["logs"]["enabled"] is False, "logs should be disabled"@provider_spec@@ -337,7 +331,7 @@ def test_update_gcp_service_without_metrics_config(get_token: Callable[[str, str], str],create_cloud_integration_account: Callable,) -> None:- """GCP services support metrics only, so a config omitting metrics is rejected."""+ """A GCP config omitting metrics is rejected."""admin_token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)account = create_cloud_integration_account(admin_token, "gcp", config=GCP_SERVICE_SPEC.account_config)