Skip to content

SigNoz is an open-source, OpenTelemetry-native observability platform for your team and their AI agents. Get logs, metrics, and traces in one tool with features like APM, distributed tracing, log management, infra monitoring, etc. Combined with SigNoz MCP and a native AI teammate (in SigNoz Cloud) it helps you build more resilient apps.

feat: enabled logs and added collection strategy for gcp integration (#13184)

Swapnil Nakade574bfa62b82e
diff --git a/pkg/modules/cloudintegration/implcloudintegration/fs/definitions/gcp/cloudsql_mysql/integration.json b/pkg/modules/cloudintegration/implcloudintegration/fs/definitions/gcp/cloudsql_mysql/integration.json
index 59f339525b..297d5e4188 100644
--- a/pkg/modules/cloudintegration/implcloudintegration/fs/definitions/gcp/cloudsql_mysql/integration.json
+++ b/pkg/modules/cloudintegration/implcloudintegration/fs/definitions/gcp/cloudsql_mysql/integration.json
@@ -118,10 +118,47 @@
"description": ""
}
],
- "logs": []
+ "logs": [
+ {
+ "name": "Project",
+ "path": "gcp.project",
+ "type": "string"
+ },
+ {
+ "name": "Resource Type",
+ "path": "gcp.resource_type",
+ "type": "string"
+ }
+ ]
},
"telemetryCollectionStrategy": {
- "gcp": {}
+ "gcp": {
+ "metrics": {
+ "metricNames": [
+ "cloudsql.googleapis.com/database/up",
+ "cloudsql.googleapis.com/database/instance_state",
+ "cloudsql.googleapis.com/database/replication/replica_lag",
+ "cloudsql.googleapis.com/database/cpu/utilization",
+ "cloudsql.googleapis.com/database/memory/utilization",
+ "cloudsql.googleapis.com/database/disk/utilization",
+ "cloudsql.googleapis.com/database/network/connections",
+ "cloudsql.googleapis.com/database/mysql/queries",
+ "cloudsql.googleapis.com/database/mysql/dml_operations_count",
+ "cloudsql.googleapis.com/database/mysql/threads",
+ "cloudsql.googleapis.com/database/mysql/innodb/buffer_pool_reads_count",
+ "cloudsql.googleapis.com/database/mysql/innodb/buffer_pool_read_requests_count",
+ "cloudsql.googleapis.com/database/mysql/slow_queries_count",
+ "cloudsql.googleapis.com/database/mysql/aborted_connects_count",
+ "cloudsql.googleapis.com/database/mysql/innodb/deadlocks_count",
+ "cloudsql.googleapis.com/database/mysql/innodb/row_lock_waits_count",
+ "cloudsql.googleapis.com/database/disk/read_ops_count",
+ "cloudsql.googleapis.com/database/disk/write_ops_count"
+ ]
+ },
+ "logs": {
+ "sinkFilter": "resource.type=\"cloudsql_database\" AND (logName:\"/logs/cloudsql.googleapis.com%2Fmysql\" OR (log_id(\"cloudaudit.googleapis.com/data_access\") AND protoPayload.request.@type=\"type.googleapis.com/google.cloud.sql.audit.v1.MysqlAuditEntry\"))"
+ }
+ }
},
"assets": {
"dashboards": [
diff --git a/pkg/modules/cloudintegration/implcloudintegration/fs/definitions/gcp/cloudsql_postgres/integration.json b/pkg/modules/cloudintegration/implcloudintegration/fs/definitions/gcp/cloudsql_postgres/integration.json
index d2c0ec80ae..516cdbccd9 100644
--- a/pkg/modules/cloudintegration/implcloudintegration/fs/definitions/gcp/cloudsql_postgres/integration.json
+++ b/pkg/modules/cloudintegration/implcloudintegration/fs/definitions/gcp/cloudsql_postgres/integration.json
@@ -5,7 +5,7 @@
"overview": "file://overview.md",
"supportedSignals": {
"metrics": true,
- "logs": false
+ "logs": true
},
"dataCollected": {
"metrics": [
@@ -75,6 +75,12 @@
"type": "Gauge",
"description": ""
},
+ {
+ "name": "cloudsql.googleapis.com/database/postgresql/insights/aggregate/io_time",
+ "unit": "Microseconds",
+ "type": "Sum",
+ "description": ""
+ },
{
"name": "cloudsql.googleapis.com/database/postgresql/insights/perquery/execution_time",
"unit": "Microseconds",
@@ -88,10 +94,43 @@
"description": ""
}
],
- "logs": []
+ "logs": [
+ {
+ "name": "Project",
+ "path": "gcp.project",
+ "type": "string"
+ },
+ {
+ "name": "Resource Type",
+ "path": "gcp.resource_type",
+ "type": "string"
+ }
+ ]
},
"telemetryCollectionStrategy": {
- "gcp": {}
+ "gcp": {
+ "metrics": {
+ "metricNames": [
+ "cloudsql.googleapis.com/database/up",
+ "cloudsql.googleapis.com/database/cpu/utilization",
+ "cloudsql.googleapis.com/database/memory/utilization",
+ "cloudsql.googleapis.com/database/memory/usage",
+ "cloudsql.googleapis.com/database/disk/bytes_used",
+ "cloudsql.googleapis.com/database/postgresql/num_backends",
+ "cloudsql.googleapis.com/database/postgresql/num_backends_by_state",
+ "cloudsql.googleapis.com/database/postgresql/transaction_count",
+ "cloudsql.googleapis.com/database/postgresql/deadlock_count",
+ "cloudsql.googleapis.com/database/postgresql/vacuum/oldest_transaction_age",
+ "cloudsql.googleapis.com/database/postgresql/insights/aggregate/execution_time",
+ "cloudsql.googleapis.com/database/postgresql/insights/aggregate/io_time",
+ "cloudsql.googleapis.com/database/postgresql/insights/perquery/execution_time",
+ "cloudsql.googleapis.com/database/postgresql/replication/replica_byte_lag"
+ ]
+ },
+ "logs": {
+ "sinkFilter": "resource.type=\"cloudsql_database\" AND (logName:\"/logs/cloudsql.googleapis.com%2Fpostgres\" OR (log_id(\"cloudaudit.googleapis.com/data_access\") AND protoPayload.request.@type=\"type.googleapis.com/google.cloud.sql.audit.v1.PgAuditEntry\"))"
+ }
+ }
},
"assets": {
"dashboards": [
diff --git a/pkg/modules/cloudintegration/implcloudintegration/fs/definitions/gcp/cloudstorage/integration.json b/pkg/modules/cloudintegration/implcloudintegration/fs/definitions/gcp/cloudstorage/integration.json
index 462970807d..a432e82500 100644
--- a/pkg/modules/cloudintegration/implcloudintegration/fs/definitions/gcp/cloudstorage/integration.json
+++ b/pkg/modules/cloudintegration/implcloudintegration/fs/definitions/gcp/cloudstorage/integration.json
@@ -5,7 +5,7 @@
"overview": "file://overview.md",
"supportedSignals": {
"metrics": true,
- "logs": false
+ "logs": true
},
"dataCollected": {
"metrics": [
@@ -52,10 +52,36 @@
"description": ""
}
],
- "logs": []
+ "logs": [
+ {
+ "name": "Project",
+ "path": "gcp.project",
+ "type": "string"
+ },
+ {
+ "name": "Resource Type",
+ "path": "gcp.resource_type",
+ "type": "string"
+ }
+ ]
},
"telemetryCollectionStrategy": {
- "gcp": {}
+ "gcp": {
+ "metrics": {
+ "metricNames": [
+ "storage.googleapis.com/storage/v2/total_bytes",
+ "storage.googleapis.com/storage/v2/total_count",
+ "storage.googleapis.com/storage/v2/deleted_bytes",
+ "storage.googleapis.com/api/request_count",
+ "storage.googleapis.com/network/received_bytes_count",
+ "storage.googleapis.com/network/sent_bytes_count",
+ "storage.googleapis.com/authn/authentication_count"
+ ]
+ },
+ "logs": {
+ "sinkFilter": "resource.type=\"gcs_bucket\""
+ }
+ }
},
"assets": {
"dashboards": [
diff --git a/pkg/modules/cloudintegration/implcloudintegration/fs/definitions/gcp/computeengine/integration.json b/pkg/modules/cloudintegration/implcloudintegration/fs/definitions/gcp/computeengine/integration.json
index 8faadc3718..d391d6a7c8 100644
--- a/pkg/modules/cloudintegration/implcloudintegration/fs/definitions/gcp/computeengine/integration.json
+++ b/pkg/modules/cloudintegration/implcloudintegration/fs/definitions/gcp/computeengine/integration.json
@@ -5,7 +5,7 @@
"overview": "file://overview.md",
"supportedSignals": {
"metrics": true,
- "logs": false
+ "logs": true
},
"dataCollected": {
"metrics": [
@@ -106,10 +106,45 @@
"description": ""
}
],
- "logs": []
+ "logs": [
+ {
+ "name": "Project",
+ "path": "gcp.project",
+ "type": "string"
+ },
+ {
+ "name": "Resource Type",
+ "path": "gcp.resource_type",
+ "type": "string"
+ }
+ ]
},
"telemetryCollectionStrategy": {
- "gcp": {}
+ "gcp": {
+ "metrics": {
+ "metricNames": [
+ "compute.googleapis.com/instance/uptime_total",
+ "compute.googleapis.com/instance/cpu/utilization",
+ "compute.googleapis.com/guest/memory/bytes_used",
+ "compute.googleapis.com/guest/memory/percent_used",
+ "agent.googleapis.com/memory/percent_used",
+ "compute.googleapis.com/instance/disk/average_io_latency",
+ "compute.googleapis.com/instance/disk/read_bytes_count",
+ "compute.googleapis.com/instance/disk/write_bytes_count",
+ "compute.googleapis.com/instance/disk/read_ops_count",
+ "compute.googleapis.com/instance/disk/write_ops_count",
+ "compute.googleapis.com/instance/disk/performance_status",
+ "compute.googleapis.com/instance/network/received_bytes_count",
+ "compute.googleapis.com/instance/network/sent_bytes_count",
+ "compute.googleapis.com/instance/network/received_packets_count",
+ "compute.googleapis.com/instance/network/sent_packets_count",
+ "compute.googleapis.com/firewall/dropped_packets_count"
+ ]
+ },
+ "logs": {
+ "sinkFilter": "resource.type=\"gce_instance\" OR protoPayload.serviceName=\"compute.googleapis.com\""
+ }
+ }
},
"assets": {
"dashboards": [
diff --git a/pkg/modules/cloudintegration/implcloudintegration/fs/definitions/gcp/gke/integration.json b/pkg/modules/cloudintegration/implcloudintegration/fs/definitions/gcp/gke/integration.json
index 277df67473..534903d0a9 100644
--- a/pkg/modules/cloudintegration/implcloudintegration/fs/definitions/gcp/gke/integration.json
+++ b/pkg/modules/cloudintegration/implcloudintegration/fs/definitions/gcp/gke/integration.json
@@ -5,7 +5,7 @@
"overview": "file://overview.md",
"supportedSignals": {
"metrics": true,
- "logs": false
+ "logs": true
},
"dataCollected": {
"metrics": [
@@ -106,10 +106,45 @@
"description": ""
}
],
- "logs": []
+ "logs": [
+ {
+ "name": "Project",
+ "path": "gcp.project",
+ "type": "string"
+ },
+ {
+ "name": "Resource Type",
+ "path": "gcp.resource_type",
+ "type": "string"
+ }
+ ]
},
"telemetryCollectionStrategy": {
- "gcp": {}
+ "gcp": {
+ "metrics": {
+ "metricNames": [
+ "kubernetes.io/container/cpu/limit_utilization",
+ "kubernetes.io/container/cpu/request_utilization",
+ "kubernetes.io/container/memory/limit_utilization",
+ "kubernetes.io/container/memory/request_utilization",
+ "kubernetes.io/container/restart_count",
+ "kubernetes.io/pod/latencies/pod_first_ready",
+ "kubernetes.io/pod/network/received_bytes_count",
+ "kubernetes.io/pod/network/sent_bytes_count",
+ "kubernetes.io/pod/volume/utilization",
+ "kubernetes.io/node/cpu/allocatable_utilization",
+ "kubernetes.io/node/memory/allocatable_utilization",
+ "kubernetes.io/node/ephemeral_storage/used_bytes",
+ "kubernetes.io/node/ephemeral_storage/allocatable_bytes",
+ "kubernetes.io/node/network/received_bytes_count",
+ "kubernetes.io/node/network/sent_bytes_count",
+ "kubernetes.io/node/status_condition"
+ ]
+ },
+ "logs": {
+ "sinkFilter": "resource.type=(\"k8s_container\" OR \"k8s_pod\" OR \"k8s_node\" OR \"k8s_cluster\" OR \"k8s_control_plane_component\" OR \"gke_cluster\" OR \"gke_nodepool\")"
+ }
+ }
},
"assets": {
"dashboards": [
diff --git a/pkg/modules/cloudintegration/implcloudintegration/fs/definitions/gcp/memorystore_redis/integration.json b/pkg/modules/cloudintegration/implcloudintegration/fs/definitions/gcp/memorystore_redis/integration.json
index b0d76c2add..138c1b99e1 100644
--- a/pkg/modules/cloudintegration/implcloudintegration/fs/definitions/gcp/memorystore_redis/integration.json
+++ b/pkg/modules/cloudintegration/implcloudintegration/fs/definitions/gcp/memorystore_redis/integration.json
@@ -5,7 +5,7 @@
"overview": "file://overview.md",
"supportedSignals": {
"metrics": true,
- "logs": false
+ "logs": true
},
"dataCollected": {
"metrics": [
@@ -76,10 +76,40 @@
"description": ""
}
],
- "logs": []
+ "logs": [
+ {
+ "name": "Project",
+ "path": "gcp.project",
+ "type": "string"
+ },
+ {
+ "name": "Resource Type",
+ "path": "gcp.resource_type",
+ "type": "string"
+ }
+ ]
},
"telemetryCollectionStrategy": {
- "gcp": {}
+ "gcp": {
+ "metrics": {
+ "metricNames": [
+ "redis.googleapis.com/server/uptime",
+ "redis.googleapis.com/stats/memory/usage_ratio",
+ "redis.googleapis.com/stats/memory/system_memory_usage_ratio",
+ "redis.googleapis.com/stats/cpu_utilization_main_thread",
+ "redis.googleapis.com/clients/connected",
+ "redis.googleapis.com/stats/cache_hit_ratio",
+ "redis.googleapis.com/keyspace/keys",
+ "redis.googleapis.com/commands/usec_per_call",
+ "redis.googleapis.com/commands/calls",
+ "redis.googleapis.com/clients/blocked",
+ "redis.googleapis.com/stats/reject_connections_count"
+ ]
+ },
+ "logs": {
+ "sinkFilter": "resource.type=\"redis_instance\" OR (protoPayload.serviceName=\"redis.googleapis.com\" AND protoPayload.methodName:\".CloudRedis.\") OR (log_id(\"maintenance.googleapis.com/activity_log\") AND labels.activity_type_name:\"CloudRedisRollout\")"
+ }
+ }
},
"assets": {
"dashboards": [
diff --git a/pkg/types/cloudintegrationtypes/cloudprovider_gcp.go b/pkg/types/cloudintegrationtypes/cloudprovider_gcp.go
index 0095db65ed..afde8c615b 100644
--- a/pkg/types/cloudintegrationtypes/cloudprovider_gcp.go
+++ b/pkg/types/cloudintegrationtypes/cloudprovider_gcp.go
@@ -24,7 +24,19 @@ type GCPConnectionArtifact struct{}
type GCPIntegrationConfig struct{}
-type GCPTelemetryCollectionStrategy struct{}
+type GCPTelemetryCollectionStrategy struct {
+ Metrics *GCPMetricsCollectionStrategy `json:"metrics" required:"false" nullable:"false"`
+ Logs *GCPLogsCollectionStrategy `json:"logs" required:"false" nullable:"false"`
+}
+
+type GCPMetricsCollectionStrategy struct {
+ MetricNames []string `json:"metricNames" required:"true" nullable:"false"`
+}
+
+type GCPLogsCollectionStrategy struct {
+ // Cloud Logging filter for the log sink, https://cloud.google.com/logging/docs/view/logging-query-language
+ SinkFilter string `json:"sinkFilter" required:"true"`
+}
type GCPServiceConfig struct {
Logs *GCPServiceLogsConfig `json:"logs,omitempty" required:"false"`
diff --git a/tests/fixtures/cloudintegrations.py b/tests/fixtures/cloudintegrations.py
index ffc752c427..ee0caaced2 100644
--- a/tests/fixtures/cloudintegrations.py
+++ b/tests/fixtures/cloudintegrations.py
@@ -49,9 +49,6 @@ class ProviderAccountSpec:
class ProviderServiceSpec:
provider: str
service_id: str
- # GCP ships every service with supportedSignals.logs false, so a logs block
- # is neither required on write nor persisted.
- supports_logs: bool
account_config: dict
# id shown in parametrized test names; defaults to the provider slug.
id: str = field(default="")
@@ -61,10 +58,12 @@ class ProviderServiceSpec:
object.__setattr__(self, "id", self.provider)
def build_service_config(self, metrics_enabled: bool, logs_enabled: bool | None = None) -> dict:
- config: dict = {"metrics": {"enabled": metrics_enabled}}
- if self.supports_logs:
- config["logs"] = {"enabled": metrics_enabled if logs_enabled is None else logs_enabled}
- return {self.provider: config}
+ return {
+ self.provider: {
+ "metrics": {"enabled": metrics_enabled},
+ "logs": {"enabled": metrics_enabled if logs_enabled is None else logs_enabled},
+ }
+ }
@pytest.fixture(scope="function")
diff --git a/tests/integration/tests/cloudintegrations/05_services.py b/tests/integration/tests/cloudintegrations/05_services.py
index cfbbd8dfeb..c7407e982d 100644
--- a/tests/integration/tests/cloudintegrations/05_services.py
+++ b/tests/integration/tests/cloudintegrations/05_services.py
@@ -19,14 +19,12 @@ logger = setup_logger(__name__)
AWS_SERVICE_SPEC = ProviderServiceSpec(
provider="aws",
service_id="rds",
- supports_logs=True,
account_config={"aws": {"deploymentRegion": "us-east-1", "regions": ["us-east-1"]}},
)
GCP_SERVICE_SPEC = ProviderServiceSpec(
provider="gcp",
service_id="cloudsql_postgres",
- supports_logs=False,
account_config={
"gcp": {
"deploymentProjectId": "signoz-test-project",
@@ -162,7 +160,7 @@ def test_get_service_details_without_account(
assert data["cloudIntegrationService"] is None, "cloudIntegrationService should be null without account context"
assert data["supportedSignals"]["metrics"] is True, "metrics should be a supported signal"
- assert data["supportedSignals"]["logs"] is spec.supports_logs, f"logs support should be {spec.supports_logs} for {spec.provider}"
+ assert data["supportedSignals"]["logs"] is True, "logs should be a supported signal"
@provider_spec
@@ -253,10 +251,7 @@ def test_update_service_config(
assert svc["config"][spec.provider]["metrics"]["enabled"] is True, "metrics should be enabled"
assert svc["cloudIntegrationId"] == account_id, "cloudIntegrationId should match the account"
- if spec.supports_logs:
- assert svc["config"][spec.provider]["logs"]["enabled"] is True, "logs should be enabled"
- else:
- assert svc["config"][spec.provider].get("logs") is None, f"logs should not be stored for {spec.provider}, got: {svc['config'][spec.provider]}"
+ assert svc["config"][spec.provider]["logs"]["enabled"] is True, "logs should be enabled"
@provider_spec
@@ -307,8 +302,7 @@ def test_update_service_config_disable(
assert svc is not None, "cloudIntegrationService should still be present after disable"
assert svc["config"][spec.provider]["metrics"]["enabled"] is False, "metrics should be disabled"
- if spec.supports_logs:
- assert svc["config"][spec.provider]["logs"]["enabled"] is False, "logs should be disabled"
+ assert svc["config"][spec.provider]["logs"]["enabled"] is False, "logs should be disabled"
@provider_spec
@@ -337,7 +331,7 @@ def test_update_gcp_service_without_metrics_config(
get_token: Callable[[str, str], str],
create_cloud_integration_account: Callable,
) -> None:
- """GCP services support metrics only, so a config omitting metrics is rejected."""
+ """A GCP config omitting metrics is rejected."""
admin_token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
account = create_cloud_integration_account(admin_token, "gcp", config=GCP_SERVICE_SPEC.account_config)