Skip to content

SigNoz is an open-source, OpenTelemetry-native observability platform for your team and their AI agents. Get logs, metrics, and traces in one tool with features like APM, distributed tracing, log management, infra monitoring, etc. Combined with SigNoz MCP and a native AI teammate (in SigNoz Cloud) it helps you build more resilient apps.

26.0 KB
Raw Blame
import datetime
import json
from abc import ABC
from collections.abc import Callable, Generator
from http import HTTPStatus
from typing import Any, Literal

import numpy as np
import pytest
import requests
from ksuid import KsuidMs

from fixtures import types
from fixtures.fingerprint import LogsOrTracesFingerprint
from fixtures.time import parse_timestamp


class LogsResource(ABC):
    labels: str
    fingerprint: str
    seen_at_ts_bucket_start: np.int64

    def __init__(
        self,
        labels: dict[str, str],
        fingerprint: str,
        seen_at_ts_bucket_start: np.int64,
    ) -> None:
        self.labels = json.dumps(labels, separators=(",", ":"))  # clickhouse treats {"a": "b"} differently from {"a":"b"}. In the first case it is not able to run json functions
        self.fingerprint = fingerprint
        self.seen_at_ts_bucket_start = seen_at_ts_bucket_start

    def np_arr(self) -> np.array:
        return np.array(
            [
                self.labels,
                self.fingerprint,
                self.seen_at_ts_bucket_start,
            ]
        )


class LogsResourceOrAttributeKeys(ABC):
    name: str
    datatype: str

    def __init__(self, name: str, datatype: str) -> None:
        self.name = name
        self.datatype = datatype

    def np_arr(self) -> np.array:
        return np.array([self.name, self.datatype])


class LogsTagAttributes(ABC):
    unix_milli: np.int64
    tag_key: str
    tag_type: str
    tag_data_type: str
    string_value: str
    number_value: np.float64

    def __init__(
        self,
        timestamp: datetime.datetime,
        tag_key: str,
        tag_type: str,
        tag_data_type: str,
        string_value: str | None,
        number_value: np.float64,
    ) -> None:
        self.unix_milli = np.int64(int(timestamp.timestamp() * 1e3))
        self.tag_key = tag_key
        self.tag_type = tag_type
        self.tag_data_type = tag_data_type
        self.string_value = string_value or ""
        self.number_value = number_value

    def np_arr(self) -> np.array:
        return np.array(
            [
                self.unix_milli,
                self.tag_key,
                self.tag_type,
                self.tag_data_type,
                self.string_value,
                self.number_value,
            ]
        )


class Logs(ABC):
    ts_bucket_start: np.uint64
    resource_fingerprint: str
    timestamp: np.uint64
    observed_timestamp: np.uint64
    id: str
    trace_id: str
    span_id: str
    trace_flags: np.uint32
    severity_text: str
    severity_number: np.uint8
    body: str
    attributes_string: dict[str, str]
    attributes_number: dict[str, np.float64]
    attributes_bool: dict[str, bool]
    resources_string: dict[str, str]
    resource_json: dict[str, str]
    scope_name: str
    scope_version: str
    scope_string: dict[str, str]

    resource: list[LogsResource]
    tag_attributes: list[LogsTagAttributes]
    resource_keys: list[LogsResourceOrAttributeKeys]
    attribute_keys: list[LogsResourceOrAttributeKeys]

    def __init__(
        self,
        timestamp: datetime.datetime | None = None,
        resources: dict[str, Any] = {},
        attributes: dict[str, Any] = {},
        body: str = "default body",
        body_v2: str | None = None,
        body_promoted: str | None = None,
        severity_text: str = "INFO",
        trace_id: str = "",
        span_id: str = "",
        trace_flags: np.uint32 = 0,
        scope_name: str = "",
        scope_version: str = "",
        scope_attributes: dict[str, str] = {},
        resource_write_mode: Literal["legacy_only", "dual_write"] = "dual_write",
    ) -> None:
        if timestamp is None:
            timestamp = datetime.datetime.now()
        self.tag_attributes = []
        self.attribute_keys = []
        self.resource_keys = []

        # Convert timestamp to uint64 nanoseconds
        self.timestamp = np.uint64(int(timestamp.timestamp() * 1e9))
        self.observed_timestamp = self.timestamp

        # Calculate ts_bucket_start (30mins bucket)
        # Round down to nearest 30-minute interval
        minute = timestamp.minute
        if minute < 30:
            bucket_minute = 0
        else:
            bucket_minute = 30

        bucket_start = timestamp.replace(minute=bucket_minute, second=0, microsecond=0)
        self.ts_bucket_start = np.uint64(int(bucket_start.timestamp()))

        # Generate ksuid by using the timestamp
        self.id = str(KsuidMs(datetime=timestamp))

        # Initialize trace fields
        self.trace_id = trace_id
        self.span_id = span_id
        self.trace_flags = trace_flags

        # Set severity fields
        self.severity_text = severity_text
        self.severity_number = self._get_severity_number(severity_text)

        # Set body
        self.body = body

        # Set body_v2 - if body is JSON, parse and stringify it, otherwise use empty string
        # ClickHouse accepts String input for JSON column
        if body_v2 is not None:
            self.body_v2 = body_v2
        else:
            # Try to parse body as JSON; if successful use it directly,
            # otherwise wrap as {"message": body} matching the normalize operator behavior.
            try:
                json.loads(body)
                self.body_v2 = body
            except (json.JSONDecodeError, TypeError):
                self.body_v2 = json.dumps({"message": body})

        # Set body_promoted - must be valid JSON
        # Tests will explicitly pass promoted column's content, but we validate it
        if body_promoted is not None:
            # Validate that it's valid JSON
            try:
                json.loads(body_promoted)
                self.body_promoted = body_promoted
            except (json.JSONDecodeError, TypeError):
                # If invalid, default to empty JSON object
                self.body_promoted = "{}"
        else:
            # Default to empty JSON object (valid JSON)
            self.body_promoted = "{}"

        # Process resources and attributes
        self.resources_string = {k: str(v) for k, v in resources.items()}
        self.resource_json = {} if resource_write_mode == "legacy_only" else dict(self.resources_string)
        for k, v in self.resources_string.items():
            self.tag_attributes.append(
                LogsTagAttributes(
                    timestamp=timestamp,
                    tag_key=k,
                    tag_type="resource",
                    tag_data_type="string",
                    string_value=v,
                    number_value=None,
                )
            )
            self.resource_keys.append(LogsResourceOrAttributeKeys(name=k, datatype="string"))

        # Calculate resource fingerprint
        self.resource_fingerprint = LogsOrTracesFingerprint(self.resources_string).calculate()

        # Process attributes by type
        self.attributes_string = {}
        self.attributes_number = {}
        self.attributes_bool = {}

        for k, v in attributes.items():
            if isinstance(v, bool):
                self.attributes_bool[k] = v
                self.tag_attributes.append(
                    LogsTagAttributes(
                        timestamp=timestamp,
                        tag_key=k,
                        tag_type="tag",
                        tag_data_type="bool",
                        string_value=None,
                        number_value=None,
                    )
                )
                self.attribute_keys.append(LogsResourceOrAttributeKeys(name=k, datatype="bool"))
            elif isinstance(v, (int, float)):
                self.attributes_number[k] = np.float64(v)
                self.tag_attributes.append(
                    LogsTagAttributes(
                        timestamp=timestamp,
                        tag_key=k,
                        tag_type="tag",
                        tag_data_type="float64",
                        string_value=None,
                        number_value=np.float64(v),
                    )
                )
                self.attribute_keys.append(LogsResourceOrAttributeKeys(name=k, datatype="float64"))
            else:
                self.attributes_string[k] = str(v)
                self.tag_attributes.append(
                    LogsTagAttributes(
                        timestamp=timestamp,
                        tag_key=k,
                        tag_type="tag",
                        tag_data_type="string",
                        string_value=str(v),
                        number_value=None,
                    )
                )
                self.attribute_keys.append(LogsResourceOrAttributeKeys(name=k, datatype="string"))

        # Initialize scope fields
        self.scope_name = scope_name
        self.scope_version = scope_version
        self.scope_string = {k: str(v) for k, v in scope_attributes.items()}
        for k, v in self.scope_string.items():
            self.tag_attributes.append(
                LogsTagAttributes(
                    timestamp=timestamp,
                    tag_key=k,
                    tag_type="scope",
                    tag_data_type="string",
                    string_value=v,
                    number_value=None,
                )
            )

        self.resource = []
        self.resource.append(
            LogsResource(
                labels=self.resources_string,
                fingerprint=self.resource_fingerprint,
                seen_at_ts_bucket_start=self.ts_bucket_start,
            )
        )

        # Log fields (severity)
        self.tag_attributes.append(
            LogsTagAttributes(
                timestamp=timestamp,
                tag_key="severity_text",
                tag_type="logfield",
                tag_data_type="string",
                string_value=self.severity_text,
                number_value=None,
            )
        )
        self.attribute_keys.append(LogsResourceOrAttributeKeys(name="severity_text", datatype="string"))

        self.tag_attributes.append(
            LogsTagAttributes(
                timestamp=timestamp,
                tag_key="severity_number",
                tag_type="logfield",
                tag_data_type="float64",
                string_value=None,
                number_value=float(self.severity_number),
            )
        )
        self.attribute_keys.append(LogsResourceOrAttributeKeys(name="severity_number", datatype="float64"))

    def _get_severity_number(self, severity_text: str) -> np.uint8:
        severity_map = {
            "TRACE": 1,
            "DEBUG": 5,
            "INFO": 9,
            "WARN": 13,
            "ERROR": 17,
            "FATAL": 21,
        }

        return np.uint8(severity_map.get(severity_text.upper(), 9))  # Default to INFO

    def np_arr(self) -> np.array:
        return np.array(
            [
                self.ts_bucket_start,
                self.resource_fingerprint,
                self.timestamp,
                self.observed_timestamp,
                self.id,
                self.trace_id,
                self.span_id,
                self.trace_flags,
                self.severity_text,
                self.severity_number,
                self.body,
                self.body_v2,
                self.body_promoted,
                self.attributes_string,
                self.attributes_number,
                self.attributes_bool,
                self.resources_string,
                self.scope_name,
                self.scope_version,
                self.scope_string,
                self.resource_json,
            ]
        )

    @classmethod
    def from_dict(
        cls,
        data: dict,
    ) -> "Logs":
        # parse timestamp from iso format
        timestamp = parse_timestamp(data["timestamp"])
        return cls(
            timestamp=timestamp,
            resources=data.get("resources", {}),
            attributes=data.get("attributes", {}),
            body=data["body"],
            severity_text=data.get("severity_text", "INFO"),
        )

    @classmethod
    def load_from_file(
        cls,
        file_path: str,
        base_time: datetime.datetime | None = None,
    ) -> list["Logs"]:
        """Load logs from a JSONL file."""

        data_list = []
        with open(file_path, encoding="utf-8") as f:
            for line in f:
                line = line.strip()
                if not line:
                    continue
                data_list.append(json.loads(line))

        # If base_time provided, calculate time offset
        time_offset = datetime.timedelta(0)
        if base_time is not None:
            # Find earliest timestamp
            earliest = None
            for data in data_list:
                ts = parse_timestamp(data["timestamp"])
                if earliest is None or ts < earliest:
                    earliest = ts
            if earliest is not None:
                time_offset = base_time - earliest

        logs = []
        for data in data_list:
            original_ts = parse_timestamp(data["timestamp"])
            adjusted_ts = original_ts + time_offset
            data["timestamp"] = adjusted_ts.isoformat()
            logs.append(cls.from_dict(data))

        return logs


def insert_logs_to_clickhouse(conn, logs: list[Logs]) -> None:
    """
    Insert logs into ClickHouse tables following the same logic as the Go exporter.
    Handles insertion into:
    - distributed_logs_v2 (main logs table)
    - distributed_logs_v2_resource (resource fingerprints)
    - distributed_tag_attributes_v2 (tag attributes)
    - distributed_logs_attribute_keys (attribute keys)
    - distributed_logs_resource_keys (resource keys)

    Pure function so the seeder container can reuse the exact insert path
    used by the pytest fixture. `conn` is a clickhouse-connect Client.
    """
    resources: list[LogsResource] = []
    for log in logs:
        resources.extend(log.resource)

    if len(resources) > 0:
        conn.insert(
            database="signoz_logs",
            table="distributed_logs_v2_resource",
            data=[resource.np_arr() for resource in resources],
            column_names=[
                "labels",
                "fingerprint",
                "seen_at_ts_bucket_start",
            ],
        )

    tag_attributes: list[LogsTagAttributes] = []
    for log in logs:
        tag_attributes.extend(log.tag_attributes)

    if len(tag_attributes) > 0:
        conn.insert(
            database="signoz_logs",
            table="distributed_tag_attributes_v2",
            data=[tag_attribute.np_arr() for tag_attribute in tag_attributes],
        )

    attribute_keys: list[LogsResourceOrAttributeKeys] = []
    for log in logs:
        attribute_keys.extend(log.attribute_keys)

    if len(attribute_keys) > 0:
        conn.insert(
            database="signoz_logs",
            table="distributed_logs_attribute_keys",
            data=[attribute_key.np_arr() for attribute_key in attribute_keys],
        )

    resource_keys: list[LogsResourceOrAttributeKeys] = []
    for log in logs:
        resource_keys.extend(log.resource_keys)

    if len(resource_keys) > 0:
        conn.insert(
            database="signoz_logs",
            table="distributed_logs_resource_keys",
            data=[resource_key.np_arr() for resource_key in resource_keys],
        )

    all_column_names = [
        "ts_bucket_start",
        "resource_fingerprint",
        "timestamp",
        "observed_timestamp",
        "id",
        "trace_id",
        "span_id",
        "trace_flags",
        "severity_text",
        "severity_number",
        "body",
        "body_v2",
        "body_promoted",
        "attributes_string",
        "attributes_number",
        "attributes_bool",
        "resources_string",
        "scope_name",
        "scope_version",
        "scope_string",
        "resource",
    ]

    result = conn.query("SELECT count() FROM system.columns WHERE database = 'signoz_logs' AND table = 'logs_v2' AND name = 'body_v2'")
    has_json_body = result.result_rows[0][0] > 0

    if has_json_body:
        column_names = all_column_names
        data = [log.np_arr() for log in logs]
    else:
        json_body_cols = {"body_v2", "body_promoted"}
        keep_indices = [i for i, c in enumerate(all_column_names) if c not in json_body_cols]
        column_names = [all_column_names[i] for i in keep_indices]
        data = [log.np_arr()[keep_indices] for log in logs]

    conn.insert(
        database="signoz_logs",
        table="distributed_logs_v2",
        data=data,
        column_names=column_names,
    )


_LOGS_TABLES_TO_TRUNCATE = [
    "logs_v2",
    "logs_v2_resource",
    "tag_attributes_v2",
    "logs_attribute_keys",
    "logs_resource_keys",
]


def truncate_logs_tables(conn, cluster: str) -> None:
    """Truncate all logs tables. Used by the pytest fixture teardown and by
    the seeder's DELETE /telemetry/logs endpoint."""
    for table in _LOGS_TABLES_TO_TRUNCATE:
        conn.query(f"TRUNCATE TABLE signoz_logs.{table} ON CLUSTER '{cluster}' SYNC")


@pytest.fixture(name="insert_logs", scope="function")
def insert_logs(
    clickhouse: types.TestContainerClickhouse,
) -> Generator[Callable[[list[Logs]], None], Any]:
    def _insert_logs(logs: list[Logs]) -> None:
        insert_logs_to_clickhouse(clickhouse.conn, logs)

    yield _insert_logs

    truncate_logs_tables(
        clickhouse.conn,
        clickhouse.env["SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_CLUSTER"],
    )


@pytest.fixture(name="insert_logs_with_unknown_keys", scope="function")
def insert_logs_with_unknown_keys(
    insert_logs: Callable[[list[Logs]], None],
) -> Callable[[list[Logs], set[str]], None]:
    """Insert logs while treating the named ATTRIBUTE keys as absent from field-key
    metadata.

    Each named key keeps its value in the row's attribute maps but has its attribute
    metadata dropped (attribute_keys -> distributed_logs_attribute_keys and the tag
    entry in distributed_tag_attributes_v2), so the querier can't resolve it and falls
    back to the unknown-key synthesize path. This is the only way to drive the
    synthesized attribute-map branches with real data — a normally-seeded attribute is
    always registered in metadata and so never counts as "unknown".

    Only attribute-side metadata is stripped: resource keys always keep their metadata
    (resource metadata is assumed never missing), so passing a resource key here has no
    effect. Teardown/truncation is inherited from the underlying insert_logs fixture.
    """

    def _insert(logs: list[Logs], unknown_keys: set[str]) -> None:
        for log in logs:
            log.attribute_keys = [k for k in log.attribute_keys if k.name not in unknown_keys]
            log.tag_attributes = [t for t in log.tag_attributes if not (t.tag_key in unknown_keys and t.tag_type == "tag")]
        insert_logs(logs)

    return _insert


@pytest.fixture(name="materialize_log_field", scope="function")
def materialize_log_field(
    signoz: types.SigNoz,
) -> Generator[Callable[[str, str, str, str], None]]:
    mat_fields: list[tuple[str, str, str]] = []

    def _materialize_log_field(
        token: str,
        name: str,
        data_type: str,
        field_type: str,
    ) -> None:
        response = requests.post(
            signoz.self.host_configs["8080"].get("/api/v1/logs/fields"),
            headers={"authorization": f"Bearer {token}"},
            json={
                "name": name,
                "dataType": data_type,
                "type": field_type,
                "selected": True,
            },
            timeout=10,
        )
        assert response.status_code == HTTPStatus.OK, f"Failed to materialize log field {name}: {response.status_code} {response.text}"
        mat_fields.append((field_type, data_type, name))

    yield _materialize_log_field

    for mat_field_type, mat_field_data_type, mat_field_name in mat_fields:
        mat_field_name = mat_field_name.replace(".", "$$")
        if mat_field_type == "resources":
            mat_field_type = "resource"
        field = f"{mat_field_type}_{mat_field_data_type}_{mat_field_name}"
        signoz.telemetrystore.conn.query(f"ALTER TABLE signoz_logs.logs_v2 ON CLUSTER '{signoz.telemetrystore.env['SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_CLUSTER']}' DROP INDEX IF EXISTS {field}_idx")
        for table in ["logs_v2", "distributed_logs_v2"]:
            signoz.telemetrystore.conn.query(f"ALTER TABLE signoz_logs.{table} ON CLUSTER '{signoz.telemetrystore.env['SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_CLUSTER']}' DROP COLUMN IF EXISTS {field}")
            signoz.telemetrystore.conn.query(f"ALTER TABLE signoz_logs.{table} ON CLUSTER '{signoz.telemetrystore.env['SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_CLUSTER']}' DROP COLUMN IF EXISTS {field}_exists")


@pytest.fixture(name="ttl_legacy_logs_v2_table_setup", scope="function")
def ttl_legacy_logs_v2_table_setup(request, signoz: types.SigNoz):
    """
    Fixture to setup and teardown legacy TTL test environment.
    It renames existing logs tables to backup names and creates new empty tables for testing.
    After the test, it restores the original tables.
    """

    # Setup code
    result = signoz.telemetrystore.conn.query(f"RENAME TABLE signoz_logs.logs_v2 TO signoz_logs.logs_v2_backup ON CLUSTER '{signoz.telemetrystore.env['SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_CLUSTER']}'").result_rows
    assert result is not None
    # Add cleanup to restore original table
    request.addfinalizer(lambda: signoz.telemetrystore.conn.query(f"RENAME TABLE signoz_logs.logs_v2_backup TO signoz_logs.logs_v2 ON CLUSTER '{signoz.telemetrystore.env['SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_CLUSTER']}'"))

    # Create new test tables
    result = signoz.telemetrystore.conn.query(
        f"""CREATE TABLE signoz_logs.logs_v2 ON CLUSTER '{signoz.telemetrystore.env["SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_CLUSTER"]}'
                                                (
                                                    `id` String,
                                                    `timestamp` UInt64 CODEC(DoubleDelta, LZ4)

                                                )
                                                ENGINE = MergeTree()
                                                ORDER BY id;"""
    ).result_rows

    assert result is not None
    # Add cleanup to drop test table
    request.addfinalizer(lambda: signoz.telemetrystore.conn.query(f"DROP TABLE IF EXISTS signoz_logs.logs_v2 ON CLUSTER '{signoz.telemetrystore.env['SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_CLUSTER']}'"))

    yield  # Test runs here


@pytest.fixture(name="ttl_legacy_logs_v2_resource_table_setup", scope="function")
def ttl_legacy_logs_v2_resource_table_setup(request, signoz: types.SigNoz):
    """
    Fixture to setup and teardown legacy TTL test environment.
    It renames existing logs tables to backup names and creates new empty tables for testing.
    After the test, it restores the original tables.
    """

    # Setup code
    result = signoz.telemetrystore.conn.query(f"RENAME TABLE signoz_logs.logs_v2_resource TO signoz_logs.logs_v2_resource_backup ON CLUSTER '{signoz.telemetrystore.env['SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_CLUSTER']}'").result_rows
    assert result is not None
    # Add cleanup to restore original table
    request.addfinalizer(lambda: signoz.telemetrystore.conn.query(f"RENAME TABLE signoz_logs.logs_v2_resource_backup TO signoz_logs.logs_v2_resource ON CLUSTER '{signoz.telemetrystore.env['SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_CLUSTER']}'"))

    # Create new test tables
    result = signoz.telemetrystore.conn.query(
        f"""CREATE TABLE signoz_logs.logs_v2_resource ON CLUSTER '{signoz.telemetrystore.env["SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_CLUSTER"]}'
                                                (
                                                    `id` String,
                                                    `seen_at_ts_bucket_start` Int64 CODEC(Delta(8), ZSTD(1))
                                                )
                                                ENGINE = MergeTree()
                                                ORDER BY id;"""
    ).result_rows

    assert result is not None
    # Add cleanup to drop test table
    request.addfinalizer(lambda: signoz.telemetrystore.conn.query(f"DROP TABLE IF EXISTS signoz_logs.logs_v2_resource ON CLUSTER '{signoz.telemetrystore.env['SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_CLUSTER']}';"))

    yield  # Test runs here


@pytest.fixture(name="remove_logs_ttl_settings", scope="function")
def remove_logs_ttl_settings(signoz: types.SigNoz):
    """
    Remove TTL settings from the specified logs table.
    This function alters the table to drop any existing TTL configurations
    and resets the _retention_days default value to 0.
    """
    tables = [
        "distributed_logs_v2",
        "distributed_logs_v2_resource",
        "logs_v2",
        "logs_v2_resource",
        "logs_attribute_keys",
        "logs_resource_keys",
    ]
    for table in tables:
        try:
            # Reset _retention_days and _retention_days_cold default values to 0 for tables that have these columns
            if table in [
                "logs_v2",
                "logs_v2_resource",
                "distributed_logs_v2",
                "distributed_logs_v2_resource",
            ]:
                reset_retention_query = f"""
                ALTER TABLE signoz_logs.{table} ON CLUSTER '{signoz.telemetrystore.env["SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_CLUSTER"]}'
                MODIFY COLUMN _retention_days UInt16 DEFAULT 0
                """
                signoz.telemetrystore.conn.query(reset_retention_query)

                reset_retention_cold_query = f"""
                ALTER TABLE signoz_logs.{table} ON CLUSTER '{signoz.telemetrystore.env["SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_CLUSTER"]}'
                MODIFY COLUMN _retention_days_cold UInt16 DEFAULT 0
                """
                signoz.telemetrystore.conn.query(reset_retention_cold_query)
            else:
                alter_query = f"""
                ALTER TABLE signoz_logs.{table} ON CLUSTER '{signoz.telemetrystore.env["SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_CLUSTER"]}'
                REMOVE TTL
                """
                signoz.telemetrystore.conn.query(alter_query)
        except Exception as e:  # pylint: disable=broad-exception-caught
            print(f"Error removing TTL from table {table}: {e}")