Skip to content

The fastest, litest AI Gateway. Rust core with Python SDK. Call 100+ LLM APIs in OpenAI (or native) format with cost tracking, guardrails, load balancing, and logging [Bedrock, Azure, OpenAI, Anthropic, OpenAI, VertexAI, vLLM, Nvidia NIM]

94.1 KB
Raw Blame
import glob
import logging
import os
import re
import subprocess
import sys
import threading
from collections.abc import Mapping, Sequence
from dataclasses import dataclass
from pathlib import Path
from typing import Final, NoReturn, Optional

import pytest

sys.path.insert(
    0,
    os.path.abspath(
        os.path.join(os.path.dirname(__file__), "../../../litellm-proxy-extras")
    ),
)

from litellm_proxy_extras.utils import (
    PARTITIONED_SPEND_LOGS_PUSH_ERROR,
    ProxyExtrasDBManager,
    _redact_command_error,
    _redact_credentials,
    filter_partitioned_spend_logs_diff,
)

# Path to the migrations directory
_MIGRATIONS_DIR = os.path.abspath(
    os.path.join(
        os.path.dirname(__file__),
        "../../../litellm-proxy-extras/litellm_proxy_extras/migrations",
    )
)


def test_coordination_failure_does_not_misclassify_fatal_password_as_unreachable(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    import psycopg
    from litellm_proxy_extras.migration_lock import _coordination_failure

    database_url: Final = "postgresql://u:[email protected]:5432/db"
    monkeypatch.setenv("DATABASE_URL", database_url)
    error: Final = psycopg.OperationalError(
        'connection failed: connection to server at "127.0.0.1", port 5432 failed: '
        'FATAL:  password authentication failed for user "u"'
    )

    result: Final = _coordination_failure(database_url, error, connecting=True)

    assert result.startswith("Lost or could not establish v2 migration coordination with the database: ")
    assert "P1001" not in result


@pytest.mark.parametrize(
    ("message", "detail"),
    [
        (
            'connection failed: connection to server at "127.0.0.1", port 5432 failed: SSL error: '
            "certificate verify failed",
            "certificate verify failed",
        ),
        (
            'connection failed: connection to server at "127.0.0.1", port 5432 failed: server certificate '
            'for "db.internal" does not match host name "127.0.0.1"',
            "does not match host name",
        ),
        (
            'connection failed: connection to server at "127.0.0.1", port 5432 failed: server does not support '
            "SSL, but SSL was required",
            "SSL was required",
        ),
    ],
)
def test_coordination_failure_does_not_label_tls_failures_as_unreachable(
    monkeypatch: pytest.MonkeyPatch,
    message: str,
    detail: str,
) -> None:
    import psycopg
    from litellm_proxy_extras.migration_lock import _coordination_failure

    database_url: Final = "postgresql://u:[email protected]:5432/db"
    monkeypatch.setenv("DATABASE_URL", database_url)
    error: Final = psycopg.OperationalError(message)

    result: Final = _coordination_failure(database_url, error, connecting=True)

    assert result.startswith("Lost or could not establish v2 migration coordination with the database: ")
    assert "P1001" not in result
    assert detail in result


@pytest.mark.parametrize(
    ("database_url", "message", "host"),
    [
        (
            "postgresql://u:[email protected]:5432/db",
            'connection failed: connection to server at "certificate-db.internal" (10.0.0.5), port 5432 failed: '
            "Connection refused",
            "certificate-db.internal",
        ),
        (
            "postgresql://u:pw@SSL-primary:5432/db",
            'connection failed: connection to server at "SSL-primary" (10.0.0.6), port 5432 failed: '
            "Connection refused",
            "ssl-primary",
        ),
        (
            "postgresql://u:[email protected]:5432/db",
            "connection failed: failed to resolve host 'certificate-db.invalid': [Errno -2] "
            "Name or service not known",
            "certificate-db.invalid",
        ),
    ],
)
def test_coordination_failure_labels_unreachable_hosts_named_like_tls_markers(
    monkeypatch: pytest.MonkeyPatch,
    database_url: str,
    message: str,
    host: str,
) -> None:
    import psycopg
    from litellm_proxy_extras.migration_lock import _coordination_failure

    monkeypatch.setenv("DATABASE_URL", database_url)
    error: Final = psycopg.OperationalError(message)

    result: Final = _coordination_failure(database_url, error, connecting=True)

    assert f"P1001: Can't reach database server at `{host}:5432`" in result


def test_coordination_failure_labels_connection_refused_as_unreachable(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    import psycopg
    from litellm_proxy_extras.migration_lock import _coordination_failure

    database_url: Final = "postgresql://u:[email protected]:5432/db"
    monkeypatch.setenv("DATABASE_URL", database_url)
    error: Final = psycopg.OperationalError(
        'connection failed: connection to server at "127.0.0.1", port 5432 failed: Connection refused'
    )

    result: Final = _coordination_failure(database_url, error, connecting=True)

    assert result.startswith("Lost or could not establish v2 migration coordination with the database: ")
    assert "P1001: Can't reach database server at `127.0.0.1:5432`" in result


def test_custom_prisma_dir(monkeypatch):
    import tempfile

    # create a temp directory
    temp_dir = tempfile.mkdtemp()
    monkeypatch.setenv("LITELLM_MIGRATION_DIR", temp_dir)

    ## Check if the prisma dir is the temp directory
    assert ProxyExtrasDBManager._get_prisma_dir() == temp_dir

    ## Check if the schema.prisma file is in the temp directory
    schema_path = os.path.join(temp_dir, "schema.prisma")
    assert os.path.exists(schema_path)

    ## Check if the migrations dir is in the temp directory
    migrations_dir = os.path.join(temp_dir, "migrations")
    assert os.path.exists(migrations_dir)


class TestPermissionErrorDetection:
    """Test cases for permission error detection in Prisma migrations"""

    def test_is_permission_error_postgres_42501(self):
        """Test detection of PostgreSQL 42501 error code (insufficient privilege)"""
        error_message = "Database error code: 42501 - permission denied for table users"
        assert ProxyExtrasDBManager._is_permission_error(error_message) is True

    def test_is_permission_error_must_be_owner(self):
        """Test detection of 'must be owner of table' error"""
        error_message = "ERROR: must be owner of table my_table"
        assert ProxyExtrasDBManager._is_permission_error(error_message) is True

    def test_is_permission_error_permission_denied_schema(self):
        """Test detection of 'permission denied for schema' error"""
        error_message = "permission denied for schema public"
        assert ProxyExtrasDBManager._is_permission_error(error_message) is True

    def test_is_permission_error_permission_denied_table(self):
        """Test detection of 'permission denied for table' error"""
        error_message = "permission denied for table my_table"
        assert ProxyExtrasDBManager._is_permission_error(error_message) is True

    def test_is_permission_error_must_be_owner_schema(self):
        """Test detection of 'must be owner of schema' error"""
        error_message = "must be owner of schema public"
        assert ProxyExtrasDBManager._is_permission_error(error_message) is True

    def test_is_permission_error_case_insensitive(self):
        """Test that permission error detection is case insensitive"""
        error_message = "PERMISSION DENIED FOR TABLE my_table"
        assert ProxyExtrasDBManager._is_permission_error(error_message) is True

    def test_is_permission_error_negative(self):
        """Test that non-permission errors are not detected as permission errors"""
        error_message = "column 'id' already exists"
        assert ProxyExtrasDBManager._is_permission_error(error_message) is False


class TestIdempotentErrorDetection:
    """Test cases for idempotent error detection in Prisma migrations"""

    def test_is_idempotent_error_already_exists(self):
        """Test detection of generic 'already exists' error"""
        error_message = "object already exists"
        assert ProxyExtrasDBManager._is_idempotent_error(error_message) is True

    def test_is_idempotent_error_column_already_exists(self):
        """Test detection of 'column already exists' error"""
        error_message = "column 'email' already exists"
        assert ProxyExtrasDBManager._is_idempotent_error(error_message) is True

    def test_is_idempotent_error_duplicate_key(self):
        """Test detection of duplicate key violation error"""
        error_message = "duplicate key value violates unique constraint"
        assert ProxyExtrasDBManager._is_idempotent_error(error_message) is True

    def test_is_idempotent_error_relation_already_exists(self):
        """Test detection of 'relation already exists' error"""
        error_message = "relation 'users_pkey' already exists"
        assert ProxyExtrasDBManager._is_idempotent_error(error_message) is True

    def test_is_idempotent_error_constraint_already_exists(self):
        """Test detection of 'constraint already exists' error"""
        error_message = "constraint 'fk_user_id' already exists"
        assert ProxyExtrasDBManager._is_idempotent_error(error_message) is True


    def test_is_idempotent_error_case_insensitive(self):
        """Test that idempotent error detection is case insensitive"""
        error_message = "COLUMN 'ID' ALREADY EXISTS"
        assert ProxyExtrasDBManager._is_idempotent_error(error_message) is True

    def test_is_idempotent_error_does_not_exist(self):
        """Test detection of 'does not exist' error"""
        error_message = "ERROR: index 'idx' does not exist"
        assert ProxyExtrasDBManager._is_idempotent_error(error_message) is True

    def test_is_idempotent_error_negative(self):
        """Test that non-idempotent errors are not detected as idempotent errors"""
        error_message = "Database error code: 42501 - permission denied"
        assert ProxyExtrasDBManager._is_idempotent_error(error_message) is False


class TestErrorClassificationPriority:
    """Test cases to ensure errors are correctly classified"""

    def test_permission_error_not_classified_as_idempotent(self):
        """Ensure permission errors are not mistakenly classified as idempotent"""
        error_message = "Database error code: 42501 - must be owner of table users"
        assert ProxyExtrasDBManager._is_permission_error(error_message) is True
        assert ProxyExtrasDBManager._is_idempotent_error(error_message) is False

    def test_idempotent_error_not_classified_as_permission(self):
        """Ensure idempotent errors are not mistakenly classified as permission errors"""
        error_message = "column 'created_at' already exists"
        assert ProxyExtrasDBManager._is_idempotent_error(error_message) is True
        assert ProxyExtrasDBManager._is_permission_error(error_message) is False

    def test_unknown_error_classified_as_neither(self):
        """Ensure unknown errors are classified as neither permission nor idempotent"""
        error_message = "connection timeout"
        assert ProxyExtrasDBManager._is_permission_error(error_message) is False
        assert ProxyExtrasDBManager._is_idempotent_error(error_message) is False


def _get_all_migrations():
    """Return (migration_name, sql_content) pairs for all migrations."""
    migration_files = sorted(
        glob.glob(os.path.join(_MIGRATIONS_DIR, "*/migration.sql"))
    )
    results = []
    for path in migration_files:
        migration_name = os.path.basename(os.path.dirname(path))
        with open(path) as f:
            results.append((migration_name, f.read()))
    return results


_LINE_COMMENT = re.compile(r"--.*$")
_BLOCK_COMMENT = re.compile(r"/\*.*?\*/", re.DOTALL)

_PRE_GUARD_MIGRATIONS = frozenset({
    "20260331000000_add_prompt_environment_and_created_by",
    "20260418000000_add_adaptive_router_tables",
    "20260429161855_workflow_runs_tables",
    "20260605182307_add_timeout_to_mcp_server_table",
    "20260626120000_add_mcp_tool_search_enabled",
    "20260629000000_add_max_concurrent_requests_to_mcp_server_table",
    "20260710000000_add_dcr_bridge_to_mcp_server_table",
    "20260713230852_add_key_type_to_litellm_verification_token",
    "20260811172448_add_shadow_eval",
    "20260813180408_add_shadow_eval_direction",
    "20260814000000_add_proxy_worker_heartbeat",
    "20260817143646_add_daily_guardrail_usage_units",
    "20260818224500_add_shadow_eval_stopped_by",
    "20260819000000_shadow_eval_max_budget",
})


def _blanked_block_comments(sql):
    """`sql` with every `/* ... */` body blanked out, newlines kept so lines still count.

    Prisma opens a destructive migration with a `/* Warnings: You are about to drop the
    column ... */` header, which is prose about the statement rather than the statement.
    """
    return _BLOCK_COMMENT.sub(lambda m: re.sub(r"[^\n]", " ", m.group(0)), sql)


def _statements(sql):
    """(line_number, sql) for each line, with comments removed.

    Prisma writes its own explanations as `-- CREATE INDEX CONCURRENTLY ...`, which a
    raw-line scan reads as the statement it is describing.
    """
    return [
        (number, _LINE_COMMENT.sub("", line))
        for number, line in enumerate(_blanked_block_comments(sql).splitlines(), 1)
    ]


def _guarded_migrations(all_migrations):
    """Migrations the DDL rules apply to. Prisma checksums an applied migration, so the
    ones that predate these rules cannot be edited without breaking `migrate deploy`
    for existing installs; they are named once, and the rules bind everything after.
    """
    return [
        (name, sql) for name, sql in all_migrations if name not in _PRE_GUARD_MIGRATIONS
    ]


class TestMigrationSQLIdempotency:
    """Ensure all migration SQL files use idempotent DDL (IF [NOT] EXISTS).

    Migrations on pre-existing instances can fail when DDL statements assume
    the target object doesn't already exist (or still exists for drops).
    These tests enforce that all migrations use safe, re-runnable SQL patterns.
    """

    @pytest.fixture(scope="class")
    def all_migrations(self):
        migrations = _get_all_migrations()
        assert len(migrations) > 0, (
            f"No migrations found. "
            f"Check that _MIGRATIONS_DIR ({_MIGRATIONS_DIR}) is correct."
        )
        return migrations

    def test_create_table_uses_if_not_exists(self, all_migrations):
        """CREATE TABLE statements must use IF NOT EXISTS"""
        violations = []
        for migration_name, sql in _guarded_migrations(all_migrations):
            for line_num, line in _statements(sql):
                if re.search(
                    r"CREATE\s+TABLE\s+", line, re.IGNORECASE
                ) and not re.search(
                    r"CREATE\s+TABLE\s+IF\s+NOT\s+EXISTS", line, re.IGNORECASE
                ):
                    violations.append(f"  {migration_name}:{line_num}: {line.strip()}")
        assert (
            not violations
        ), "CREATE TABLE without IF NOT EXISTS found in migrations:\n" + "\n".join(
            violations
        )

    def test_add_column_uses_if_not_exists(self, all_migrations):
        """ADD COLUMN statements must use IF NOT EXISTS"""
        violations = []
        for migration_name, sql in _guarded_migrations(all_migrations):
            for line_num, line in _statements(sql):
                if re.search(r"ADD\s+COLUMN\s+", line, re.IGNORECASE) and not re.search(
                    r"ADD\s+COLUMN\s+IF\s+NOT\s+EXISTS", line, re.IGNORECASE
                ):
                    violations.append(f"  {migration_name}:{line_num}: {line.strip()}")
        assert not violations, (
            "ADD COLUMN without IF NOT EXISTS found in recent migrations:\n"
            + "\n".join(violations)
        )

    def test_drop_column_uses_if_exists(self, all_migrations):
        """DROP COLUMN statements must use IF EXISTS"""
        violations = []
        for migration_name, sql in _guarded_migrations(all_migrations):
            for line_num, line in _statements(sql):
                if re.search(
                    r"DROP\s+COLUMN\s+", line, re.IGNORECASE
                ) and not re.search(
                    r"DROP\s+COLUMN\s+IF\s+EXISTS", line, re.IGNORECASE
                ):
                    violations.append(f"  {migration_name}:{line_num}: {line.strip()}")
        assert (
            not violations
        ), "DROP COLUMN without IF EXISTS found in recent migrations:\n" + "\n".join(
            violations
        )

    _DROP_COLUMN_ALLOWLIST = {
        "20250918083359_drop_spec_version_column_from_mcp_table",
        "20260213170952_access_group_change_to_model_name",
        "20260224203854_add_agent_object_permissions_table",
    }

    def test_no_drop_column_statements(self, all_migrations):
        """Migrations must not drop columns — dropping columns is destructive
        and can break running application instances during rolling deploys."""
        violations = []
        for migration_name, sql in all_migrations:
            if migration_name in self._DROP_COLUMN_ALLOWLIST:
                continue
            for line_num, line in _statements(sql):
                if re.search(r"DROP\s+COLUMN", line, re.IGNORECASE):
                    violations.append(f"  {migration_name}:{line_num}: {line.strip()}")
        assert (
            not violations
        ), "DROP COLUMN found in migrations (destructive, not allowed):\n" + "\n".join(
            violations
        )

    def test_drop_index_uses_if_exists(self, all_migrations):
        """DROP INDEX statements must use IF EXISTS"""
        violations = []
        for migration_name, sql in _guarded_migrations(all_migrations):
            for line_num, line in _statements(sql):
                if re.search(r"DROP\s+INDEX\s+", line, re.IGNORECASE) and not re.search(
                    r"DROP\s+INDEX\s+IF\s+EXISTS", line, re.IGNORECASE
                ):
                    violations.append(f"  {migration_name}:{line_num}: {line.strip()}")
        assert (
            not violations
        ), "DROP INDEX without IF EXISTS found in recent migrations:\n" + "\n".join(
            violations
        )

    def test_create_index_uses_if_not_exists(self, all_migrations):
        """CREATE INDEX statements must use IF NOT EXISTS"""
        violations = []
        for migration_name, sql in _guarded_migrations(all_migrations):
            for line_num, line in _statements(sql):
                if re.search(
                    r"CREATE\s+(?:UNIQUE\s+)?INDEX\s+", line, re.IGNORECASE
                ) and not re.search(
                    r"CREATE\s+(?:UNIQUE\s+)?INDEX\s+(?:CONCURRENTLY\s+)?IF\s+NOT\s+EXISTS",
                    line,
                    re.IGNORECASE,
                ):
                    violations.append(f"  {migration_name}:{line_num}: {line.strip()}")
        assert not violations, (
            "CREATE INDEX without IF NOT EXISTS found in recent migrations:\n"
            + "\n".join(violations)
        )

    def test_rename_column_is_guarded(self, all_migrations):
        """RENAME COLUMN must be inside a DO $$ IF EXISTS block"""
        violations = []
        for migration_name, sql in _guarded_migrations(all_migrations):
            in_do_block = False
            for line_num, line in _statements(sql):
                if re.search(r"DO\s+\$\$", line, re.IGNORECASE):
                    in_do_block = True
                if re.search(r"END\s+\$\$", line, re.IGNORECASE):
                    in_do_block = False
                if (
                    re.search(r"RENAME\s+COLUMN\s+", line, re.IGNORECASE)
                    and not in_do_block
                ):
                    violations.append(f"  {migration_name}:{line_num}: {line.strip()}")
        assert not violations, (
            "RENAME COLUMN without DO $$ IF EXISTS guard found in migrations:\n"
            + "\n".join(violations)
        )

    def test_add_constraint_is_guarded(self, all_migrations):
        """ADD CONSTRAINT must be inside a DO $$ IF NOT EXISTS block"""
        violations = []
        for migration_name, sql in _guarded_migrations(all_migrations):
            in_do_block = False
            for line_num, line in _statements(sql):
                if re.search(r"DO\s+\$\$", line, re.IGNORECASE):
                    in_do_block = True
                if re.search(r"END\s+\$\$", line, re.IGNORECASE):
                    in_do_block = False
                if (
                    re.search(r"ADD\s+CONSTRAINT\s+", line, re.IGNORECASE)
                    and not in_do_block
                ):
                    violations.append(f"  {migration_name}:{line_num}: {line.strip()}")
        assert not violations, (
            "ADD CONSTRAINT without DO $$ IF NOT EXISTS guard found in migrations:\n"
            + "\n".join(violations)
        )

    def test_drop_constraint_is_guarded(self, all_migrations):
        """DROP CONSTRAINT must be inside a DO $$ IF EXISTS block"""
        violations = []
        for migration_name, sql in _guarded_migrations(all_migrations):
            in_do_block = False
            for line_num, line in _statements(sql):
                if re.search(r"DO\s+\$\$", line, re.IGNORECASE):
                    in_do_block = True
                if re.search(r"END\s+\$\$", line, re.IGNORECASE):
                    in_do_block = False
                if (
                    re.search(r"DROP\s+CONSTRAINT\s+", line, re.IGNORECASE)
                    and not in_do_block
                ):
                    violations.append(f"  {migration_name}:{line_num}: {line.strip()}")
        assert not violations, (
            "DROP CONSTRAINT without DO $$ IF EXISTS guard found in migrations:\n"
            + "\n".join(violations)
        )


class TestMigrationGuardScope:
    """The guard must ignore SQL comments, exempt only the named pre-guard migrations,
    and still fail on a new migration that uses bare DDL."""

    _NEW = "20990101000000_a_new_migration"

    def _run_rules(self, migrations):
        suite = TestMigrationSQLIdempotency()
        failures = []
        for name in (
            "test_create_table_uses_if_not_exists",
            "test_add_column_uses_if_not_exists",
            "test_create_index_uses_if_not_exists",
            "test_add_constraint_is_guarded",
        ):
            try:
                getattr(suite, name)(migrations)
            except AssertionError:
                failures.append(name)
        return failures

    def test_a_comment_describing_ddl_is_not_the_ddl(self):
        sql = '-- CREATE TABLE "Foo" (id TEXT);\n-- ADD COLUMN "bar" TEXT;\n'
        assert self._run_rules([(self._NEW, sql)]) == []

    def test_a_prisma_warning_block_is_not_the_ddl_it_describes(self):
        sql = (
            "/*\n"
            "  Warnings:\n"
            "\n"
            "  - You are about to CREATE TABLE \"Foo\" and ADD COLUMN \"bar\".\n"
            "\n"
            "*/\n"
            'CREATE TABLE IF NOT EXISTS "Foo" (id TEXT);\n'
        )
        assert self._run_rules([(self._NEW, sql)]) == []

    def test_a_block_comment_does_not_shift_the_reported_line(self):
        sql = "/* filler\nfiller */\n" + 'CREATE TABLE "Foo" (id TEXT);\n'
        suite = TestMigrationSQLIdempotency()
        with pytest.raises(AssertionError) as failure:
            suite.test_create_table_uses_if_not_exists([(self._NEW, sql)])
        assert f"{self._NEW}:3:" in str(failure.value)

    def test_a_new_migration_with_bare_create_table_fails(self):
        assert "test_create_table_uses_if_not_exists" in self._run_rules(
            [(self._NEW, 'CREATE TABLE "Foo" (id TEXT);\n')]
        )

    def test_a_new_migration_with_bare_add_column_fails(self):
        assert "test_add_column_uses_if_not_exists" in self._run_rules(
            [(self._NEW, 'ALTER TABLE "Foo" ADD COLUMN "bar" TEXT;\n')]
        )

    def test_the_guarded_forms_pass(self):
        sql = (
            'CREATE TABLE IF NOT EXISTS "Foo" (id TEXT);\n'
            'ALTER TABLE "Foo" ADD COLUMN IF NOT EXISTS "bar" TEXT;\n'
            'CREATE INDEX IF NOT EXISTS "Foo_bar_idx" ON "Foo"("bar");\n'
        )
        assert self._run_rules([(self._NEW, sql)]) == []

    def test_a_pre_guard_migration_is_exempt_but_a_new_one_is_not(self):
        bare = 'CREATE TABLE "Foo" (id TEXT);\n'
        exempt = sorted(_PRE_GUARD_MIGRATIONS)[0]
        assert self._run_rules([(exempt, bare)]) == []
        assert self._run_rules([(self._NEW, bare)]) != []

    def test_every_pre_guard_migration_still_exists_on_disk(self):
        present = {name for name, _ in _get_all_migrations()}
        missing = _PRE_GUARD_MIGRATIONS - present
        assert not missing, f"pre-guard entries naming no migration: {sorted(missing)}"

    def test_no_pre_guard_entry_is_already_clean(self):
        by_name = dict(_get_all_migrations())
        redundant = [
            name
            for name in sorted(_PRE_GUARD_MIGRATIONS)
            if not self._run_rules([(TestMigrationGuardScope._NEW, by_name[name])])
        ]
        assert not redundant, f"these no longer violate and should be removed: {redundant}"


_PARTITIONED_DRIFT_SQL = """-- AlterTable
ALTER TABLE "LiteLLM_BudgetTable" ADD COLUMN     "updated_by" TEXT;

-- AlterTable
ALTER TABLE "LiteLLM_SpendLogs" DROP CONSTRAINT "LiteLLM_SpendLogs_pkey",
ADD COLUMN     "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
ADD COLUMN     "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
ADD CONSTRAINT "LiteLLM_SpendLogs_pkey" PRIMARY KEY ("request_id");

-- DropTable
DROP TABLE "LiteLLM_SpendLogs_legacy";
"""


class TestPartitionedSpendLogsDriftFilter:
    """A doc-partitioned LiteLLM_SpendLogs (db_scripts/partition_spend_logs.sql) has a
    composite primary key that schema.prisma cannot express, so `prisma migrate diff`
    emits a primary-key rewrite that Postgres rejects, aborting the whole drift script
    before its legitimate statements run."""

    def test_pk_rewrite_and_runbook_artifact_drops_are_removed(self):
        filtered = filter_partitioned_spend_logs_diff(_PARTITIONED_DRIFT_SQL)
        assert 'DROP CONSTRAINT "LiteLLM_SpendLogs_pkey"' not in filtered
        assert 'PRIMARY KEY ("request_id")' not in filtered
        assert "LiteLLM_SpendLogs_legacy" not in filtered

    def test_legitimate_statements_in_the_same_script_are_kept(self):
        filtered = filter_partitioned_spend_logs_diff(_PARTITIONED_DRIFT_SQL)
        assert 'ALTER TABLE "LiteLLM_BudgetTable" ADD COLUMN     "updated_by" TEXT;' in filtered
        assert 'ADD COLUMN     "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP' in filtered
        assert 'ADD COLUMN     "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP' in filtered
        assert filtered.count('ALTER TABLE "LiteLLM_SpendLogs"') == 1

    def test_an_alter_containing_only_the_pk_rewrite_is_dropped_entirely(self):
        sql = (
            'ALTER TABLE "LiteLLM_SpendLogs" DROP CONSTRAINT "LiteLLM_SpendLogs_pkey",\n'
            'ADD CONSTRAINT "LiteLLM_SpendLogs_pkey" PRIMARY KEY ("request_id");\n'
        )
        assert filter_partitioned_spend_logs_diff(sql).strip() == ""

    def test_other_tables_pk_changes_are_untouched(self):
        sql = (
            'ALTER TABLE "LiteLLM_TeamTable" DROP CONSTRAINT "LiteLLM_TeamTable_pkey",\n'
            'ADD CONSTRAINT "LiteLLM_TeamTable_pkey" PRIMARY KEY ("team_id");\n'
        )
        filtered = filter_partitioned_spend_logs_diff(sql)
        assert 'DROP CONSTRAINT "LiteLLM_TeamTable_pkey"' in filtered
        assert 'PRIMARY KEY ("team_id")' in filtered


class _FakeCompleted:
    stdout = ""
    stderr = ""


class TestResolveAllMigrationsLedger:
    def _run(self, monkeypatch, tmp_path, partitioned, execute_fails):
        import subprocess as subprocess_module

        import litellm_proxy_extras.utils as utils_module

        monkeypatch.setenv("DATABASE_URL", "postgresql://u:p@localhost:5432/db")
        monkeypatch.delenv("DIRECT_URL", raising=False)
        monkeypatch.setattr(
            ProxyExtrasDBManager, "spend_logs_is_partitioned", staticmethod(lambda: partitioned)
        )
        monkeypatch.setattr(
            ProxyExtrasDBManager,
            "_get_migration_names",
            staticmethod(lambda migrations_dir: ["20250326162113_baseline"]),
        )
        calls = []

        def fake_run(cmd, **kwargs):
            calls.append(cmd)
            if "diff" in cmd:
                kwargs["stdout"].write(_PARTITIONED_DRIFT_SQL)
                return _FakeCompleted()
            if "execute" in cmd:
                executed_sql = open(cmd[cmd.index("--file") + 1]).read()
                calls.append(("executed_sql", executed_sql))
                if execute_fails:
                    raise subprocess_module.CalledProcessError(1, cmd, stderr="boom")
                return _FakeCompleted()
            return _FakeCompleted()

        monkeypatch.setattr(utils_module.prisma_toolchain, "run_prisma", fake_run)
        ProxyExtrasDBManager._resolve_all_migrations(str(tmp_path), "schema.prisma")
        return calls

    def _resolved(self, calls):
        return [c for c in calls if isinstance(c, list) and "resolve" in c]

    def _executed_sql(self, calls):
        return next(c[1] for c in calls if isinstance(c, tuple) and c[0] == "executed_sql")

    def test_failed_drift_apply_does_not_mark_migrations_applied(self, monkeypatch, tmp_path):
        calls = self._run(monkeypatch, tmp_path, partitioned=False, execute_fails=True)
        assert self._resolved(calls) == []

    def test_successful_drift_apply_still_marks_migrations_applied(self, monkeypatch, tmp_path):
        calls = self._run(monkeypatch, tmp_path, partitioned=False, execute_fails=False)
        assert len(self._resolved(calls)) == 1

    def test_partitioned_spend_logs_gets_the_filtered_drift_script(self, monkeypatch, tmp_path):
        calls = self._run(monkeypatch, tmp_path, partitioned=True, execute_fails=False)
        executed_sql = self._executed_sql(calls)
        assert 'PRIMARY KEY ("request_id")' not in executed_sql
        assert "LiteLLM_SpendLogs_legacy" not in executed_sql
        assert 'ALTER TABLE "LiteLLM_BudgetTable" ADD COLUMN     "updated_by" TEXT;' in executed_sql
        assert 'ADD COLUMN     "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP' in executed_sql
        assert len(self._resolved(calls)) == 1

    def test_unpartitioned_spend_logs_drift_script_is_untouched(self, monkeypatch, tmp_path):
        calls = self._run(monkeypatch, tmp_path, partitioned=False, execute_fails=False)
        assert self._executed_sql(calls) == _PARTITIONED_DRIFT_SQL


class TestPartitionedSpendLogsPushGuard:
    def _forbid_subprocess(self, monkeypatch):
        import litellm_proxy_extras.utils as utils_module

        def fail_run(cmd, **kwargs):
            raise AssertionError(f"run_prisma should not be called, got: {cmd}")

        monkeypatch.setattr(utils_module.prisma_toolchain, "run_prisma", fail_run)

    def test_v1_db_push_fails_fast_with_guidance(self, monkeypatch):
        monkeypatch.setattr(
            ProxyExtrasDBManager, "spend_logs_is_partitioned", staticmethod(lambda: True)
        )
        self._forbid_subprocess(monkeypatch)
        with pytest.raises(RuntimeError) as err:
            ProxyExtrasDBManager._run_migrations(use_migrate=False, use_v2_resolver=False)
        assert str(err.value) == PARTITIONED_SPEND_LOGS_PUSH_ERROR

    def test_v2_db_push_fails_fast_with_guidance(self, monkeypatch):
        monkeypatch.setattr(
            ProxyExtrasDBManager, "spend_logs_is_partitioned", staticmethod(lambda: True)
        )
        self._forbid_subprocess(monkeypatch)
        with pytest.raises(RuntimeError) as err:
            ProxyExtrasDBManager._setup_database_v2(use_migrate=False)
        assert str(err.value) == PARTITIONED_SPEND_LOGS_PUSH_ERROR


class _FakeCursor:
    def fetchone(self):
        return (1,)


class _FakePsycopgConn:
    def __init__(self, executed):
        self._executed = executed

    def __enter__(self):
        return self

    def __exit__(self, *args):
        return False

    def execute(self, query, params):
        self._executed.append((query, params))
        return _FakeCursor()


class TestLensRenamePendingCheck:
    _DATABASE_URL: Final = "postgresql://litellm:hunter2@localhost:5432/litellm"

    def test_database_failure_text_reaches_the_raised_message(self, monkeypatch: pytest.MonkeyPatch) -> None:
        import psycopg

        monkeypatch.setenv("DATABASE_URL", self._DATABASE_URL)

        def refuse(conninfo: str, *, connect_timeout: int, autocommit: bool) -> NoReturn:
            raise psycopg.OperationalError("FATAL:  sorry, too many clients already")

        with pytest.raises(RuntimeError) as err:
            ProxyExtrasDBManager.raise_if_lens_rename_pending(connect=refuse)
        assert "FATAL:  sorry, too many clients already" in str(err.value)

    def test_password_libpq_echoes_is_redacted_from_the_message(self, monkeypatch: pytest.MonkeyPatch) -> None:
        import psycopg

        password: Final = "p%zzword"
        database_url: Final = f"postgresql://litellm:{password}@localhost:5432/litellm"
        monkeypatch.setenv("DATABASE_URL", database_url)
        monkeypatch.delenv("DIRECT_URL", raising=False)
        with pytest.raises(psycopg.Error) as libpq:
            psycopg.connect(database_url, connect_timeout=10, autocommit=True)

        def raise_libpq_error(conninfo: str, *, connect_timeout: int, autocommit: bool) -> NoReturn:
            raise libpq.value

        with pytest.raises(RuntimeError) as err:
            ProxyExtrasDBManager.raise_if_lens_rename_pending(connect=raise_libpq_error)
        assert password not in str(err.value)
        assert str(err.value).endswith(str(libpq.value).strip().replace(password, "REDACTED"))

    def test_legacy_tables_keep_their_own_message(self, monkeypatch: pytest.MonkeyPatch) -> None:
        monkeypatch.setenv("DATABASE_URL", self._DATABASE_URL)
        executed: Final[list[tuple[str, tuple[str]]]] = []

        def legacy_tables_present(conninfo: str, *, connect_timeout: int, autocommit: bool) -> _FakePsycopgConn:
            return _FakePsycopgConn(executed)

        with pytest.raises(RuntimeError) as err:
            ProxyExtrasDBManager.raise_if_lens_rename_pending(connect=legacy_tables_present)
        assert str(err.value).startswith("Legacy Lens tables exist.")
        assert executed[0][1] == ("public",)


class TestSpendLogsPartitionDetectionSchemaScope:
    """A same-named LiteLLM_SpendLogs in another schema must not trip the
    detector: the catalog lookup has to be scoped to Prisma's target schema."""

    def _detect(self, monkeypatch, database_url):
        import sys
        import types

        executed = []
        fake_psycopg = types.ModuleType("psycopg")
        fake_psycopg.connect = lambda url, **kwargs: _FakePsycopgConn(executed)
        fake_psycopg.OperationalError = type("OperationalError", (Exception,), {})
        fake_psycopg.DatabaseError = type("DatabaseError", (Exception,), {})
        monkeypatch.setitem(sys.modules, "psycopg", fake_psycopg)
        monkeypatch.setenv("DATABASE_URL", database_url)
        assert ProxyExtrasDBManager.spend_logs_is_partitioned() is True
        return executed[0]

    def test_lookup_is_scoped_to_the_schema_url_param(self, monkeypatch):
        query, params = self._detect(
            monkeypatch, "postgresql://u:p@localhost:5432/db?schema=tenant_a"
        )
        assert "pg_namespace" in query
        assert "n.nspname = %s" in query
        assert params == ("tenant_a",)

    def test_lookup_falls_back_to_public_without_a_schema_param(self, monkeypatch):
        query, params = self._detect(monkeypatch, "postgresql://u:p@localhost:5432/db")
        assert "n.nspname = %s" in query
        assert params == ("public",)

    def test_only_partitioned_relations_match(self, monkeypatch):
        query, _ = self._detect(monkeypatch, "postgresql://u:p@localhost:5432/db")
        assert "pg_partitioned_table" in query


class TestSpendLogsPartitionDetectionMissingPsycopg:
    """psycopg ships in the `extra_proxy` install, but a stripped-down image
    can still lack it. When it does, detection must fail closed to False
    (never crash the migration path) and say so loudly, because a silent
    False here is what let a genuinely partitioned LiteLLM_SpendLogs hit the
    unfiltered primary-key rewrite in production."""

    def test_missing_psycopg_returns_false(self, monkeypatch):
        monkeypatch.setitem(sys.modules, "psycopg", None)
        monkeypatch.setenv("DATABASE_URL", "postgresql://u:p@localhost:5432/db")
        assert ProxyExtrasDBManager.spend_logs_is_partitioned() is False

    def test_missing_psycopg_logs_a_warning(self, monkeypatch, caplog):
        monkeypatch.setitem(sys.modules, "psycopg", None)
        monkeypatch.setenv("DATABASE_URL", "postgresql://u:p@localhost:5432/db")
        with caplog.at_level("WARNING", logger="litellm_proxy_extras"):
            ProxyExtrasDBManager.spend_logs_is_partitioned()
        assert any(
            "psycopg is not installed" in record.message for record in caplog.records
        )


_ATTEMPT_BUDGET = 4
_P3009_MIGRATION_NAME = "20260415120000_health_check_latest_per_model_index"
_P3009_STARTED_AT = "2026-10-02 23:20:56.439594 UTC"
_P3009_DEADLOCK_LOGS = "ERROR: deadlock detected\nDETAIL: Process 72 waits for ShareLock on transaction 991"

_P3005_STDERR = """Error: P3005

The database schema is not empty. Read more about how to baseline an existing production database: https://pris.ly/d/migrate-baseline
"""

_LOCK_TIMEOUT_MIGRATION = "20260818000000_add_spend_log_timestamps"

_P3018_LOCK_TIMEOUT_STDERR = (
    "Error: P3018\n\n"
    "A migration failed to apply. New migrations cannot be applied before the error is recovered from. "
    "Read more about how to resolve migration issues in a production database: "
    "https://pris.ly/d/migrate-resolve\n\n"
    "Migration name: 20260818000000_add_spend_log_timestamps\n\n"
    "Database error code: 55P03\n\n"
    "Database error:\n"
    "ERROR: canceling statement due to lock timeout\n\n"
    'DbError { severity: "ERROR", parsed_severity: Some(Error), code: SqlState(E55P03), '
    'message: "canceling statement due to lock timeout", detail: None, hint: None, position: None, '
    "where_: None, schema: None, table: None, column: None, datatype: None, constraint: None, "
    'file: Some("postgres.c"), line: Some(3298), routine: Some("ProcessInterrupts") }\n'
)

_LOCK_TIMEOUT_CONTENTION_STDERR = """Error: ERROR: canceling statement due to lock timeout
   0: schema_core::state::ApplyMigrations
             at schema-engine/core/src/state.rs:201
"""


def _p3018_stderr(migration_name):
    return f"""Error: P3018

A migration failed to apply. New migrations cannot be applied before the error is recovered from.

Migration name: {migration_name}

Database error code: 42P07

Database error:
ERROR: relation "SomeTable" already exists
"""


def _p3009_stderr(migration_name: str, started_at: str) -> str:
    return (
        "Error: P3009\n\n"
        "migrate found failed migrations in the target database, new migrations will not be applied. "
        "Read more about how to resolve migration issues in a production database: "
        "https://pris.ly/d/migrate-resolve\n"
        f"The `{migration_name}` migration started at {started_at} failed\n"
    )


@dataclass(frozen=True, slots=True)
class _LedgerRow:
    migration_name: str
    started_at: str
    finished: bool = False
    rolled_back: bool = False
    logs: str | None = None


@dataclass(frozen=True, slots=True)
class _LedgerCursor:
    row: tuple[object, ...] | None = None

    def fetchone(self) -> tuple[object, ...] | None:
        return self.row

    def fetchall(self) -> tuple[tuple[object, ...], ...]:
        return ()


class _LedgerConnection:
    def __init__(self, ledger: "_FakeLedger") -> None:
        self.ledger = ledger

    def __enter__(self) -> "_LedgerConnection":
        return self

    def __exit__(self, *args: object) -> None:
        return None

    def execute(self, query: object, params: tuple[object, ...] = ()) -> _LedgerCursor:
        return self.ledger.execute(query, params)


class _FakeLedger:
    def __init__(self, at_error: tuple[_LedgerRow, ...], after_peer: tuple[_LedgerRow, ...]) -> None:
        self.rows = at_error
        self.after_peer = after_peer
        self._peer_observed = False

    def connect(self, *args: object, **kwargs: object) -> _LedgerConnection:
        return _LedgerConnection(self)

    def execute(self, query: object, params: tuple[object, ...]) -> _LedgerCursor:
        text: Final = str(query)
        if "WHERE migration_name = %s" not in text or not params:
            return _LedgerCursor()
        if not self._peer_observed:
            self.rows = self.after_peer
            self._peer_observed = True
        matching: Final = tuple(
            row
            for row in self.rows
            if row.migration_name == params[0] and (len(params) == 1 or row.started_at == params[1])
        )
        if "rolled_back_at IS NULL" in text:
            unresolved: Final = next((row for row in matching if not row.finished and not row.rolled_back), None)
            return _LedgerCursor((unresolved.logs,) if unresolved else None)
        if "IS NOT NULL" in text:
            resolved: Final = next((row for row in matching if row.finished or row.rolled_back), None)
            return _LedgerCursor((1,) if resolved else None)
        return _LedgerCursor()


@pytest.mark.parametrize(
    "pooled,direct,expected",
    (
        ("postgresql://pool/db?pgbouncer=true", None, "postgresql://pool/db?pgbouncer=true"),
        (
            "postgresql://pool/db?pgbouncer=true",
            "postgresql://writer/db",
            "postgresql://writer/db?schema=public",
        ),
        (
            "postgresql://pool/db?schema=tenant%20one&pgbouncer=true",
            "postgresql://writer/db?sslmode=require&schema=wrong",
            "postgresql://writer/db?sslmode=require&schema=tenant+one",
        ),
    ),
)
def test_v2_migrations_use_the_direct_connection_with_the_runtime_schema(pooled, direct, expected):
    from litellm_proxy_extras.migration_lock import migration_environment

    environment = {"DATABASE_URL": pooled, "PRISMA_OFFLINE_MODE": "true"}
    configured = {**environment, **({"DIRECT_URL": direct} if direct else {})}
    migrated = migration_environment(configured)

    assert migrated["DATABASE_URL"] == expected
    assert migrated["PRISMA_OFFLINE_MODE"] == "true"
    assert configured["DATABASE_URL"] == pooled


class _MigrateDeployHarness:
    """Drives _setup_database_v2 with a scripted sequence of
    `prisma migrate deploy` outcomes, with every recovery command faked out so
    nothing touches a database or the packaged migrations directory."""

    def __init__(
        self,
        monkeypatch,
        tmp_path,
        outcomes,
        repeat_last=False,
        confirmed_migrations=(),
        ledger: "_FakeLedger | None" = None,
        database_url: str = "postgresql://u:p@localhost:9/x",
        prisma_env: dict[str, str] | None = None,
    ):
        import subprocess as subprocess_module

        import litellm_proxy_extras.utils as utils_module

        self.deploy_calls = []
        self.deploy_environments = []
        self.resolved = []
        self.rolled_back = []
        self.sleeps = []
        self.baselines = 0
        self._outcomes = list(outcomes)
        self._repeat_last = repeat_last
        self._subprocess_module = subprocess_module
        self.confirmed_migrations = set(confirmed_migrations)

        if ledger is None:
            monkeypatch.delenv("DATABASE_URL", raising=False)
            monkeypatch.delenv("DIRECT_URL", raising=False)
        else:
            monkeypatch.setenv("DATABASE_URL", database_url)
            monkeypatch.delenv("DIRECT_URL", raising=False)
            monkeypatch.setattr("psycopg.connect", ledger.connect)
        monkeypatch.setenv("LITELLM_MIGRATION_DIR", str(tmp_path))
        monkeypatch.setattr(utils_module.prisma_toolchain, "run_prisma", self._fake_run)
        monkeypatch.setattr(utils_module, "_get_prisma_env", lambda: prisma_env or {})
        monkeypatch.setattr(utils_module.time, "sleep", self.sleeps.append)

        self.baseline_succeeds = True

    def _fake_baseline(self, *args, **kwargs):
        self.baselines += 1
        if not self.baseline_succeeds:
            raise RuntimeError("The existing schema was not verified")

    def _next_outcome(self):
        if self._outcomes:
            if self._repeat_last and len(self._outcomes) == 1:
                return self._outcomes[0]
            return self._outcomes.pop(0)
        raise AssertionError("prisma migrate deploy called more times than scripted")

    def _fake_run(self, cmd, **kwargs):
        if cmd[1:] == ["migrate", "deploy"]:
            self.deploy_calls.append(cmd)
            self.deploy_environments.append(kwargs["env"].get("DATABASE_URL", ""))
            outcome = self._next_outcome()
            if outcome == "ok":
                return _FakeCompleted()
            if outcome == "timeout":
                raise self._subprocess_module.TimeoutExpired(cmd, 1)
            raise self._subprocess_module.CalledProcessError(1, cmd, stderr=outcome)
        if cmd[1:4] == ["migrate", "resolve", "--rolled-back"] and len(cmd) == 5:
            self.rolled_back.append(cmd[4])
            return _FakeCompleted()
        raise AssertionError(f"unexpected prisma command: {cmd}")

    def run(self):
        while not ProxyExtrasDBManager._run_database_v2(
            use_migrate=True,
            recover_completed=self._fake_recovery,
            baseline_existing=self._fake_baseline,
        ):
            continue
        return True

    def _fake_recovery(self, name):
        if name not in self.confirmed_migrations:
            return False
        self.confirmed_migrations.remove(name)
        self.resolved.append(name)
        return True


class TestConcurrentP3009Recovery:
    @pytest.mark.parametrize(
        "after_peer",
        (
            (_LedgerRow(_P3009_MIGRATION_NAME, _P3009_STARTED_AT, rolled_back=True, logs=_P3009_DEADLOCK_LOGS),),
            (
                _LedgerRow(_P3009_MIGRATION_NAME, _P3009_STARTED_AT, rolled_back=True, logs=_P3009_DEADLOCK_LOGS),
                _LedgerRow(_P3009_MIGRATION_NAME, "2026-10-02 23:21:11.539224 UTC"),
            ),
            (_LedgerRow(_P3009_MIGRATION_NAME, _P3009_STARTED_AT, finished=True, logs=_P3009_DEADLOCK_LOGS),),
        ),
        ids=("rolled-back", "rolled-back-beside-a-fresh-in-flight-row", "finished"),
    )
    def test_a_p3009_row_a_peer_already_recovered_is_retried(
        self,
        monkeypatch: pytest.MonkeyPatch,
        tmp_path: Path,
        after_peer: tuple[_LedgerRow, ...],
    ) -> None:
        deadlocked_row: Final = _LedgerRow(
            _P3009_MIGRATION_NAME,
            _P3009_STARTED_AT,
            logs=_P3009_DEADLOCK_LOGS,
        )
        harness: Final = _MigrateDeployHarness(
            monkeypatch,
            tmp_path,
            [_p3009_stderr(_P3009_MIGRATION_NAME, _P3009_STARTED_AT), "ok"],
            ledger=_FakeLedger(at_error=(deadlocked_row,), after_peer=after_peer),
        )

        assert harness.run() is True
        assert len(harness.deploy_calls) == 2

    @pytest.mark.parametrize(
        "ledger_rows",
        (
            (
                _LedgerRow(
                    _P3009_MIGRATION_NAME,
                    _P3009_STARTED_AT,
                    logs='ERROR: syntax error at or near "SLECT"',
                ),
            ),
            (
                _LedgerRow(
                    _P3009_MIGRATION_NAME,
                    _P3009_STARTED_AT,
                    logs='ERROR: syntax error at or near "SLECT"',
                ),
                _LedgerRow(
                    _P3009_MIGRATION_NAME,
                    "2026-10-02 23:19:40.120000 UTC",
                    rolled_back=True,
                    logs=_P3009_DEADLOCK_LOGS,
                ),
            ),
        ),
        ids=("only-row", "beside-a-recovered-earlier-attempt"),
    )
    def test_an_unresolved_p3009_row_without_the_deadlock_marker_stops(
        self,
        monkeypatch: pytest.MonkeyPatch,
        tmp_path: Path,
        ledger_rows: tuple[_LedgerRow, ...],
    ) -> None:
        harness: Final = _MigrateDeployHarness(
            monkeypatch,
            tmp_path,
            [_p3009_stderr(_P3009_MIGRATION_NAME, _P3009_STARTED_AT)],
            ledger=_FakeLedger(at_error=ledger_rows, after_peer=ledger_rows),
        )

        with pytest.raises(RuntimeError, match="Migration completion could not be verified"):
            harness.run()
        assert len(harness.deploy_calls) == 1


class TestMigrateDeployAttemptAccounting:
    def test_migrate_deploy_does_not_add_startup_options_to_a_pooled_url(
        self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path
    ) -> None:
        from urllib.parse import parse_qsl, urlsplit

        pooled_url: Final = "postgresql://pool/db?schema=tenant&connection_limit=5"
        harness: Final = _MigrateDeployHarness(
            monkeypatch,
            tmp_path,
            ["ok"],
            ledger=_FakeLedger(at_error=(), after_peer=()),
            database_url=pooled_url,
            prisma_env={"DATABASE_URL": pooled_url},
        )

        assert harness.run() is True
        received_url: Final = harness.deploy_environments[0]
        assert received_url == pooled_url
        assert all(key != "options" for key, _ in parse_qsl(urlsplit(received_url).query))

    def test_a_push_created_database_finishes_bootstrapping(self, monkeypatch, tmp_path):
        harness = _MigrateDeployHarness(
            monkeypatch,
            tmp_path,
            [_P3005_STDERR, "ok"],
        )

        assert harness.run() is True
        assert harness.baselines == 1
        assert harness.resolved == []
        assert len(harness.deploy_calls) == 2

    def test_repeated_recovery_of_one_migration_still_gives_up(self, monkeypatch, tmp_path):
        harness = _MigrateDeployHarness(
            monkeypatch,
            tmp_path,
            [_p3018_stderr("20250329084805_new_cron_job_table")],
            repeat_last=True,
            confirmed_migrations=("20250329084805_new_cron_job_table",),
        )

        with pytest.raises(RuntimeError):
            harness.run()
        assert len(harness.deploy_calls) == 2
        assert harness.resolved == ["20250329084805_new_cron_job_table"]

    def test_timeouts_still_spend_the_budget(self, monkeypatch, tmp_path):
        harness = _MigrateDeployHarness(monkeypatch, tmp_path, ["timeout"], repeat_last=True)

        with pytest.raises(RuntimeError):
            harness.run()
        assert len(harness.deploy_calls) == _ATTEMPT_BUDGET

    def test_an_unverified_baseline_stops_without_replaying_migrations(self, monkeypatch, tmp_path):
        harness = _MigrateDeployHarness(monkeypatch, tmp_path, [_P3005_STDERR], repeat_last=True)
        harness.baseline_succeeds = False

        with pytest.raises(RuntimeError):
            harness.run()
        assert len(harness.deploy_calls) == 1

    def test_lock_contention_does_not_spend_the_failure_budget(self, monkeypatch, tmp_path):
        harness = _MigrateDeployHarness(
            monkeypatch,
            tmp_path,
            ["Error: P1002\nTimed out waiting for the advisory lock"] * 6 + ["ok"],
        )
        assert harness.run() is True
        assert len(harness.deploy_calls) == 7

    def test_duplicate_object_error_without_completion_proof_is_fatal(self, monkeypatch, tmp_path):
        harness = _MigrateDeployHarness(monkeypatch, tmp_path, [_p3018_stderr("20260101000000_x")])
        with pytest.raises(RuntimeError, match="cannot be auto-recovered"):
            harness.run()
        assert harness.resolved == []
        assert len(harness.deploy_calls) == 1

    @pytest.mark.parametrize("name", ("20260101000000_x", "20260101000000_migration with spaces"))
    def test_an_interrupted_migration_with_confirmed_sql_can_finish(self, monkeypatch, tmp_path, name):
        harness = _MigrateDeployHarness(
            monkeypatch,
            tmp_path,
            [f"Error: P3009\nThe `{name}` migration failed", "ok"],
            confirmed_migrations=(name,),
        )
        assert harness.run() is True
        assert harness.resolved == [name]

    def test_an_interrupted_migration_without_confirmation_stops(self, monkeypatch, tmp_path):
        name = "20260101000000_x"
        started = "2026-09-12 20:15:06.694553 UTC"
        report = f"Error: P3009\nThe `{name}` migration started at {started} failed"
        harness = _MigrateDeployHarness(
            monkeypatch,
            tmp_path,
            [report],
        )
        with pytest.raises(RuntimeError, match="Migration completion could not be verified") as failure:
            harness.run()
        message = str(failure.value)
        assert name in message
        assert started in message
        assert "start record but no successful completion record" in message
        assert "cannot determine whether its SQL committed" in message
        assert "avoid repeating or skipping database changes" in message
        assert "_prisma_migrations" in message
        assert "migration.sql" in message
        assert "same database" in message
        assert "Only after verifying every migration change is present" in message
        assert "prisma migrate resolve --applied <migration_name>" in message
        assert "Only after verifying no migration changes remain" in message
        assert "prisma migrate resolve --rolled-back <migration_name>" in message
        assert "leave migration history unchanged" in message
        assert "Repeated restarts alone" in message
        assert report in message
        assert len(harness.deploy_calls) == 1
        assert harness.resolved == []

    def test_an_unrecoverable_error_is_not_retried(self, monkeypatch, tmp_path):
        harness = _MigrateDeployHarness(
            monkeypatch,
            tmp_path,
            ['Error: P3018\n\nMigration name: 20260101000000_x\n\nERROR: syntax error at or near "SLECT"\n'],
            repeat_last=True,
        )

        with pytest.raises(RuntimeError):
            harness.run()
        assert len(harness.deploy_calls) == 1
        assert harness.resolved == []


class TestMigrationLockTimeoutRecovery:
    """A migration whose DDL wait Postgres cancelled (55P03, from the lock_timeout the
    migration SQL sets locally) is retryable: the half-written ledger row is rolled back and
    the next attempt reruns the migration, priced like a deadlock retry."""

    def test_a_lock_timed_out_migration_rolls_back_and_retries(self, monkeypatch, tmp_path):
        harness = _MigrateDeployHarness(
            monkeypatch,
            tmp_path,
            [_P3018_LOCK_TIMEOUT_STDERR, "ok"],
        )

        assert harness.run() is True
        assert len(harness.deploy_calls) == 2
        assert harness.rolled_back == [_LOCK_TIMEOUT_MIGRATION]
        assert len(harness.sleeps) == 1

    def test_repeated_lock_timeouts_spend_the_failure_budget(self, monkeypatch, tmp_path):
        harness = _MigrateDeployHarness(
            monkeypatch,
            tmp_path,
            [_P3018_LOCK_TIMEOUT_STDERR],
            repeat_last=True,
        )

        with pytest.raises(RuntimeError, match="SET LOCAL lock_timeout"):
            harness.run()
        assert len(harness.deploy_calls) == _ATTEMPT_BUDGET
        assert harness.rolled_back == [_LOCK_TIMEOUT_MIGRATION] * _ATTEMPT_BUDGET

    def test_lock_timeout_on_prismas_own_lock_wait_is_contention(self, monkeypatch, tmp_path):
        harness = _MigrateDeployHarness(
            monkeypatch,
            tmp_path,
            [_LOCK_TIMEOUT_CONTENTION_STDERR] * 6 + ["ok"],
        )

        assert harness.run() is True
        assert len(harness.deploy_calls) == 7
        assert harness.rolled_back == []
        assert harness.sleeps == []

    def test_a_p3009_row_logging_a_lock_timeout_rolls_back_and_retries(self, monkeypatch, tmp_path):
        locked_row = _LedgerRow(
            _LOCK_TIMEOUT_MIGRATION,
            _P3009_STARTED_AT,
            logs=_P3018_LOCK_TIMEOUT_STDERR,
        )
        harness = _MigrateDeployHarness(
            monkeypatch,
            tmp_path,
            [_p3009_stderr(_LOCK_TIMEOUT_MIGRATION, _P3009_STARTED_AT), "ok"],
            ledger=_FakeLedger(at_error=(locked_row,), after_peer=(locked_row,)),
        )

        assert harness.run() is True
        assert len(harness.deploy_calls) == 2
        assert harness.rolled_back == [_LOCK_TIMEOUT_MIGRATION]

    def test_a_lock_timeout_with_a_permission_error_stays_fatal(self, monkeypatch, tmp_path):
        stderr = _P3018_LOCK_TIMEOUT_STDERR + "ERROR: permission denied for table LiteLLM_SpendLogs\n"
        harness = _MigrateDeployHarness(monkeypatch, tmp_path, [stderr])

        with pytest.raises(RuntimeError, match="insufficient"):
            harness.run()
        assert len(harness.deploy_calls) == 1
        assert harness.rolled_back == []


class TestLogMigrationLockHolders:
    _MIGRATION_SQL = (
        "-- AlterTable\n"
        'ALTER TABLE "LiteLLM_SpendLogs" ADD COLUMN IF NOT EXISTS "created_at" TIMESTAMP(3);\n'
    )

    def _migration_file(self, tmp_path: Path) -> None:
        migration_dir = tmp_path / "migrations" / _LOCK_TIMEOUT_MIGRATION
        migration_dir.mkdir(parents=True)
        (migration_dir / "migration.sql").write_text(self._MIGRATION_SQL)

    def test_the_current_lock_holders_are_logged(self, monkeypatch, tmp_path, caplog):
        self._migration_file(tmp_path)
        monkeypatch.setenv("LITELLM_MIGRATION_DIR", str(tmp_path))
        monkeypatch.setenv("DATABASE_URL", "postgresql://u:p@localhost:9/x")
        executed: list[tuple[object, object]] = []
        connect_kwargs: list[dict[str, object]] = []

        class _Conn:
            def __enter__(self):
                return self

            def __exit__(self, *args):
                return None

            def execute(self, query, params):
                executed.append((query, params))
                row = (
                    1234,
                    "LiteLLM_SpendLogs",
                    "AccessExclusiveLock",
                    "idle in transaction",
                    "proxy-1",
                    "0:00:42",
                )
                return type("Cur", (), {"fetchall": staticmethod(lambda: (row,))})()

        def fake_connect(*args, **kwargs):
            connect_kwargs.append(kwargs)
            return _Conn()

        monkeypatch.setattr("psycopg.connect", fake_connect)

        with caplog.at_level("WARNING", logger="litellm_proxy_extras"):
            ProxyExtrasDBManager._log_migration_lock_holders(_LOCK_TIMEOUT_MIGRATION)

        assert len(executed) == 1
        assert connect_kwargs[0]["options"] == "-c statement_timeout=5000"
        assert executed[0][1] == ("public", ["LiteLLM_SpendLogs"])
        assert "pg_locks" in str(executed[0][0])
        assert "a.query" not in str(executed[0][0])
        assert "1234" in caplog.text
        assert "LiteLLM_SpendLogs" in caplog.text
        assert "AccessExclusiveLock" in caplog.text
        assert "0:00:42" in caplog.text

    def test_a_database_that_cannot_be_reached_is_skipped_quietly(self, monkeypatch, tmp_path):
        import psycopg

        self._migration_file(tmp_path)
        monkeypatch.setenv("LITELLM_MIGRATION_DIR", str(tmp_path))
        monkeypatch.setenv("DATABASE_URL", "postgresql://u:p@localhost:9/x")

        def refuse(*args, **kwargs):
            raise psycopg.OperationalError("connection refused")

        monkeypatch.setattr("psycopg.connect", refuse)

        assert ProxyExtrasDBManager._log_migration_lock_holders(_LOCK_TIMEOUT_MIGRATION) is None

    def test_a_migration_without_sql_on_disk_never_connects(self, monkeypatch, tmp_path):
        monkeypatch.setenv("LITELLM_MIGRATION_DIR", str(tmp_path))
        monkeypatch.setenv("DATABASE_URL", "postgresql://u:p@localhost:9/x")
        connections = []
        monkeypatch.setattr("psycopg.connect", lambda *args, **kwargs: connections.append(1))

        ProxyExtrasDBManager._log_migration_lock_holders("99999999999999_not_on_disk")

        assert connections == []


@pytest.mark.parametrize(
    "steps,logs,script,expected",
    (
        (1, "", b"CREATE TABLE item (id int);", True),
        (0, "", b"CREATE TABLE item (id int);", False),
        (0, "already exists", b"CREATE TABLE item (id int);", False),
        (1, "permission denied", b"CREATE TABLE item (id int);", False),
        (1, "", b"CREATE TABLE item (id text);", False),
        (2, "", b"CREATE TABLE item (id int);", False),
    ),
)
def test_migration_completion_requires_a_matching_successful_script(steps, logs, script, expected):
    import hashlib

    from litellm_proxy_extras.migration_recovery import MigrationProgress

    progress = MigrationProgress(hashlib.sha256(b"CREATE TABLE item (id int);").hexdigest(), steps, logs)
    assert progress.confirms_completion(script) is expected


def test_prisma_lock_waiting_has_its_own_deadline():
    from litellm_proxy_extras.utils import _MigrateAttemptBudget

    budget = _MigrateAttemptBudget(attempts_left=4, contention_seconds_left=2)
    waiting = budget.after_contention(1)
    assert waiting.attempts_left == 4
    with pytest.raises(RuntimeError, match="advisory lock"):
        waiting.after_contention(2)


class TestJWTKeyMappingCascade:
    """Regression tests for issue #33702.

    A virtual key referenced by a LiteLLM_JWTKeyMapping row could not be deleted
    because LiteLLM_JWTKeyMapping_token_fkey was created ON DELETE RESTRICT, so
    deleting the key (Admin UI, /key/delete, team delete, ...) raised a foreign
    key violation. The mapping must be removed automatically when its key is
    deleted, which the FK now enforces via ON DELETE CASCADE.
    """

    _FK_NAME = "LiteLLM_JWTKeyMapping_token_fkey"

    def _effective_on_delete(self):
        """Replay every migration in order and return the last ON DELETE action
        declared for the JWT key mapping FK."""
        action = None
        for _migration_name, sql in _get_all_migrations():
            for match in re.finditer(
                rf'ADD\s+CONSTRAINT\s+"{re.escape(self._FK_NAME)}".*?'
                r"ON\s+DELETE\s+(CASCADE|RESTRICT|SET\s+NULL|NO\s+ACTION|SET\s+DEFAULT)",
                sql,
                re.IGNORECASE | re.DOTALL,
            ):
                action = re.sub(r"\s+", " ", match.group(1).upper())
        return action

    def test_fk_effective_on_delete_is_cascade(self):
        """The final FK definition across all migrations must cascade deletes."""
        assert self._effective_on_delete() == "CASCADE", (
            f"{self._FK_NAME} must end up ON DELETE CASCADE so deleting a "
            "virtual key removes its JWT key mapping (issue #33702)"
        )

    def test_schema_declares_cascade_on_relation(self):
        """schema.prisma must declare onDelete: Cascade on the mapping relation
        so the generated client and DB agree."""
        schema_paths = glob.glob(
            os.path.abspath(
                os.path.join(
                    os.path.dirname(__file__), "../../../**/schema.prisma"
                )
            ),
            recursive=True,
        )
        declaring = tuple(
            (path, schema)
            for path, schema in ((p, Path(p).read_text()) for p in schema_paths)
            if "model LiteLLM_JWTKeyMapping" in schema
        )
        assert declaring, "No schema.prisma declaring LiteLLM_JWTKeyMapping found"
        for path, schema in declaring:
            match = re.search(
                r"litellm_verification_token\s+LiteLLM_VerificationToken\s+@relation\(([^)]*)\)",
                schema,
            )
            assert match is not None, (
                f"{path} declares LiteLLM_JWTKeyMapping but its verification token "
                "relation could not be parsed, so this test cannot vouch for it "
                "(issue #33702)"
            )
            assert "onDelete: Cascade" in match.group(1), (
                f"{path} must declare onDelete: Cascade on the JWT key mapping "
                "relation (issue #33702)"
            )



class TestStripPrismaQueryParams:
    """The psycopg URL the job connects with is derived from the Prisma-dialect
    DATABASE_URL, whose TLS params mean something else to libpq."""

    @staticmethod
    def _query(url: str) -> dict[str, str]:
        from urllib.parse import parse_qsl, urlparse

        return dict(parse_qsl(urlparse(url).query))

    @pytest.mark.parametrize(
        "url,expected_user,expected_password,expected_host,expected_port,expected_sslmode",
        (
            (
                "postgresql://u:wr ong'pw9@writer:5433/litellm",
                "u",
                "wr ong'pw9",
                "writer",
                "5433",
                None,
            ),
            (
                "postgresql://u:wr ong'pw9@writer:5433/litellm?schema=x&sslmode=require",
                "u",
                "wr ong'pw9",
                "writer",
                "5433",
                "require",
            ),
            (
                "postgresql://u:p%40ss%20w@writer:5433/litellm",
                "u",
                "p@ss w",
                "writer",
                "5433",
                None,
            ),
            (
                "postgresql://u:p%z@writer:5433/litellm",
                "u",
                "p%z",
                "writer",
                "5433",
                None,
            ),
            (
                "postgresql://u:wr ong'pw9@[::1]:5433/litellm",
                "u",
                "wr ong'pw9",
                "::1",
                "5433",
                None,
            ),
        ),
    )
    def test_userinfo_is_encoded_for_libpq(
        self,
        url: str,
        expected_user: str,
        expected_password: str,
        expected_host: str,
        expected_port: str,
        expected_sslmode: str | None,
    ) -> None:
        from psycopg.conninfo import conninfo_to_dict

        result: Final = ProxyExtrasDBManager._strip_prisma_query_params(url)
        conninfo: Final = conninfo_to_dict(result)

        assert conninfo["user"] == expected_user
        assert conninfo["password"] == expected_password
        assert conninfo["host"] == expected_host
        assert conninfo["port"] == expected_port
        assert conninfo["dbname"] == "litellm"
        assert conninfo.get("sslmode") == expected_sslmode
        assert "schema" not in conninfo

    def test_empty_password_userinfo_keeps_its_separator(self) -> None:
        url: Final = "postgresql://u:@writer/litellm"

        assert ProxyExtrasDBManager._strip_prisma_query_params(url) == url

    def test_prisma_ca_sslcert_becomes_sslrootcert_with_verify_full(self):
        url = "postgresql://u:p@writer:5432/db?schema=public&sslmode=require&sslcert=/tmp/pinned.pem&sslaccept=strict"

        cleaned = ProxyExtrasDBManager._strip_prisma_query_params(url)

        assert self._query(cleaned) == {"sslmode": "verify-full", "sslrootcert": "/tmp/pinned.pem"}
        assert cleaned.startswith("postgresql://u:p@writer:5432/db?")

    @pytest.mark.parametrize("sslmode", ["prefer", "require"])
    @pytest.mark.parametrize("sslaccept", ["strict", "unknown-mode-prisma-treats-as-strict"])
    def test_strict_verifies_chain_and_hostname_whatever_sslmode_prisma_was_given(self, sslmode, sslaccept):
        url = f"postgresql://writer/db?sslmode={sslmode}&sslcert=/certs/ca.pem&sslaccept={sslaccept}"

        cleaned = ProxyExtrasDBManager._strip_prisma_query_params(url)

        assert self._query(cleaned) == {"sslmode": "verify-full", "sslrootcert": "/certs/ca.pem"}

    def test_strict_with_tls_disabled_stays_off(self):
        url = "postgresql://writer/db?sslmode=disable&sslcert=/certs/ca.pem&sslaccept=strict"

        cleaned = ProxyExtrasDBManager._strip_prisma_query_params(url)

        assert self._query(cleaned) == {"sslmode": "disable"}

    @pytest.mark.parametrize("sslaccept", ["&sslaccept=accept_invalid_certs", ""])
    def test_without_strict_the_ca_is_dropped_so_libpq_checks_nothing_like_prisma(self, sslaccept):
        url = f"postgresql://writer/db?sslmode=require&sslcert=/certs/ca.pem{sslaccept}"

        cleaned = ProxyExtrasDBManager._strip_prisma_query_params(url)

        assert self._query(cleaned) == {"sslmode": "require"}

    def test_a_ca_alone_without_strict_or_sslmode_leaves_libpq_its_defaults(self):
        cleaned = ProxyExtrasDBManager._strip_prisma_query_params("postgresql://writer/db?sslcert=/certs/ca.pem")

        assert cleaned == "postgresql://writer/db"

    def test_a_libpq_client_certificate_pair_is_left_alone(self):
        url = "postgresql://writer/db?sslmode=verify-full&sslrootcert=/ca.pem&sslcert=/client.crt&sslkey=/client.key"

        cleaned = ProxyExtrasDBManager._strip_prisma_query_params(url)

        assert self._query(cleaned) == {
            "sslmode": "verify-full",
            "sslrootcert": "/ca.pem",
            "sslcert": "/client.crt",
            "sslkey": "/client.key",
        }

    def test_an_explicit_sslrootcert_wins_over_the_prisma_sslcert(self):
        url = "postgresql://writer/db?sslmode=require&sslrootcert=/ca.pem&sslcert=/pinned.pem&sslaccept=strict"

        cleaned = ProxyExtrasDBManager._strip_prisma_query_params(url)

        assert self._query(cleaned) == {"sslmode": "verify-full", "sslrootcert": "/ca.pem"}

    def test_prisma_only_params_are_dropped_and_plain_urls_pass_through(self):
        url = "postgresql://u:p@pooler:6543/db?schema=tenant&pgbouncer=true&connection_limit=5&connect_timeout=3"

        cleaned = ProxyExtrasDBManager._strip_prisma_query_params(url)

        assert cleaned == "postgresql://u:p@pooler:6543/db?connect_timeout=3"
        assert (
            ProxyExtrasDBManager._strip_prisma_query_params("postgresql://u:p@writer/db")
            == "postgresql://u:p@writer/db"
        )


class TestBuildRequestLogIndexes:
    """The migration job hands the index build the direct database URL and the schema
    the migrations target, waits for it, and reports its result."""

    @pytest.fixture
    def builds(self):
        return []

    @pytest.fixture
    def build(self, builds):
        def record(database_url: str, schema: str) -> bool:
            builds.append((database_url, schema))
            return True

        return record

    def test_the_build_gets_the_direct_url_without_prisma_params_and_the_prisma_schema(self, monkeypatch, builds, build):
        monkeypatch.setenv("DATABASE_URL", "postgresql://u:p@pooler:6543/db?schema=tenant&pgbouncer=true")
        monkeypatch.setenv("DIRECT_URL", "postgresql://u:p@primary:5432/db?connection_limit=1")

        assert ProxyExtrasDBManager.build_request_log_indexes(build=build) is True

        assert builds == [("postgresql://u:p@primary:5432/db", "tenant")]

    def test_the_build_defaults_to_the_database_url_and_the_public_schema(self, monkeypatch, builds, build):
        monkeypatch.setenv("DATABASE_URL", "postgresql://u:p@primary:5432/db")
        monkeypatch.delenv("DIRECT_URL", raising=False)

        assert ProxyExtrasDBManager.build_request_log_indexes(build=build) is True

        assert builds == [("postgresql://u:p@primary:5432/db", "public")]

    def test_a_build_that_leaves_indexes_missing_is_reported_so_the_job_reruns(self, monkeypatch):
        monkeypatch.setenv("DATABASE_URL", "postgresql://u:p@primary:5432/db")

        assert ProxyExtrasDBManager.build_request_log_indexes(build=lambda url, schema: False) is False

    def test_without_a_database_url_nothing_is_built(self, monkeypatch, builds, build):
        monkeypatch.delenv("DATABASE_URL", raising=False)

        assert ProxyExtrasDBManager.build_request_log_indexes(build=build) is True

        assert builds == []


class TestStartRequestLogIndexBuild:
    """A serving proxy that ran the migrations starts the index build on a daemon thread
    and goes on to serve while it runs."""

    def test_the_build_runs_on_a_daemon_thread_that_does_not_hold_up_the_caller(self):
        release: Final = threading.Event()
        builds: Final[list[str]] = []  # mutable-ok: the builder thread hands back the thread it ran on

        def build() -> bool:
            assert release.wait(5), "the caller never came back from start_request_log_index_build"
            builds.append(threading.current_thread().name)
            return True

        thread: Final = ProxyExtrasDBManager.start_request_log_index_build(build=build)

        assert builds == [], "the build ran before start_request_log_index_build returned"
        assert thread.daemon is True
        release.set()
        thread.join(5)
        assert builds == ["litellm-request-log-indexes"]


class TestRunMigrationJob:
    """`run_migration_job` is `setup_database` followed by the index build, each step's
    result deciding whether the job reports success."""

    @pytest.fixture
    def calls(self):
        return []

    @pytest.fixture
    def setup(self, calls):
        def record(result: bool):
            def setup_database(use_migrate: bool, use_v2_resolver: bool) -> bool:
                calls.append(("setup", use_migrate, use_v2_resolver))
                return result

            return setup_database

        return record

    @pytest.fixture
    def build(self, calls):
        def record(result: bool):
            def build_request_log_indexes() -> bool:
                calls.append(("build",))
                return result

            return build_request_log_indexes

        return record

    def test_the_job_builds_the_indexes_after_the_migrations_succeed(self, calls, setup, build):
        assert ProxyExtrasDBManager.run_migration_job(True, False, setup=setup(True), build=build(True)) is True

        assert calls == [("setup", True, False), ("build",)]

    def test_the_job_fails_without_building_when_the_migrations_fail(self, calls, setup, build):
        assert ProxyExtrasDBManager.run_migration_job(True, True, setup=setup(False), build=build(True)) is False

        assert calls == [("setup", True, True)]

    def test_the_job_fails_when_an_index_could_not_be_built(self, calls, setup, build):
        assert ProxyExtrasDBManager.run_migration_job(True, True, setup=setup(True), build=build(False)) is False

        assert calls == [("setup", True, True), ("build",)]


class TestMigrationJobOwnedDrift:
    JOB_INDEXES = (
        "-- CreateIndex\n"
        'CREATE INDEX "LiteLLM_SpendLogs_litellm_call_id_idx" ON "LiteLLM_SpendLogs"("litellm_call_id");\n'
        "\n-- CreateIndex\n"
        'CREATE INDEX "LiteLLM_SpendLogs_api_key_startTime_idx" ON "LiteLLM_SpendLogs"("api_key", "startTime");\n'
    )

    def test_a_plain_spend_logs_table_only_loses_the_migration_job_indexes(self):
        filtered = ProxyExtrasDBManager._filter_migration_job_owned_drift(
            _PARTITIONED_DRIFT_SQL + self.JOB_INDEXES, partitioned=False
        )
        assert "LiteLLM_SpendLogs_litellm_call_id_idx" not in filtered
        assert "LiteLLM_SpendLogs_api_key_startTime_idx" not in filtered
        assert 'PRIMARY KEY ("request_id")' in filtered

    def test_a_partitioned_spend_logs_table_also_loses_its_partitioning_artifacts(self):
        filtered = ProxyExtrasDBManager._filter_migration_job_owned_drift(
            _PARTITIONED_DRIFT_SQL + self.JOB_INDEXES, partitioned=True
        )
        assert "LiteLLM_SpendLogs_litellm_call_id_idx" not in filtered
        assert 'PRIMARY KEY ("request_id")' not in filtered
        assert "LiteLLM_SpendLogs_legacy" not in filtered
        assert 'ALTER TABLE "LiteLLM_BudgetTable" ADD COLUMN     "updated_by" TEXT;' in filtered


_P3018_UNCLASSIFIED_STDERR: Final = (
    "Error: P3018\n\n"
    "A migration failed to apply. New migrations cannot be applied before the error is "
    "recovered from.\n\n"
    "Migration name: 20260921190000_agent_identity\n\n"
    "Database error code: 23505\n\n"
    "Database error:\n"
    'ERROR: could not create unique index "agent_identity_key"\n'
    "DETAIL: Key (agent_id)=(agent-1) is duplicated.\n"
)


_FAKE_PRISMA_PID: Final = 424242


class TestV1MigrationFailuresLogAtError:
    @staticmethod
    def _run_v1_migrations(
        monkeypatch: pytest.MonkeyPatch,
        tmp_path: Path,
        *,
        deploy_stderr: Optional[str] = None,
        deploy_timeout: bool = False,
        diff_stderr: Optional[str] = None,
        database_url: Optional[str] = None,
    ) -> tuple[bool, list[list[str]], list[int]]:
        import litellm_proxy_extras.utils as utils_module

        calls: Final[list[list[str]]] = []
        killed_pids: Final[list[int]] = []

        class _FakePrismaPopen:
            def __init__(
                self,
                argv: tuple[str, ...],
                *,
                env: Optional[dict[str, str]] = None,
                stdout: object = None,
                stderr: object = None,
                text: object = None,
                start_new_session: object = None,
            ) -> None:
                self.args: Final = argv
                self.argv: Final = argv
                self.pid: Final = _FAKE_PRISMA_PID
                self.returncode: Optional[int] = None
                calls.append(list(argv))

            def __enter__(self) -> "_FakePrismaPopen":
                return self

            def __exit__(self, *args: object) -> None:
                return None

            def _subcommand(self) -> tuple[str, str]:
                known: Final = (
                    ("migrate", "deploy"),
                    ("migrate", "diff"),
                    ("migrate", "resolve"),
                    ("db", "execute"),
                )
                for index in range(len(self.argv) - 1):
                    pair: Final = tuple(self.argv[index : index + 2])
                    if pair in known:
                        return pair
                return ("", "")

            def communicate(self, timeout: Optional[float] = None) -> tuple[str, str]:
                subcommand: Final = self._subcommand()
                if subcommand == ("migrate", "deploy"):
                    if deploy_timeout:
                        raise subprocess.TimeoutExpired(self.argv, timeout)
                    if deploy_stderr is not None:
                        self.returncode = 1
                        return "", deploy_stderr
                    self.returncode = 0
                    return "No pending migrations to apply", ""
                if subcommand == ("migrate", "diff") and diff_stderr is not None:
                    self.returncode = 1
                    return "", diff_stderr
                self.returncode = 0
                return "", ""

        migration_dir: Final = tmp_path / "migration_dir"
        migration_dir.mkdir()
        if database_url is None:
            monkeypatch.delenv("DATABASE_URL", raising=False)
        else:
            monkeypatch.setenv("DATABASE_URL", database_url)
        monkeypatch.setenv("LITELLM_MIGRATION_DIR", str(migration_dir))
        monkeypatch.setattr(
            utils_module.prisma_toolchain.subprocess, "Popen", _FakePrismaPopen
        )
        monkeypatch.setattr(
            utils_module.prisma_toolchain.os, "killpg", lambda pid, sig: killed_pids.append(pid)
        )
        monkeypatch.setattr(utils_module.time, "sleep", lambda seconds: None)

        succeeded: Final = ProxyExtrasDBManager._run_migrations(use_migrate=True, use_v2_resolver=False)
        return succeeded, calls, killed_pids

    @staticmethod
    def _deploy_call_count(calls: list[list[str]]) -> int:
        return sum(1 for call in calls if tuple(call[-2:]) == ("migrate", "deploy"))

    @staticmethod
    def _error_messages(caplog: pytest.LogCaptureFixture) -> list[str]:
        return [
            record.getMessage()
            for record in caplog.records
            if record.levelno >= logging.ERROR and record.name.startswith("litellm_proxy_extras")
        ]

    def test_an_unrecognized_prisma_error_logs_its_stderr_at_error(
        self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path, caplog: pytest.LogCaptureFixture
    ) -> None:
        stderr: Final = "Error: P1001: Can't reach database server at db:5432"
        with caplog.at_level(logging.ERROR, logger="litellm_proxy_extras"):
            succeeded, calls, _ = self._run_v1_migrations(
                monkeypatch, tmp_path, deploy_stderr=stderr
            )

        assert succeeded is False
        assert self._deploy_call_count(calls) == 4
        assert any(stderr in message for message in self._error_messages(caplog))

    def test_an_unclassified_p3018_logs_its_stderr_and_retry_failure_at_error(
        self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path, caplog: pytest.LogCaptureFixture
    ) -> None:
        with caplog.at_level(logging.ERROR, logger="litellm_proxy_extras"):
            succeeded, calls, _ = self._run_v1_migrations(
                monkeypatch, tmp_path, deploy_stderr=_P3018_UNCLASSIFIED_STDERR
            )

        assert succeeded is False
        assert self._deploy_call_count(calls) == 4
        messages: Final = self._error_messages(caplog)
        assert any(
            "20260921190000_agent_identity" in message and "is duplicated" in message
            for message in messages
        )
        assert any(
            "The process failed to execute" in message and "Retrying... (3 attempts left)" in message
            for message in messages
        )

    def test_called_process_error_with_no_command_retries_all_v1_attempts(
        self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path
    ) -> None:
        import litellm_proxy_extras.utils as utils_module

        migration_dir: Final = tmp_path / "migration_dir"
        migration_dir.mkdir()
        monkeypatch.setenv("LITELLM_MIGRATION_DIR", str(migration_dir))
        monkeypatch.delenv("DATABASE_URL", raising=False)
        monkeypatch.delenv("DIRECT_URL", raising=False)
        monkeypatch.setenv("PRISMA_OFFLINE_MODE", "true")
        monkeypatch.setenv("PRISMA_CLI_PATH", sys.executable)
        monkeypatch.setattr(utils_module.time, "sleep", lambda seconds: None)
        calls: Final[list[None]] = []

        class _FakePrismaPopen:
            def __init__(
                self,
                argv: tuple[str, ...],
                *,
                env: Optional[dict[str, str]] = None,
                stdout: object = None,
                stderr: object = None,
                text: object = None,
                start_new_session: object = None,
            ) -> None:
                self.args: Final = None
                self.returncode: Final = 1
                calls.append(None)

            def __enter__(self) -> "_FakePrismaPopen":
                return self

            def __exit__(self, *args: object) -> None:
                return None

            def communicate(self, timeout: Optional[float] = None) -> tuple[str, str]:
                return "", "Error: P3018 unclassified"

        monkeypatch.setattr(
            utils_module.prisma_toolchain.subprocess, "Popen", _FakePrismaPopen
        )

        try:
            succeeded: Final = ProxyExtrasDBManager._run_migrations(
                use_migrate=True, use_v2_resolver=False
            )
        except TypeError as error:
            pytest.fail(
                f"_run_migrations raised TypeError after {len(calls)} Popen calls: {error}",
                pytrace=False,
            )

        assert succeeded is False
        assert len(calls) == 4

    def test_a_timeout_logs_at_error_naming_the_migrate_deploy_timeout_env_var(
        self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path, caplog: pytest.LogCaptureFixture
    ) -> None:
        from litellm_proxy_extras.prisma_toolchain import PRISMA_MIGRATE_DEPLOY_TIMEOUT_ENV_VAR

        with caplog.at_level(logging.ERROR, logger="litellm_proxy_extras"):
            succeeded, calls, killed_pids = self._run_v1_migrations(
                monkeypatch, tmp_path, deploy_timeout=True
            )

        assert succeeded is False
        assert self._deploy_call_count(calls) == 4
        assert killed_pids == [_FAKE_PRISMA_PID] * 4
        assert any(
            "timed out" in message and PRISMA_MIGRATE_DEPLOY_TIMEOUT_ENV_VAR in message
            for message in self._error_messages(caplog)
        )

    def test_a_recovered_baseline_logs_nothing_at_error(
        self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path, caplog: pytest.LogCaptureFixture
    ) -> None:
        with caplog.at_level(logging.ERROR, logger="litellm_proxy_extras"):
            succeeded, calls, _ = self._run_v1_migrations(
                monkeypatch,
                tmp_path,
                deploy_stderr=_P3005_STDERR,
                database_url="postgresql://user:pass@db:5432/litellm",
            )

        assert succeeded is True
        assert tuple(calls[0][-2:]) == ("migrate", "deploy")
        assert self._error_messages(caplog) == []

    def test_a_failed_baseline_recovery_logs_its_stderr_at_error(
        self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path, caplog: pytest.LogCaptureFixture
    ) -> None:
        database_url: Final = "postgresql://llmproxy:s3cr3t 'p\"w@db:5432/litellm"
        monkeypatch.delenv("DIRECT_URL", raising=False)
        with caplog.at_level(logging.DEBUG, logger="litellm_proxy_extras"):
            succeeded, calls, _ = self._run_v1_migrations(
                monkeypatch,
                tmp_path,
                deploy_stderr=_P3005_STDERR,
                diff_stderr=f"baseline diff failed: XYZ-7731 for {database_url}",
                database_url=database_url,
            )

        assert succeeded is False
        assert self._deploy_call_count(calls) == 4
        assert [
            record.getMessage()
            for record in caplog.records
            if "s3cr3t" in record.getMessage() or 'p"w' in record.getMessage()
        ] == []
        messages: Final = self._error_messages(caplog)
        assert any("postgresql://REDACTED@db:5432/litellm" in message for message in messages)
        assert any("XYZ-7731" in message for message in messages)


@pytest.mark.parametrize(
    "database_url,direct_url,text,expected",
    (
        (
            "postgresql://u:pa ss@db:5432/litellm",
            None,
            'Error: P1000: Authentication failed against database server at "postgresql://u:pa ss@db:5432/litellm"',
            'Error: P1000: Authentication failed against database server at "postgresql://REDACTED@db:5432/litellm"',
        ),
        (
            "postgresql://u:pa'ss@db:5432/litellm",
            None,
            "postgresql://u:pa'ss@db:5432/litellm",
            "postgresql://REDACTED@db:5432/litellm",
        ),
        (
            'postgresql://u:pa"ss@db:5432/litellm',
            None,
            'postgresql://u:pa"ss@db:5432/litellm',
            "postgresql://REDACTED@db:5432/litellm",
        ),
        (
            "postgresql://u:p@ss@db:5432/litellm",
            None,
            "postgresql://u:p@ss@db:5432/litellm",
            "postgresql://REDACTED@db:5432/litellm",
        ),
        (
            "postgresql://u:p%20ss@db:5432/litellm",
            None,
            "postgresql://u:p ss@db:5432/litellm",
            "postgresql://REDACTED@db:5432/litellm",
        ),
        (
            "postgresql://db/litellm?password=a b&sslmode=require",
            None,
            "postgresql://db/litellm?password=a b&sslmode=require",
            "postgresql://db/litellm?REDACTED&sslmode=require",
        ),
        (
            "postgresql://db/litellm?sslpassword=zq'7x",
            None,
            "postgresql://db/litellm?sslpassword=zq'7x",
            "postgresql://db/litellm?REDACTED",
        ),
        (
            None,
            "postgresql://u:pa ss@db:5432/litellm",
            "postgresql://u:pa ss@db:5432/litellm",
            "postgresql://REDACTED@db:5432/litellm",
        ),
        (
            None,
            None,
            "postgresql://u:pw@db/x",
            "postgresql://REDACTED@db/x",
        ),
        (
            "postgresql://u:p@db:5432/litellm",
            None,
            "Error: P1001: Can't reach database server at db:5432",
            "Error: P1001: Can't reach database server at db:5432",
        ),
        (
            "postgresql://u:p@db:5432/litellm",
            None,
            "Error:P1001: Can't reach database server at db:5432",
            "Error:P1001: Can't reach database server at db:5432",
        ),
        (
            None,
            None,
            "plain text with no URL",
            "plain text with no URL",
        ),
    ),
)
def test_redact_credentials_masks_passwords_in_embedded_urls(
    database_url: str | None,
    direct_url: str | None,
    text: str,
    expected: str,
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    if database_url is None:
        monkeypatch.delenv("DATABASE_URL", raising=False)
    else:
        monkeypatch.setenv("DATABASE_URL", database_url)
    if direct_url is None:
        monkeypatch.delenv("DIRECT_URL", raising=False)
    else:
        monkeypatch.setenv("DIRECT_URL", direct_url)
    assert _redact_credentials(text) == expected


@pytest.mark.parametrize("password", ("zq'7x", 'zq"7x', "zq'\"7x", "zq 7x", "zq@7x"))
def test_redact_command_error_masks_url_arguments(password: str, monkeypatch: pytest.MonkeyPatch) -> None:
    database_url: Final = f"postgresql://u:{password}@db:5432/litellm"
    monkeypatch.setenv("DATABASE_URL", database_url)
    monkeypatch.delenv("DIRECT_URL", raising=False)
    error: Final = subprocess.CalledProcessError(1, ["prisma", "migrate", "diff", "--to-url", database_url])

    message: Final = _redact_command_error(error)

    assert "zq" not in message
    assert "7x" not in message
    assert "postgresql://REDACTED@db:5432/litellm" in message
    assert "returned non-zero exit status 1" in message


@pytest.mark.parametrize(
    "command",
    (
        None,
        Path("/usr/bin/prisma"),
        7,
        ("prisma", "migrate", "deploy"),
        ["prisma", "migrate", "deploy"],
        "prisma migrate deploy",
    ),
)
def test_redact_command_error_preserves_unredacted_command_format(
    command: object, monkeypatch: pytest.MonkeyPatch
) -> None:
    monkeypatch.delenv("DATABASE_URL", raising=False)
    monkeypatch.delenv("DIRECT_URL", raising=False)
    error: Final = subprocess.CalledProcessError(1, command)

    assert _redact_command_error(error) == str(error)


def test_redact_command_error_masks_password_in_tuple_url_argument(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    password: Final = "zq 7x"
    database_url: Final = f"postgresql://u:{password}@db:5432/litellm"
    monkeypatch.setenv("DATABASE_URL", database_url)
    monkeypatch.delenv("DIRECT_URL", raising=False)
    error: Final = subprocess.CalledProcessError(1, ("prisma", "migrate", "deploy", "--to-url", database_url))

    message: Final = _redact_command_error(error)

    assert message.startswith("Command '('")
    assert password not in message
    assert "postgresql://REDACTED@db:5432/litellm" in message


def test_repair_survives_an_unreachable_database(monkeypatch: pytest.MonkeyPatch) -> None:
    monkeypatch.delenv("DIRECT_URL", raising=False)
    monkeypatch.setenv("DATABASE_URL", "postgresql://u:[email protected]:9/x?schema=whatever")

    assert ProxyExtrasDBManager.repair_invalid_indexes() is False


@pytest.mark.parametrize(
    "environment",
    (
        {"DATABASE_URL": "postgresql://pool/db"},
        {"DATABASE_URL": "postgresql://pool/db?pgbouncer=true", "DIRECT_URL": "postgresql://writer/db"},
    ),
    ids=("pooled-only", "pooled-with-direct"),
)
def test_prisma_runs_its_migrations_without_a_session_advisory_lock(environment):
    from litellm_proxy_extras.migration_lock import migration_environment

    migrated = migration_environment(environment)

    assert migrated["PRISMA_SCHEMA_DISABLE_ADVISORY_LOCK"] == "1"
    assert "PRISMA_SCHEMA_DISABLE_ADVISORY_LOCK" not in migration_environment({"PRISMA_OFFLINE_MODE": "true"})


class TestMigrateDeployRunsUnderTheCoordinatorLock:
    """`prisma migrate deploy` only runs while the injected lock is held, and the lock is
    released again before the resolver handles the outcome, success or failure."""

    @staticmethod
    def _recording_lock(events: list[str]):
        from contextlib import contextmanager

        @contextmanager
        def lock():
            events.append("lock")
            try:
                yield
            finally:
                events.append("unlock")

        return lock

    def test_deploy_runs_between_lock_and_unlock(self, monkeypatch, tmp_path):
        import litellm_proxy_extras.utils as utils_module

        events: list[str] = []
        harness = _MigrateDeployHarness(monkeypatch, tmp_path, ["ok"])
        original_run = harness._fake_run

        def run_and_record(cmd, **kwargs):
            events.append("deploy")
            return original_run(cmd, **kwargs)

        monkeypatch.setattr(utils_module.prisma_toolchain, "run_prisma", run_and_record)

        assert ProxyExtrasDBManager._run_database_v2(True, deploy_lock=self._recording_lock(events)) is True
        assert events == ["lock", "deploy", "unlock"]

    def test_a_failed_deploy_releases_the_lock_before_the_retry(self, monkeypatch, tmp_path):
        import litellm_proxy_extras.utils as utils_module

        events: list[str] = []
        harness = _MigrateDeployHarness(
            monkeypatch, tmp_path, ["Error: P1002\nTimed out waiting for the advisory lock", "ok"]
        )
        original_run = harness._fake_run

        def run_and_record(cmd, **kwargs):
            events.append("deploy")
            return original_run(cmd, **kwargs)

        monkeypatch.setattr(utils_module.prisma_toolchain, "run_prisma", run_and_record)

        assert ProxyExtrasDBManager._run_database_v2(True, deploy_lock=self._recording_lock(events)) is True
        assert events == ["lock", "deploy", "unlock", "lock", "deploy", "unlock"]

    def test_a_deadlock_rollback_runs_under_the_coordinator_lock(
        self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path
    ) -> None:
        import litellm_proxy_extras.utils as utils_module

        migration_name: Final = "20260921190000_agent_identity"
        stderr: Final = (
            "Error: P3018\n\n"
            "A migration failed to apply. New migrations cannot be applied before the error is recovered from.\n\n"
            f"Migration name: {migration_name}\n\n"
            "Database error code: 40P01\n\n"
            f"Database error:\n{_P3009_DEADLOCK_LOGS}"
        )
        events: Final[list[str]] = []
        harness: Final = _MigrateDeployHarness(monkeypatch, tmp_path, [stderr, "ok"])
        original_run: Final = harness._fake_run

        def run_and_record(cmd: Sequence[str], *, timeout: float, env: Mapping[str, str]) -> _FakeCompleted:
            if cmd[1:] == ["migrate", "deploy"]:
                events.append("deploy")
            elif cmd[1:4] == ["migrate", "resolve", "--rolled-back"]:
                events.append("rollback")
            return original_run(cmd, timeout=timeout, env=env)

        monkeypatch.setattr(utils_module.prisma_toolchain, "run_prisma", run_and_record)

        assert ProxyExtrasDBManager._run_database_v2(True, deploy_lock=self._recording_lock(events)) is True
        assert events == ["lock", "deploy", "unlock", "lock", "rollback", "unlock", "lock", "deploy", "unlock"]