Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

26.8 KB
Raw Blame
"""Notification Worker Tasks

This module contains Celery tasks for processing all types of notifications.
It uses the provider registry pattern for platform-specific notification handling
while maintaining backward compatibility.
"""

import os
import time
from typing import Any

import httpx
from notification.enums import PlatformType
from notification.providers.base_provider import (
    DeliveryError,
    NotificationError,
    ValidationError,
)
from notification.providers.registry import create_provider_from_config
from notification.providers.webhook_provider import WebhookProvider
from notification.utils import (
    log_notification_attempt,
    log_notification_failure,
    log_notification_success,
)
from queue_backend import worker_task
from shared.infrastructure.config import WorkerConfig
from shared.infrastructure.logging import WorkerLogger

from unstract.core.network.ssrf import safe_host
from unstract.core.notification_enums import NotificationType

logger = WorkerLogger.get_logger(__name__)


class TerminalWebhookRefusal(Exception):
    """A refusal no retry can clear, already dead-lettered by its raiser.

    Distinct from the errors below it so ``send_webhook_notification``'s broad
    ``except Exception`` cannot route it back into ``self.retry`` — a plain
    ``Exception`` here was retried ``max_retries`` times for a URL that will
    never be dialable, and dead-lettered the buffer rows a second time.
    """


# Initialize worker configuration
config = WorkerConfig.from_env("NOTIFICATION")


def _get_webhook_provider_for_url(url: str):
    """Get webhook provider for backward compatibility.

    For backward compatibility with legacy webhook tasks that don't provide platform info.
    Always defaults to API provider since platform should come from notification configuration.

    Args:
        url: Webhook URL (used for logging only)

    Returns:
        API webhook provider instance
    """
    logger.debug(
        f"Legacy webhook task called without platform info for URL: {url[:50]}..."
    )

    # Always use API provider for backward compatibility
    # Platform detection should be done in backend and stored in database
    try:
        config = {
            "notification_type": NotificationType.WEBHOOK.value,
            "platform": PlatformType.API.value,
        }
        return create_provider_from_config(config)
    except Exception as e:
        logger.warning(
            f"Failed to create API provider: {e}. Using fallback WebhookProvider"
        )
        return WebhookProvider()


@worker_task(name="process_notification")
def process_notification(
    notification_type: str, priority: bool = False, **kwargs: Any
) -> dict[str, Any]:
    """Universal notification processor.

    This task routes notifications to the appropriate provider based on type.
    It provides a unified interface for all notification types while maintaining
    extensibility for future implementations.

    Args:
        notification_type: Type of notification (WEBHOOK, EMAIL, SMS, PUSH)
        **kwargs: Notification-specific parameters

    Returns:
        Dictionary containing the processing result

    Raises:
        NotificationError: If notification processing fails
    """
    destination = (
        kwargs.get("url") or kwargs.get("email") or kwargs.get("phone") or "unknown"
    )

    try:
        logger.info(f"Processing {notification_type} notification to {destination}")

        # Use the registry pattern with platform from config
        if notification_type == NotificationType.WEBHOOK.value:
            platform = kwargs.get("platform")
            if platform:
                config = {"notification_type": notification_type, "platform": platform}
                provider = create_provider_from_config(config)
                logger.debug(f"Selected provider: {provider.__class__.__name__}")
            else:
                # Backward compatibility: Default to API provider
                logger.warning("No platform specified, using API provider")
                provider = WebhookProvider()
        else:
            # For future notification types (EMAIL, SMS, etc.)
            raise NotificationError(
                f"Unsupported notification type: {notification_type}",
                provider="NotificationDispatcher",
                destination=destination,
            )

        log_notification_attempt(
            notification_type=notification_type,
            destination=destination,
            attempt=1,  # TODO: Track actual retry attempts
        )

        # Send notification
        result = provider.send(kwargs)

        if result.get("success"):
            log_notification_success(
                notification_type=notification_type,
                destination=destination,
                attempt=result.get("attempts", 1),
                response_info=result.get("details"),
            )
        else:
            log_notification_failure(
                notification_type=notification_type,
                destination=destination,
                error=Exception(result.get("message", "Unknown error")),
                attempt=result.get("attempts", 1),
                is_final=True,
            )

        return result

    except (ValidationError, DeliveryError) as e:
        logger.error(f"Notification error: {str(e)}")
        log_notification_failure(
            notification_type=notification_type,
            destination=destination,
            error=e,
            attempt=1,
            is_final=True,
        )
        return {
            "success": False,
            "message": str(e),
            "destination": destination,
            "error_type": e.__class__.__name__,
        }
    except Exception as e:
        logger.error(
            f"Unexpected error processing {notification_type} notification: {str(e)}"
        )
        return {
            "success": False,
            "message": f"Unexpected error: {str(e)}",
            "destination": destination,
            "error_type": e.__class__.__name__,
        }


def _mark_buffer_outcome(
    buffer_row_ids: list[str] | None,
    organization_id: Any,
    *,
    dispatched: bool,
) -> None:
    """Report a clubbed dispatch's buffer rows as DISPATCHED / DEAD_LETTER.

    Replaces the old Celery ``link``/``link_error`` callbacks with an internal
    HTTP POST, so marking no longer depends on a backend task being registered
    on whichever worker happens to drain the ``celery`` queue. Bearer-authed
    with org scoping carried in the body (matches the ``buffer/process`` flush
    call style — ``organization_id`` is the numeric org pk, not a header).
    Best-effort: if this POST fails the rows stay SENDING and the backend reaper
    reclaims them after the lease — the same safety net as before.
    """
    if not buffer_row_ids or organization_id is None:
        return
    base_url = os.getenv("INTERNAL_API_BASE_URL")
    api_key = os.getenv("INTERNAL_SERVICE_API_KEY")
    if not base_url or not api_key:
        logger.warning(
            "Cannot mark buffer rows: INTERNAL_API_BASE_URL / "
            "INTERNAL_SERVICE_API_KEY not set"
        )
        return
    suffix = "dispatched" if dispatched else "dead-letter"
    url = f"{base_url.rstrip('/')}/v1/webhook/buffer/mark/{suffix}/"
    try:
        with httpx.Client(transport=httpx.HTTPTransport(retries=2)) as client:
            response = client.post(
                url,
                headers={"Authorization": f"Bearer {api_key}"},
                json={
                    "buffer_row_ids": buffer_row_ids,
                    "organization_id": organization_id,
                },
                timeout=10.0,
            )
        if response.status_code != 200:
            logger.warning(
                "metric=notification_buffer_mark_failed_total result=%s "
                "reason=http_%s rows=%d body=%s; reaper will reclaim",
                suffix,
                response.status_code,
                len(buffer_row_ids),
                response.text[:200],
            )
    except Exception as e:  # noqa: BLE001
        logger.warning(
            "metric=notification_buffer_mark_failed_total result=%s "
            "reason=exception rows=%d exc=%r; reaper will reclaim",
            suffix,
            len(buffer_row_ids),
            e,
        )


@worker_task(bind=True, name="send_webhook_notification")
def send_webhook_notification(
    self,
    url: str,
    payload: Any,
    headers: Any = None,
    timeout: int = 10,
    max_retries: int | None = None,
    retry_delay: int = 10,
    platform: str | None = None,
    raise_on_final_failure: bool = False,
    buffer_row_ids: list[str] | None = None,
    organization_id: str | None = None,
) -> None:
    """Backward compatible webhook notification task.

    This task maintains 100% compatibility with the existing backend
    send_webhook_notification task. It delegates to the WebhookProvider
    but preserves the exact same interface and behavior.

    Args:
        url: The URL to which the webhook should be sent
        payload: The payload to be sent in the webhook request
        headers: Optional headers to include in the request
        timeout: The request timeout in seconds
        max_retries: The maximum number of retries allowed
        retry_delay: The delay between retries in seconds
        platform: Platform type from notification config (SLACK, API, etc.)
        raise_on_final_failure: When True, re-raise on retry exhaustion so the
            task ends in FAILURE (a Celery monitoring signal). NOTE: the buffer
            rows are dead-lettered by ``_mark_buffer_outcome(dispatched=False)``
            over the internal API *before* this re-raise, on BOTH transports —
            ``link``/``link_error`` callbacks are no longer wired (they routed to
            the ``celery`` queue and were dropped as "unregistered task"), so the
            raise itself dead-letters nothing. When False (default), preserve the
            legacy "return None" behavior. The PG dispatch seam forces this False
            (and ``max_retries`` 0) because under the consumer's eager
            ``task.apply()`` a raise means redelivery, not a FAILURE state.

    Returns:
        None (matches original behavior)

    Raises:
        Exception: If webhook delivery fails (for Celery retry mechanism), or on
            final failure when ``raise_on_final_failure`` is set.
    """
    try:
        logger.debug(
            f"[{os.getpid()}] Processing webhook notification to {url} "
            f"(attempt {self.request.retries + 1})"
        )
        logger.debug(f"Task received platform parameter: {platform}")
        logger.debug(f"Task received payload type: {type(payload)}")
        logger.debug(f"Task received headers: {headers}")

        # Use platform-specific provider if provided, otherwise default to API for backward compatibility
        if platform:
            config = {
                "notification_type": NotificationType.WEBHOOK.value,
                "platform": platform,
            }
            webhook_provider = create_provider_from_config(config)
        else:
            webhook_provider = _get_webhook_provider_for_url(url)

        # Prepare notification data in the format expected by WebhookProvider
        notification_data = {
            "url": url,
            "payload": payload,
            "headers": headers,
            "timeout": timeout,
            "max_retries": max_retries,
            "retry_delay": retry_delay,
            "platform": platform,
        }

        # Send webhook notification
        result = webhook_provider.send(notification_data)

        # Handle result based on success/failure
        if result.get("success"):
            logger.info(
                f"Webhook delivered successfully to {url} "
                f"(status: {result.get('details', {}).get('status_code', 'unknown')})"
            )
            _mark_buffer_outcome(buffer_row_ids, organization_id, dispatched=True)
            return None  # Success - matches original behavior
        else:
            error_message = result.get("message", "Unknown webhook delivery error")

            # A refusal the sink marked non-retryable is a property of the URL
            # itself, so no attempt can change it. Dead-letter now instead of
            # re-resolving a tenant-supplied hostname up to max_retries times.
            if result.get("details", {}).get("retryable") is False:
                # The host, not the URL: a webhook URL routinely carries a
                # token in its query string and this line goes to shared logs.
                logger.error(
                    f"Webhook to host={safe_host(url)} refused and not "
                    f"retryable: {error_message}"
                )
                _mark_buffer_outcome(buffer_row_ids, organization_id, dispatched=False)
                if raise_on_final_failure:
                    raise TerminalWebhookRefusal(error_message)
                return None

            # Failed delivery - raise exception for retry handling
            raise Exception(error_message)

    except TerminalWebhookRefusal:
        # Buffer rows are already dead-lettered above and no attempt can change
        # the outcome. Surface FAILURE to the caller without re-entering retry.
        raise

    except (ValidationError, DeliveryError) as e:
        # Handle provider-specific errors
        if max_retries is not None:
            if self.request.retries < max_retries:
                logger.warning(
                    f"Request to {url} failed. Retrying in {retry_delay} seconds. "
                    f"Attempt {self.request.retries + 1}/{max_retries}. Error: {e}"
                )
                # Use Celery's retry mechanism - identical to original behavior
                raise self.retry(exc=e, countdown=retry_delay)
            else:
                logger.error(
                    f"Failed to send webhook to {url} after {max_retries} attempts. "
                    f"Error: {e}"
                )
                _mark_buffer_outcome(buffer_row_ids, organization_id, dispatched=False)
                if raise_on_final_failure:
                    raise
                return None  # Final failure - matches original behavior
        else:
            logger.error(f"Webhook request to {url} failed with error: {e}")
            _mark_buffer_outcome(buffer_row_ids, organization_id, dispatched=False)
            if raise_on_final_failure:
                raise
            return None  # No retries configured - matches original behavior

    except Exception as e:
        # Handle unexpected errors - preserve original retry logic
        if max_retries is not None:
            if self.request.retries < max_retries:
                logger.warning(
                    f"Request to {url} failed. Retrying in {retry_delay} seconds. "
                    f"Attempt {self.request.retries + 1}/{max_retries}. Error: {e}"
                )
                raise self.retry(exc=e, countdown=retry_delay)
            else:
                logger.error(
                    f"Failed to send webhook to {url} after {max_retries} attempts. "
                    f"Error: {e}"
                )
                _mark_buffer_outcome(buffer_row_ids, organization_id, dispatched=False)
                if raise_on_final_failure:
                    raise
                return None
        else:
            logger.error(f"Webhook request to {url} failed with error: {e}")
            _mark_buffer_outcome(buffer_row_ids, organization_id, dispatched=False)
            if raise_on_final_failure:
                raise
            return None


@worker_task(name="send_batch_notifications")
def send_batch_notifications(
    notifications: list[dict[str, Any]],
    batch_id: str | None = None,
    delay_between: int = 0,
) -> dict[str, Any]:
    """Send multiple notifications in batch.

    This task processes multiple notifications with optional delays between them.
    It's designed for future enhancement of the notification system.

    Args:
        notifications: List of notification configurations
        batch_id: Optional batch identifier
        delay_between: Delay between notifications in seconds

    Returns:
        Dictionary with batch processing results
    """
    import uuid
    from datetime import datetime

    batch_id = batch_id or str(uuid.uuid4())

    logger.info(f"Processing batch {batch_id} with {len(notifications)} notifications")

    results = {
        "batch_id": batch_id,
        "total_notifications": len(notifications),
        "successful": [],
        "failed": [],
        "started_at": datetime.now().isoformat(),
    }

    for i, notification in enumerate(notifications):
        try:
            notification_type = notification.get("type", NotificationType.WEBHOOK.value)

            # Add delay between notifications if specified
            if delay_between > 0 and i > 0:
                import time

                time.sleep(delay_between)

            # Process notification
            result = process_notification(notification_type, **notification)

            if result.get("success"):
                results["successful"].append(
                    {
                        "index": i,
                        "destination": result.get("destination"),
                        "type": notification_type,
                    }
                )
            else:
                results["failed"].append(
                    {
                        "index": i,
                        "destination": result.get("destination"),
                        "type": notification_type,
                        "error": result.get("message"),
                    }
                )

        except Exception as e:
            logger.error(f"Batch notification {i} failed: {str(e)}")
            results["failed"].append(
                {
                    "index": i,
                    "destination": notification.get("url", "unknown"),
                    "type": notification.get("type", "unknown"),
                    "error": str(e),
                }
            )

    results["completed_at"] = datetime.now().isoformat()

    logger.info(
        f"Batch {batch_id} completed: {len(results['successful'])} successful, "
        f"{len(results['failed'])} failed"
    )

    return results


@worker_task(name="priority_notification")
def priority_notification(notification_type: str, **kwargs: Any) -> dict[str, Any]:
    """High-priority notification processor.

    This task is routed to the priority queue for urgent notifications
    that need immediate processing.

    Args:
        notification_type: Type of notification (WEBHOOK, EMAIL, SMS, PUSH)
        **kwargs: Notification-specific parameters

    Returns:
        Dictionary containing the processing result
    """
    logger.info(f"Processing priority {notification_type} notification")

    # Set priority flag and delegate to main processor
    return process_notification(notification_type, priority=True, **kwargs)


# Retries for a transient backend problem (restart, 5xx). Kept inside the task
# so a brief blip is absorbed here rather than costing a full lease-expiry
# redelivery (minutes) plus one of the consumer's bounded attempts.
_GROUP_NOTIFICATION_ATTEMPTS = 3
_GROUP_NOTIFICATION_RETRY_DELAY = 2.0
# Per-phase, because httpx has NO whole-request timeout: a scalar one is applied
# to connect, write and read separately. The loop below is the only retry --
# transport-level retries would stack their own timeouts underneath these.
# Worst case per attempt is connect+write+read+pool = 50s. Across
# _GROUP_NOTIFICATION_ATTEMPTS attempts plus the sleep between each retry,
# that's a NOMINAL budget of 3*50 + 2*_GROUP_NOTIFICATION_RETRY_DELAY = 154s --
# not a hard bound, since ``read`` times out per socket read, not on the
# total: a response trickling in under 30s per chunk runs past this
# regardless. Sized to stay under WORKER_PG_QUEUE_CONSUMER_HEALTH_STALE_SECONDS
# (the heartbeat is frozen for the task's duration) and VT_SECONDS in the
# common case. Excludes DNS, which connect does not cover.
_GROUP_NOTIFICATION_TIMEOUT = httpx.Timeout(connect=5.0, write=10.0, read=30.0, pool=5.0)


def _build_group_notification_request(
    endpoint: str, organization_id: str
) -> tuple[str, dict[str, str]]:
    """URL and headers for one group-notification POST."""
    base_url = os.getenv("INTERNAL_API_BASE_URL")
    api_key = os.getenv("INTERNAL_SERVICE_API_KEY")
    if not base_url or not api_key:
        raise RuntimeError(
            "INTERNAL_API_BASE_URL / INTERNAL_SERVICE_API_KEY not set; "
            "cannot send group notification"
        )
    url = f"{base_url.rstrip('/')}/v1/group-notification/{endpoint}/"
    headers = {
        "Authorization": f"Bearer {api_key}",
        # The backend resolves the tenant from this header; without it every
        # org-scoped query comes back empty.
        "X-Organization-ID": organization_id,
    }
    return url, headers


def _post_group_notification_once(
    url: str, headers: dict[str, str], payload: dict
) -> tuple[bool, bool, str]:
    """One POST attempt. Returns ``(succeeded, retryable, error)``.

    A response lost after the backend already received it is treated like a
    sub-500 (not retryable): the backend does not stop when we disconnect and
    mails group by group with no checkpoint, so re-posting would re-mail every
    group that already succeeded.
    """
    try:
        with httpx.Client() as client:
            response = client.post(
                url, headers=headers, json=payload, timeout=_GROUP_NOTIFICATION_TIMEOUT
            )
    except (
        httpx.ReadTimeout,
        httpx.WriteTimeout,
        httpx.ReadError,
        httpx.WriteError,
        httpx.RemoteProtocolError,
    ) as e:
        return False, False, f"timeout_after_send={e!r}"
    except Exception as e:  # noqa: BLE001
        return False, True, f"exception={e!r}"
    if response.status_code == 200:
        return True, False, ""
    error = f"http_{response.status_code} body={response.text[:200]}"
    return False, response.status_code >= 500, error


def _fail_group_notification(endpoint: str, organization_id: str, error: str) -> None:
    """Log and raise once a retryable failure exhausts its in-process attempts.

    The raise leaves the message on the queue for redelivery -- correct here
    because the failure is transient (5xx / connection-level), so a later
    attempt has a real chance of succeeding.
    """
    logger.error(
        "metric=group_notification_post_failed_total endpoint=%s org_id=%s error=%s",
        endpoint,
        organization_id,
        error,
    )
    raise RuntimeError(f"Group notification {endpoint} failed: {error}")


def _drop_group_notification(
    endpoint: str, organization_id: str, error: str, payload: dict
) -> None:
    """Log a permanent failure without raising.

    A non-retryable failure (a definitive 4xx, or a response lost after the
    backend already sent the group's emails) will not succeed on redelivery --
    and since one send call mails a whole group with no per-recipient
    checkpoint, redelivering it re-mails everyone who already got it. Raising
    here would trade a dropped notification for a duplicated one.

    The message is acked and deleted once this returns -- nothing else records
    what was lost, so the payload goes in the log line (a dropped *revoke* is
    compliance-visible, not just an inconvenience).
    """
    logger.error(
        "metric=group_notification_dropped_total endpoint=%s org_id=%s error=%s "
        "payload=%s",
        endpoint,
        organization_id,
        error,
        payload,
    )


def _post_group_notification(endpoint: str, organization_id: str, payload: dict) -> None:
    """POST a group-notification job to the backend and insist it succeeded.

    Raises only on a retryable failure -- nothing tracks an unsent group
    email, so a swallowed transient error would be a silent drop, and the
    queue's own redelivery is the backstop for that. A non-retryable failure
    is dropped instead of raised: see :func:`_drop_group_notification`.
    """
    url, headers = _build_group_notification_request(endpoint, organization_id)
    last_error = ""
    # Seeded False: a zero-iteration loop (only possible if
    # _GROUP_NOTIFICATION_ATTEMPTS were ever misconfigured to <= 0) means
    # nothing was ever attempted, so drop rather than raise -- raising here
    # would redeliver forever with no attempt ever being made.
    retryable = False
    for attempt in range(1, _GROUP_NOTIFICATION_ATTEMPTS + 1):
        succeeded, retryable, last_error = _post_group_notification_once(
            url, headers, payload
        )
        if succeeded:
            return
        if not retryable:
            break
        if attempt < _GROUP_NOTIFICATION_ATTEMPTS:
            logger.warning(
                "Group notification %s attempt %d/%d failed (%s); retrying",
                endpoint,
                attempt,
                _GROUP_NOTIFICATION_ATTEMPTS,
                last_error,
            )
            time.sleep(_GROUP_NOTIFICATION_RETRY_DELAY)
    if retryable:
        _fail_group_notification(endpoint, organization_id, last_error)
    else:
        _drop_group_notification(endpoint, organization_id, last_error, payload)


@worker_task(name="notify_resource_shared_with_group")
def notify_resource_shared_with_group(
    group_ids: list[int],
    actor_id: int,
    resource_kind: str,
    resource_id: str,
    organization_id: str,
    share_action: str,
    revoked_at: str | None = None,
) -> None:
    """Email every current member of the groups whose access just changed.

    ``revoked_at`` is set on a revoke only; the backend uses it to skip members
    who joined the group after the access was taken away.
    """
    _post_group_notification(
        "resource-shared",
        organization_id,
        {
            "group_ids": group_ids,
            "actor_id": actor_id,
            "resource_kind": resource_kind,
            "resource_id": resource_id,
            "share_action": share_action,
            "revoked_at": revoked_at,
        },
    )


@worker_task(name="notify_group_membership_changed")
def notify_group_membership_changed(
    group_id: int,
    actor_id: int,
    membership_action: str,
    user_ids: list[int],
    organization_id: str,
) -> None:
    """Email the users whose membership of a group just changed."""
    _post_group_notification(
        "membership-changed",
        organization_id,
        {
            "group_id": group_id,
            "actor_id": actor_id,
            "membership_action": membership_action,
            "user_ids": user_ids,
        },
    )


@worker_task(name="notification_health_check")
def notification_health_check() -> dict[str, Any]:
    """Health check task for notification worker."""
    try:
        queue_name = os.getenv("NOTIFICATION_QUEUE_NAME", "notifications")
        return {
            "worker": "notification",
            "status": "healthy",
            "queue": queue_name,
        }
    except Exception as e:
        return {"worker": "notification", "status": "unhealthy", "error": str(e)}