Skip to content

armgit / plane

Public Imported from GitHub
0

πŸ”₯πŸ”₯πŸ”₯ Open-source Jira, Linear, Monday, and ClickUp alternative. Plane is a modern project management platform to manage tasks, sprints, docs, and triage.

refactor: logging with retention + API token hardening (#9148)

sriram veeraghantaedf247541301
diff --git a/.gitignore b/.gitignore
index 6f329c7393..61351715a1 100644
--- a/.gitignore
+++ b/.gitignore
@@ -114,3 +114,6 @@ scripts/
# i18n auto-generated types (regenerated on every build)
packages/i18n/src/types/keys.generated.ts
+
+# Local security advisory notes (not for version control)
+/advisories.md
diff --git a/apps/api/plane/app/serializers/api.py b/apps/api/plane/app/serializers/api.py
index 05c6198f59..71ffd442bf 100644
--- a/apps/api/plane/app/serializers/api.py
+++ b/apps/api/plane/app/serializers/api.py
@@ -22,6 +22,7 @@ class APITokenSerializer(BaseSerializer):
"is_active",
"last_used",
"user_type",
+ "allowed_rate_limit",
]
diff --git a/apps/api/plane/bgtasks/cleanup_task.py b/apps/api/plane/bgtasks/cleanup_task.py
index 407a67ca69..3414438ccf 100644
--- a/apps/api/plane/bgtasks/cleanup_task.py
+++ b/apps/api/plane/bgtasks/cleanup_task.py
@@ -5,19 +5,16 @@
# Python imports
from datetime import timedelta
import logging
-from typing import List, Dict, Any, Callable, Optional
-import os
+from typing import Callable, Iterable
# Django imports
+from django.conf import settings
from django.utils import timezone
from django.db.models import F, Window, Subquery
from django.db.models.functions import RowNumber
# Third party imports
from celery import shared_task
-from pymongo.errors import BulkWriteError
-from pymongo.collection import Collection
-from pymongo.operations import InsertOne
# Module imports
from plane.db.models import (
@@ -27,7 +24,6 @@ from plane.db.models import (
IssueDescriptionVersion,
WebhookLog,
)
-from plane.settings.mongo import MongoConnection
from plane.utils.exception_logger import log_exception
@@ -35,285 +31,75 @@ logger = logging.getLogger("plane.worker")
BATCH_SIZE = 500
-def get_mongo_collection(collection_name: str) -> Optional[Collection]:
- """Get MongoDB collection if available, otherwise return None."""
- if not MongoConnection.is_configured():
- logger.info("MongoDB not configured")
- return None
-
- try:
- mongo_collection = MongoConnection.get_collection(collection_name)
- logger.info(f"MongoDB collection '{collection_name}' connected successfully")
- return mongo_collection
- except Exception as e:
- logger.error(f"Failed to get MongoDB collection: {str(e)}")
- log_exception(e)
- return None
-
-
-def flush_to_mongo_and_delete(
- mongo_collection: Optional[Collection],
- buffer: List[Dict[str, Any]],
- ids_to_delete: List[int],
- model,
- mongo_available: bool,
-) -> None:
- """
- Inserts a batch of records into MongoDB and deletes the corresponding rows from PostgreSQL.
- """
- if not buffer:
- logger.debug("No records to flush - buffer is empty")
- return
-
- logger.info(f"Starting batch flush: {len(buffer)} records, {len(ids_to_delete)} IDs to delete")
-
- mongo_archival_failed = False
-
- # Try to insert into MongoDB if available
- if mongo_collection is not None and mongo_available:
- try:
- mongo_collection.bulk_write([InsertOne(doc) for doc in buffer])
- except BulkWriteError as bwe:
- logger.error(f"MongoDB bulk write error: {str(bwe)}")
- log_exception(bwe)
- mongo_archival_failed = True
-
- # If MongoDB is available and archival failed, log the error and return
- if mongo_available and mongo_archival_failed:
- logger.error(f"MongoDB archival failed for {len(buffer)} records")
- return
-
- # Delete from PostgreSQL - delete() returns (count, {model: count})
- delete_result = model.all_objects.filter(id__in=ids_to_delete).delete()
- deleted_count = delete_result[0] if delete_result and isinstance(delete_result, tuple) else 0
- logger.info(f"Batch flush completed: {deleted_count} records deleted")
-
-
def process_cleanup_task(
- queryset_func: Callable,
- transform_func: Callable[[Dict], Dict],
+ queryset_func: Callable[[], Iterable],
model,
task_name: str,
- collection_name: str,
):
"""
- Generic function to process cleanup tasks.
+ Batch-delete expired rows for the given model from PostgreSQL.
Args:
- queryset_func: Function that returns the queryset to process
- transform_func: Function to transform each record for MongoDB
- model: Django model class
- task_name: Name of the task for logging
- collection_name: MongoDB collection name
+ queryset_func: Callable returning an iterable of primary keys to delete.
+ model: Django model class.
+ task_name: Name of the task for logging.
"""
logger.info(f"Starting {task_name} cleanup task")
- # Get MongoDB collection
- mongo_collection = get_mongo_collection(collection_name)
- mongo_available = mongo_collection is not None
-
- # Get queryset
- queryset = queryset_func()
-
- # Process records in batches
- buffer: List[Dict[str, Any]] = []
- ids_to_delete: List[int] = []
- total_processed = 0
+ total_deleted = 0
total_batches = 0
+ batch: list = []
- for record in queryset:
- # Transform record for MongoDB
- buffer.append(transform_func(record))
- ids_to_delete.append(record["id"])
-
- # Flush batch when it reaches BATCH_SIZE
- if len(buffer) >= BATCH_SIZE:
- total_batches += 1
- flush_to_mongo_and_delete(
- mongo_collection=mongo_collection,
- buffer=buffer,
- ids_to_delete=ids_to_delete,
- model=model,
- mongo_available=mongo_available,
- )
- total_processed += len(buffer)
- buffer.clear()
- ids_to_delete.clear()
-
- # Process final batch if any records remain
- if buffer:
+ def flush(ids: list) -> None:
+ nonlocal total_deleted, total_batches
+ if not ids:
+ return
total_batches += 1
- flush_to_mongo_and_delete(
- mongo_collection=mongo_collection,
- buffer=buffer,
- ids_to_delete=ids_to_delete,
- model=model,
- mongo_available=mongo_available,
- )
- total_processed += len(buffer)
+ try:
+ # `all_objects` is a plain manager, so this is a hard delete β€” rows
+ # are removed from PostgreSQL immediately rather than soft-deleted.
+ delete_result = model.all_objects.filter(id__in=ids).delete()
+ deleted = delete_result[0] if isinstance(delete_result, tuple) else 0
+ total_deleted += deleted
+ except Exception as e:
+ # Log and skip a failed batch rather than aborting the whole run, so
+ # a single bad batch doesn't block cleanup of the remaining rows.
+ log_exception(e)
+
+ for record_id in queryset_func():
+ batch.append(record_id)
+ if len(batch) >= BATCH_SIZE:
+ flush(batch)
+ batch = []
+
+ # Flush the final partial batch
+ flush(batch)
logger.info(
f"{task_name} cleanup task completed",
- extra={
- "total_records_processed": total_processed,
- "total_batches": total_batches,
- "mongo_available": mongo_available,
- "collection_name": collection_name,
- },
+ extra={"total_records_deleted": total_deleted, "total_batches": total_batches},
)
-# Transform functions for each model
-def transform_api_log(record: Dict) -> Dict:
- """Transform API activity log record."""
- return {
- "id": str(record["id"]),
- "created_at": str(record["created_at"]) if record.get("created_at") else None,
- "token_identifier": str(record["token_identifier"]),
- "path": record["path"],
- "method": record["method"],
- "query_params": record.get("query_params"),
- "headers": record.get("headers"),
- "body": record.get("body"),
- "response_code": record["response_code"],
- "response_body": record["response_body"],
- "ip_address": record["ip_address"],
- "user_agent": record["user_agent"],
- "created_by_id": str(record["created_by_id"]),
- }
-
-
-def transform_email_log(record: Dict) -> Dict:
- """Transform email notification log record."""
- return {
- "id": str(record["id"]),
- "created_at": str(record["created_at"]) if record.get("created_at") else None,
- "receiver_id": str(record["receiver_id"]),
- "triggered_by_id": str(record["triggered_by_id"]),
- "entity_identifier": str(record["entity_identifier"]),
- "entity_name": record["entity_name"],
- "data": record["data"],
- "processed_at": (str(record["processed_at"]) if record.get("processed_at") else None),
- "sent_at": str(record["sent_at"]) if record.get("sent_at") else None,
- "entity": record["entity"],
- "old_value": str(record["old_value"]),
- "new_value": str(record["new_value"]),
- "created_by_id": str(record["created_by_id"]),
- }
-
-
-def transform_page_version(record: Dict) -> Dict:
- """Transform page version record."""
- return {
- "id": str(record["id"]),
- "created_at": str(record["created_at"]) if record.get("created_at") else None,
- "page_id": str(record["page_id"]),
- "workspace_id": str(record["workspace_id"]),
- "owned_by_id": str(record["owned_by_id"]),
- "description_html": record["description_html"],
- "description_binary": record["description_binary"],
- "description_stripped": record["description_stripped"],
- "description_json": record["description_json"],
- "sub_pages_data": record["sub_pages_data"],
- "created_by_id": str(record["created_by_id"]),
- "updated_by_id": str(record["updated_by_id"]),
- "deleted_at": str(record["deleted_at"]) if record.get("deleted_at") else None,
- "last_saved_at": (str(record["last_saved_at"]) if record.get("last_saved_at") else None),
- }
-
-
-def transform_issue_description_version(record: Dict) -> Dict:
- """Transform issue description version record."""
- return {
- "id": str(record["id"]),
- "created_at": str(record["created_at"]) if record.get("created_at") else None,
- "issue_id": str(record["issue_id"]),
- "workspace_id": str(record["workspace_id"]),
- "project_id": str(record["project_id"]),
- "created_by_id": str(record["created_by_id"]),
- "updated_by_id": str(record["updated_by_id"]),
- "owned_by_id": str(record["owned_by_id"]),
- "last_saved_at": (str(record["last_saved_at"]) if record.get("last_saved_at") else None),
- "description_binary": record["description_binary"],
- "description_html": record["description_html"],
- "description_stripped": record["description_stripped"],
- "description_json": record["description_json"],
- "deleted_at": str(record["deleted_at"]) if record.get("deleted_at") else None,
- }
-
-
-def transform_webhook_log(record: Dict):
- """Transfer webhook logs to a new destination."""
- return {
- "id": str(record["id"]),
- "created_at": str(record["created_at"]) if record.get("created_at") else None,
- "workspace_id": str(record["workspace_id"]),
- "webhook": str(record["webhook"]),
- # Request
- "event_type": str(record["event_type"]),
- "request_method": str(record["request_method"]),
- "request_headers": str(record["request_headers"]),
- "request_body": str(record["request_body"]),
- # Response
- "response_status": str(record["response_status"]),
- "response_body": str(record["response_body"]),
- "response_headers": str(record["response_headers"]),
- # retry count
- "retry_count": str(record["retry_count"]),
- }
-
-
-# Queryset functions for each cleanup task
+# Queryset functions for each cleanup task β€” each yields primary keys to delete
def get_api_logs_queryset():
- """Get API logs older than cutoff days."""
- cutoff_days = int(os.environ.get("HARD_DELETE_AFTER_DAYS", 30))
- cutoff_time = timezone.now() - timedelta(days=cutoff_days)
+ """Get API activity logs older than the API retention window."""
+ cutoff_time = timezone.now() - timedelta(days=settings.API_ACTIVITY_LOG_RETENTION_DAYS)
logger.info(f"API logs cutoff time: {cutoff_time}")
-
return (
APIActivityLog.all_objects.filter(created_at__lte=cutoff_time)
- .values(
- "id",
- "created_at",
- "token_identifier",
- "path",
- "method",
- "query_params",
- "headers",
- "body",
- "response_code",
- "response_body",
- "ip_address",
- "user_agent",
- "created_by_id",
- )
+ .values_list("id", flat=True)
.iterator(chunk_size=BATCH_SIZE)
)
def get_email_logs_queryset():
- """Get email logs older than cutoff days."""
- cutoff_days = int(os.environ.get("HARD_DELETE_AFTER_DAYS", 30))
- cutoff_time = timezone.now() - timedelta(days=cutoff_days)
+ """Get email logs older than the email retention window."""
+ cutoff_time = timezone.now() - timedelta(days=settings.EMAIL_LOG_RETENTION_DAYS)
logger.info(f"Email logs cutoff time: {cutoff_time}")
-
return (
EmailNotificationLog.all_objects.filter(sent_at__lte=cutoff_time)
- .values(
- "id",
- "created_at",
- "receiver_id",
- "triggered_by_id",
- "entity_identifier",
- "entity_name",
- "data",
- "processed_at",
- "sent_at",
- "entity",
- "old_value",
- "new_value",
- "created_by_id",
- )
+ .values_list("id", flat=True)
.iterator(chunk_size=BATCH_SIZE)
)
@@ -334,22 +120,7 @@ def get_page_versions_queryset():
return (
PageVersion.all_objects.filter(id__in=Subquery(subq))
- .values(
- "id",
- "created_at",
- "page_id",
- "workspace_id",
- "owned_by_id",
- "description_html",
- "description_binary",
- "description_stripped",
- "description_json",
- "sub_pages_data",
- "created_by_id",
- "updated_by_id",
- "deleted_at",
- "last_saved_at",
- )
+ .values_list("id", flat=True)
.iterator(chunk_size=BATCH_SIZE)
)
@@ -370,52 +141,20 @@ def get_issue_description_versions_queryset():
return (
IssueDescriptionVersion.all_objects.filter(id__in=Subquery(subq))
- .values(
- "id",
- "created_at",
- "issue_id",
- "workspace_id",
- "project_id",
- "created_by_id",
- "updated_by_id",
- "owned_by_id",
- "last_saved_at",
- "description_binary",
- "description_html",
- "description_stripped",
- "description_json",
- "deleted_at",
- )
+ .values_list("id", flat=True)
.iterator(chunk_size=BATCH_SIZE)
)
def get_webhook_logs_queryset():
- """Get email logs older than cutoff days."""
- cutoff_days = int(os.environ.get("HARD_DELETE_AFTER_DAYS", 30))
- cutoff_time = timezone.now() - timedelta(days=cutoff_days)
+ """Get webhook logs older than the webhook retention window."""
+ cutoff_time = timezone.now() - timedelta(days=settings.WEBHOOK_LOG_RETENTION_DAYS)
logger.info(f"Webhook logs cutoff time: {cutoff_time}")
-
return (
WebhookLog.all_objects.filter(created_at__lte=cutoff_time)
- .values(
- "id",
- "created_at",
- "workspace_id",
- "webhook",
- "event_type",
- # Request
- "request_method",
- "request_headers",
- "request_body",
- # Response
- "response_status",
- "response_body",
- "response_headers",
- "retry_count",
- )
.order_by("created_at")
- .iterator(chunk_size=100)
+ .values_list("id", flat=True)
+ .iterator(chunk_size=BATCH_SIZE)
)
@@ -424,10 +163,8 @@ def delete_api_logs():
"""Delete old API activity logs."""
process_cleanup_task(
queryset_func=get_api_logs_queryset,
- transform_func=transform_api_log,
model=APIActivityLog,
task_name="API Activity Log",
- collection_name="api_activity_logs",
)
@@ -436,10 +173,8 @@ def delete_email_notification_logs():
"""Delete old email notification logs."""
process_cleanup_task(
queryset_func=get_email_logs_queryset,
- transform_func=transform_email_log,
model=EmailNotificationLog,
task_name="Email Notification Log",
- collection_name="email_notification_logs",
)
@@ -448,10 +183,8 @@ def delete_page_versions():
"""Delete excess page versions."""
process_cleanup_task(
queryset_func=get_page_versions_queryset,
- transform_func=transform_page_version,
model=PageVersion,
task_name="Page Version",
- collection_name="page_versions",
)
@@ -460,20 +193,16 @@ def delete_issue_description_versions():
"""Delete excess issue description versions."""
process_cleanup_task(
queryset_func=get_issue_description_versions_queryset,
- transform_func=transform_issue_description_version,
model=IssueDescriptionVersion,
task_name="Issue Description Version",
- collection_name="issue_description_versions",
)
@shared_task
def delete_webhook_logs():
- """Delete old webhook logs"""
+ """Delete old webhook logs."""
process_cleanup_task(
queryset_func=get_webhook_logs_queryset,
- transform_func=transform_webhook_log,
model=WebhookLog,
task_name="Webhook Log",
- collection_name="webhook_logs",
)
diff --git a/apps/api/plane/bgtasks/logger_task.py b/apps/api/plane/bgtasks/logger_task.py
index 4a74e54bc5..3285e544a3 100644
--- a/apps/api/plane/bgtasks/logger_task.py
+++ b/apps/api/plane/bgtasks/logger_task.py
@@ -4,14 +4,12 @@
# Python imports
import logging
-from typing import Optional, Dict, Any
+from typing import Dict, Any
# Third party imports
-from pymongo.collection import Collection
from celery import shared_task
# Django imports
-from plane.settings.mongo import MongoConnection
from plane.utils.exception_logger import log_exception
from plane.db.models import APIActivityLog
@@ -19,66 +17,9 @@ from plane.db.models import APIActivityLog
logger = logging.getLogger("plane.worker")
-def get_mongo_collection() -> Optional[Collection]:
- """
- Returns the MongoDB collection for external API activity logs.
- """
- if not MongoConnection.is_configured():
- logger.info("MongoDB not configured")
- return None
-
- try:
- return MongoConnection.get_collection("api_activity_logs")
- except Exception as e:
- logger.error(f"Error getting MongoDB collection: {str(e)}")
- log_exception(e)
- return None
-
-
-def safe_decode_body(content: bytes) -> Optional[str]:
- """
- Safely decodes request/response body content, handling binary data.
- Returns "[Binary Content]" if the content is binary, or a string representation of the content.
- Returns None if the content is None or empty.
- """
- # If the content is None, return None
- if content is None:
- return None
-
- # If the content is an empty bytes object, return None
- if content == b"":
- return None
-
- # Check if content is binary by looking for common binary file signatures
- if content.startswith(b"\x89PNG") or content.startswith(b"\xff\xd8\xff") or content.startswith(b"%PDF"):
- return "[Binary Content]"
-
- try:
- return content.decode("utf-8")
- except UnicodeDecodeError:
- return "[Could not decode content]"
-
-
-def log_to_mongo(log_document: Dict[str, Any]) -> bool:
- """
- Logs the request to MongoDB if available.
- """
- mongo_collection = get_mongo_collection()
- if mongo_collection is None:
- logger.error("MongoDB not configured")
- return False
-
- try:
- mongo_collection.insert_one(log_document)
- return True
- except Exception as e:
- log_exception(e)
- return False
-
-
def log_to_postgres(log_data: Dict[str, Any]) -> bool:
"""
- Fallback to logging to PostgreSQL if MongoDB is unavailable.
+ Persist an external API request log to PostgreSQL.
"""
try:
APIActivityLog.objects.create(**log_data)
@@ -89,12 +30,12 @@ def log_to_postgres(log_data: Dict[str, Any]) -> bool:
@shared_task
-def process_logs(log_data: Dict[str, Any], mongo_log: Dict[str, Any]) -> None:
- """
- Process logs to save to MongoDB or Postgres based on the configuration
+def process_logs(log_data: Dict[str, Any], **_: Any) -> None:
"""
+ Persist external API request logs to PostgreSQL.
- if MongoConnection.is_configured():
- log_to_mongo(mongo_log)
- else:
- log_to_postgres(log_data)
+ The catch-all kwargs keep this task signature compatible with jobs enqueued
+ by an older release (which passed a `mongo_log` argument), so in-flight tasks
+ don't fail during a rolling deploy. It can be dropped once no such jobs remain.
+ """
+ log_to_postgres(log_data)
diff --git a/apps/api/plane/bgtasks/webhook_task.py b/apps/api/plane/bgtasks/webhook_task.py
index ddb5299fb5..02e6915d4e 100644
--- a/apps/api/plane/bgtasks/webhook_task.py
+++ b/apps/api/plane/bgtasks/webhook_task.py
@@ -53,7 +53,6 @@ from plane.license.utils.instance_value import get_email_configuration
from plane.utils.email import generate_plain_text_from_html
from plane.utils.exception_logger import log_exception
from plane.utils.ip_address import validate_url
-from plane.settings.mongo import MongoConnection
SERIALIZER_MAPPER = {
@@ -102,9 +101,6 @@ def save_webhook_log(
retry_count: int,
event_type: str,
) -> None:
- # webhook_logs
- mongo_collection = MongoConnection.get_collection("webhook_logs")
-
log_data = {
"workspace_id": str(webhook.workspace_id),
"webhook": str(webhook.id),
@@ -118,27 +114,12 @@ def save_webhook_log(
"retry_count": retry_count,
}
- mongo_save_success = False
- if mongo_collection is not None:
- try:
- # insert the log data into the mongo collection
- mongo_collection.insert_one(log_data)
- logger.info("Webhook log saved successfully to mongo")
- mongo_save_success = True
- except Exception as e:
- log_exception(e, warning=True)
- logger.error(f"Failed to save webhook log: {e}")
- mongo_save_success = False
-
- # if the mongo save is not successful, save the log data into the database
- if not mongo_save_success:
- try:
- # insert the log data into the database
- WebhookLog.objects.create(**log_data)
- logger.info("Webhook log saved successfully to database")
- except Exception as e:
- log_exception(e, warning=True)
- logger.error(f"Failed to save webhook log: {e}")
+ try:
+ WebhookLog.objects.create(**log_data)
+ logger.info("Webhook log saved successfully to database")
+ except Exception as e:
+ log_exception(e, warning=True)
+ logger.error(f"Failed to save webhook log: {e}")
def get_model_data(event: str, event_id: Union[str, List[str]], many: bool = False) -> Dict[str, Any]:
diff --git a/apps/api/plane/middleware/logger.py b/apps/api/plane/middleware/logger.py
index b8cf6f9c04..343409c4e1 100644
--- a/apps/api/plane/middleware/logger.py
+++ b/apps/api/plane/middleware/logger.py
@@ -3,12 +3,14 @@
# See the LICENSE file for details.
# Python imports
+import hashlib
+import hmac
import logging
import time
# Django imports
+from django.conf import settings
from django.http import HttpRequest
-from django.utils import timezone
# Third party imports
from rest_framework.request import Request
@@ -77,7 +79,7 @@ class RequestLoggerMiddleware:
class APITokenLogMiddleware:
"""
- Middleware to log External API requests to MongoDB or PostgreSQL.
+ Middleware to log External API requests to PostgreSQL.
"""
def __init__(self, get_response):
@@ -111,6 +113,20 @@ class APITokenLogMiddleware:
except UnicodeDecodeError:
return "[Could not decode content]"
+ # Headers whose values must never be persisted in plaintext logs
+ SENSITIVE_HEADERS = frozenset({"x-api-key", "authorization", "cookie"})
+
+ def _redacted_headers(self, request):
+ """
+ Returns the request headers as a string with sensitive values redacted,
+ so that credentials such as the API key are never stored in plaintext.
+ """
+ redacted = {
+ key: ("[REDACTED]" if key.lower() in self.SENSITIVE_HEADERS else value)
+ for key, value in request.headers.items()
+ }
+ return str(redacted)
+
def process_request(self, request, response, request_body):
api_key_header = "X-Api-Key"
api_key = request.headers.get(api_key_header)
@@ -121,32 +137,25 @@ class APITokenLogMiddleware:
try:
log_data = {
- "token_identifier": api_key,
+ # Tokenize the (high-entropy) API key into a stable, non-reversible
+ # identifier so logs can be correlated to a token without ever
+ # persisting the raw key. A keyed HMAC is used rather than a bare
+ # hash so the digest cannot be precomputed from a known key value.
+ "token_identifier": hmac.new(
+ settings.SECRET_KEY.encode(), api_key.encode(), hashlib.sha256
+ ).hexdigest(),
"path": request.path,
"method": request.method,
"query_params": request.META.get("QUERY_STRING", ""),
- "headers": str(request.headers),
+ "headers": self._redacted_headers(request),
"body": self._safe_decode_body(request_body) if request_body else None,
"response_body": self._safe_decode_body(response.content) if response.content else None,
"response_code": response.status_code,
"ip_address": get_client_ip(request=request),
"user_agent": request.META.get("HTTP_USER_AGENT", None),
}
- user_id = (
- str(request.user.id)
- if getattr(request, "user") and getattr(request.user, "is_authenticated", False)
- else None
- )
- # Additional fields for MongoDB
- mongo_log = {
- **log_data,
- "created_at": timezone.now(),
- "updated_at": timezone.now(),
- "created_by": user_id,
- "updated_by": user_id,
- }
- process_logs.delay(log_data=log_data, mongo_log=mongo_log)
+ process_logs.delay(log_data=log_data)
except Exception as e:
log_exception(e)
diff --git a/apps/api/plane/settings/common.py b/apps/api/plane/settings/common.py
index 9a6ec57c93..59140c070d 100644
--- a/apps/api/plane/settings/common.py
+++ b/apps/api/plane/settings/common.py
@@ -402,6 +402,34 @@ WEB_URL = os.environ.get("WEB_URL")
HARD_DELETE_AFTER_DAYS = int(os.environ.get("HARD_DELETE_AFTER_DAYS", 60))
+
+def _retention_days(env_var, default):
+ """
+ Read a retention window (in days) from the environment, falling back to the
+ default when the variable is unset, unparseable, or negative β€” a negative
+ window would otherwise select rows with a future cutoff and delete everything.
+ """
+ raw = os.environ.get(env_var)
+ if raw is None:
+ return default
+ try:
+ days = int(raw)
+ except ValueError:
+ return default
+ return days if days >= 0 else default
+
+
+# API activity logs hold request/response payloads, so they are retained for a
+# shorter window than other logs.
+API_ACTIVITY_LOG_RETENTION_DAYS = _retention_days("API_ACTIVITY_LOG_RETENTION_DAYS", 14)
+
+# Webhook delivery logs are retained on their own window, independent of the
+# generic HARD_DELETE_AFTER_DAYS.
+WEBHOOK_LOG_RETENTION_DAYS = _retention_days("WEBHOOK_LOG_RETENTION_DAYS", 14)
+
+# Email notification logs are retained on their own window.
+EMAIL_LOG_RETENTION_DAYS = _retention_days("EMAIL_LOG_RETENTION_DAYS", 7)
+
# Instance Changelog URL
INSTANCE_CHANGELOG_URL = os.environ.get("INSTANCE_CHANGELOG_URL", "")
@@ -504,7 +532,3 @@ if ENABLE_DRF_SPECTACULAR:
REST_FRAMEWORK["DEFAULT_SCHEMA_CLASS"] = "drf_spectacular.openapi.AutoSchema"
INSTALLED_APPS.append("drf_spectacular")
from .openapi import SPECTACULAR_SETTINGS # noqa: F401
-
-# MongoDB Settings
-MONGO_DB_URL = os.environ.get("MONGO_DB_URL", False)
-MONGO_DB_DATABASE = os.environ.get("MONGO_DB_DATABASE", False)
diff --git a/apps/api/plane/settings/local.py b/apps/api/plane/settings/local.py
index e71f40bf14..a346eca141 100644
--- a/apps/api/plane/settings/local.py
+++ b/apps/api/plane/settings/local.py
@@ -75,11 +75,6 @@ LOGGING = {
"handlers": ["console"],
"propagate": False,
},
- "plane.mongo": {
- "level": "INFO",
- "handlers": ["console"],
- "propagate": False,
- },
"plane.authentication": {
"level": "INFO",
"handlers": ["console"],
diff --git a/apps/api/plane/settings/mongo.py b/apps/api/plane/settings/mongo.py
deleted file mode 100644
index 7855a52d51..0000000000
--- a/apps/api/plane/settings/mongo.py
+++ /dev/null
@@ -1,126 +0,0 @@
-# Copyright (c) 2023-present Plane Software, Inc. and contributors
-# SPDX-License-Identifier: AGPL-3.0-only
-# See the LICENSE file for details.
-
-# Django imports
-from django.conf import settings
-import logging
-
-# Third party imports
-from pymongo import MongoClient
-from pymongo.database import Database
-from pymongo.collection import Collection
-from typing import Optional, TypeVar, Type
-
-
-T = TypeVar("T", bound="MongoConnection")
-
-# Set up logger
-logger = logging.getLogger("plane.mongo")
-
-
-class MongoConnection:
- """
- A singleton class that manages MongoDB connections.
-
- This class ensures only one MongoDB connection is maintained throughout the application.
- It provides methods to access the MongoDB client, database, and collections.
-
- Attributes:
- _instance (Optional[MongoConnection]): The singleton instance of this class
- _client (Optional[MongoClient]): The MongoDB client instance
- _db (Optional[Database]): The MongoDB database instance
- """
-
- _instance: Optional["MongoConnection"] = None
- _client: Optional[MongoClient] = None
- _db: Optional[Database] = None
-
- def __new__(cls: Type[T]) -> T:
- """
- Creates a new instance of MongoConnection if one doesn't exist.
-
- Returns:
- MongoConnection: The singleton instance
- """
- if cls._instance is None:
- cls._instance = super(MongoConnection, cls).__new__(cls)
- try:
- mongo_url = getattr(settings, "MONGO_DB_URL", None)
- mongo_db_database = getattr(settings, "MONGO_DB_DATABASE", None)
-
- if not mongo_url or not mongo_db_database:
- logger.warning(
- "MongoDB connection parameters not configured. MongoDB functionality will be disabled."
- )
- return cls._instance
-
- cls._client = MongoClient(mongo_url)
- cls._db = cls._client[mongo_db_database]
-
- # Test the connection
- cls._client.server_info()
- logger.info("MongoDB connection established successfully")
- except Exception as e:
- logger.warning(
- f"Failed to initialize MongoDB connection: {str(e)}. MongoDB functionality will be disabled."
- )
- return cls._instance
-
- @classmethod
- def get_client(cls) -> Optional[MongoClient]:
- """
- Returns the MongoDB client instance.
-
- Returns:
- Optional[MongoClient]: The MongoDB client instance or None if not configured
- """
- if cls._client is None:
- cls._instance = cls()
- return cls._client
-
- @classmethod
- def get_db(cls) -> Optional[Database]:
- """
- Returns the MongoDB database instance.
-
- Returns:
- Optional[Database]: The MongoDB database instance or None if not configured
- """
- if cls._db is None:
- cls._instance = cls()
- return cls._db
-
- @classmethod
- def get_collection(cls, collection_name: str) -> Optional[Collection]:
- """
- Returns a MongoDB collection by name.
-
- Args:
- collection_name (str): The name of the collection to retrieve
-
- Returns:
- Optional[Collection]: The MongoDB collection instance or None if not configured
- """
- try:
- db = cls.get_db()
- if db is None:
- logger.warning(f"Cannot access collection '{collection_name}': MongoDB not configured")
- return None
- return db[collection_name]
- except Exception as e:
- logger.warning(f"Failed to access collection '{collection_name}': {str(e)}")
- return None
-
- @classmethod
- def is_configured(cls) -> bool:
- """
- Check if MongoDB is properly configured and connected.
-
- Returns:
- bool: True if MongoDB is configured and connected, False otherwise
- """
-
- if cls._client is None:
- cls._instance = cls()
- return cls._client is not None and cls._db is not None
diff --git a/apps/api/plane/settings/production.py b/apps/api/plane/settings/production.py
index f58450a9da..1dbf43196a 100644
--- a/apps/api/plane/settings/production.py
+++ b/apps/api/plane/settings/production.py
@@ -85,11 +85,6 @@ LOGGING = {
"handlers": ["console"],
"propagate": False,
},
- "plane.mongo": {
- "level": "INFO",
- "handlers": ["console"],
- "propagate": False,
- },
"plane.authentication": {
"level": "DEBUG" if DEBUG else "INFO",
"handlers": ["console"],
diff --git a/apps/api/plane/tests/README.md b/apps/api/plane/tests/README.md
index df9aba6da1..3d6936d785 100644
--- a/apps/api/plane/tests/README.md
+++ b/apps/api/plane/tests/README.md
@@ -91,7 +91,7 @@ When writing tests, follow these guidelines:
- For web app API (`/api/`), use `session_client`
- For smoke tests with real HTTP, use `plane_server`
3. Use the correct URL namespace when reverse-resolving URLs:
- - For external API, use `reverse("api:endpoint_name")`
+ - For external API, use `reverse("api:endpoint_name")`
- For web app API, use `reverse("endpoint_name")`
4. Add the `@pytest.mark.django_db` decorator to tests that interact with the database.
5. Add the appropriate markers (`@pytest.mark.contract`, etc.) to categorize tests.
@@ -101,7 +101,7 @@ When writing tests, follow these guidelines:
Common fixtures are defined in:
- `conftest.py`: General fixtures for authentication, database access, etc.
-- `conftest_external.py`: Fixtures for external services (Redis, Elasticsearch, Celery, MongoDB)
+- `conftest_external.py`: Fixtures for external services (Redis, Elasticsearch, Celery)
- `factories.py`: Test factories for easy model instance creation
## Best Practices
@@ -125,7 +125,7 @@ When writing tests, follow these guidelines:
Tests for components that interact with external services should:
-1. Use the `mock_redis`, `mock_elasticsearch`, `mock_mongodb`, and `mock_celery` fixtures for unit and most contract tests.
+1. Use the `mock_redis`, `mock_elasticsearch`, and `mock_celery` fixtures for unit and most contract tests.
2. For more comprehensive contract tests, use Docker-based test containers (optional).
## Coverage Reports
@@ -140,4 +140,4 @@ This creates an HTML report in the `htmlcov/` directory.
## Migration from Old Tests
-Some tests are still in the old format in the `api/` directory. These need to be migrated to the new contract test structure in the appropriate directories.
\ No newline at end of file
+Some tests are still in the old format in the `api/` directory. These need to be migrated to the new contract test structure in the appropriate directories.
diff --git a/apps/api/plane/tests/conftest_external.py b/apps/api/plane/tests/conftest_external.py
index cd5469caa6..16bc31a430 100644
--- a/apps/api/plane/tests/conftest_external.py
+++ b/apps/api/plane/tests/conftest_external.py
@@ -51,41 +51,6 @@ def mock_elasticsearch():
yield mock_es_client
-def mock_mongodb():
- """
- Mock MongoDB for testing without actual MongoDB connection.
-
- This fixture patches PyMongo to return a MagicMock that behaves like a MongoDB client.
- """
- # Create mock MongoDB clients and collections
- mock_mongo_client = MagicMock()
- mock_mongo_db = MagicMock()
- mock_mongo_collection = MagicMock()
-
- # Set up the chain: client -> database -> collection
- mock_mongo_client.__getitem__.return_value = mock_mongo_db
- mock_mongo_client.get_database.return_value = mock_mongo_db
- mock_mongo_db.__getitem__.return_value = mock_mongo_collection
-
- # Configure common MongoDB collection operations
- mock_mongo_collection.find_one.return_value = None
- mock_mongo_collection.find.return_value = MagicMock(__iter__=lambda x: iter([]), count=lambda: 0)
- mock_mongo_collection.insert_one.return_value = MagicMock(inserted_id="mock_id_123", acknowledged=True)
- mock_mongo_collection.insert_many.return_value = MagicMock(
- inserted_ids=["mock_id_123", "mock_id_456"], acknowledged=True
- )
- mock_mongo_collection.update_one.return_value = MagicMock(modified_count=1, matched_count=1, acknowledged=True)
- mock_mongo_collection.update_many.return_value = MagicMock(modified_count=2, matched_count=2, acknowledged=True)
- mock_mongo_collection.delete_one.return_value = MagicMock(deleted_count=1, acknowledged=True)
- mock_mongo_collection.delete_many.return_value = MagicMock(deleted_count=2, acknowledged=True)
- mock_mongo_collection.count_documents.return_value = 0
-
- # Start the patch
- with patch("pymongo.MongoClient", return_value=mock_mongo_client):
- yield mock_mongo_client
-
-
@pytest.fixture
def mock_celery():
"""
diff --git a/apps/api/plane/tests/contract/app/test_api_token.py b/apps/api/plane/tests/contract/app/test_api_token.py
index ed071b98cb..60ebd8fde8 100644
--- a/apps/api/plane/tests/contract/app/test_api_token.py
+++ b/apps/api/plane/tests/contract/app/test_api_token.py
@@ -366,6 +366,23 @@ class TestApiTokenEndpoint:
create_api_token_for_user.refresh_from_db()
assert create_api_token_for_user.user_type == 0
+ @pytest.mark.django_db
+ def test_patch_cannot_modify_allowed_rate_limit(self, session_client, create_user, create_api_token_for_user):
+ """Test that allowed_rate_limit cannot be modified via PATCH"""
+ # Arrange
+ session_client.force_authenticate(user=create_user)
+ url = reverse("api-tokens-details", kwargs={"pk": create_api_token_for_user.pk})
+ original_rate_limit = create_api_token_for_user.allowed_rate_limit
+ update_data = {"allowed_rate_limit": "100000/min"}
+
+ # Act
+ response = session_client.patch(url, update_data, format="json")
+
+ # Assert
+ assert response.status_code == status.HTTP_200_OK
+ create_api_token_for_user.refresh_from_db()
+ assert create_api_token_for_user.allowed_rate_limit == original_rate_limit
+
@pytest.mark.django_db
def test_patch_cannot_modify_service_token(self, session_client, create_user):
"""Test that service tokens cannot be modified through user token endpoint"""
diff --git a/apps/api/plane/tests/unit/bg_tasks/test_cleanup_task.py b/apps/api/plane/tests/unit/bg_tasks/test_cleanup_task.py
new file mode 100644
index 0000000000..d0a85a19bc
--- /dev/null
+++ b/apps/api/plane/tests/unit/bg_tasks/test_cleanup_task.py
@@ -0,0 +1,143 @@
+# Copyright (c) 2023-present Plane Software, Inc. and contributors
+# SPDX-License-Identifier: AGPL-3.0-only
+# See the LICENSE file for details.
+
+"""
+Unit tests for the log cleanup tasks.
+
+Verifies that API activity logs past the retention window are hard-deleted
+(removed from PostgreSQL, not soft-deleted) and that fresh logs are retained.
+"""
+
+from datetime import timedelta
+
+import pytest
+from django.conf import settings
+from django.utils import timezone
+
+from uuid import uuid4
+
+from plane.bgtasks.cleanup_task import (
+ delete_api_logs,
+ delete_email_notification_logs,
+ delete_webhook_logs,
+ process_cleanup_task,
+)
+from plane.db.models import APIActivityLog, EmailNotificationLog, WebhookLog
+from plane.tests.factories import UserFactory, WorkspaceFactory
+
+
+def _make_api_log(created_at):
+ log = APIActivityLog.objects.create(
+ token_identifier="hashed-token",
+ path="/api/v1/workspaces/",
+ method="GET",
+ response_code=200,
+ )
+ # created_at is auto-set on insert, so backdate it explicitly afterwards.
+ APIActivityLog.all_objects.filter(pk=log.pk).update(created_at=created_at)
+ return log
+
+
+def _make_webhook_log(workspace, created_at):
+ log = WebhookLog.objects.create(
+ workspace=workspace,
+ webhook=uuid4(),
+ event_type="issue",
+ request_method="POST",
+ response_status="200",
+ )
+ WebhookLog.all_objects.filter(pk=log.pk).update(created_at=created_at)
+ return log
+
+
+def _make_email_log(user, sent_at):
+ return EmailNotificationLog.objects.create(
+ receiver=user,
+ triggered_by=user,
+ entity_name="issue",
+ entity="issue",
+ sent_at=sent_at,
+ )
+
+
+class TestDeleteApiLogs:
+ def test_expired_logs_are_hard_deleted(self):
+ retention_days = settings.API_ACTIVITY_LOG_RETENTION_DAYS
+ expired = _make_api_log(timezone.now() - timedelta(days=retention_days + 1))
+
+ delete_api_logs()
+
+ # Hard delete: the row must be gone even from the unfiltered manager.
+ assert not APIActivityLog.all_objects.filter(pk=expired.pk).exists()
+
+ def test_recent_logs_are_retained(self):
+ retention_days = settings.API_ACTIVITY_LOG_RETENTION_DAYS
+ recent = _make_api_log(timezone.now() - timedelta(days=retention_days - 1))
+
+ delete_api_logs()
+
+ assert APIActivityLog.all_objects.filter(pk=recent.pk).exists()
+
+
+class TestDeleteWebhookLogs:
+ def test_expired_logs_are_hard_deleted(self):
+ workspace = WorkspaceFactory()
+ retention_days = settings.WEBHOOK_LOG_RETENTION_DAYS
+ expired = _make_webhook_log(workspace, timezone.now() - timedelta(days=retention_days + 1))
+
+ delete_webhook_logs()
+
+ assert not WebhookLog.all_objects.filter(pk=expired.pk).exists()
+
+ def test_recent_logs_are_retained(self):
+ workspace = WorkspaceFactory()
+ retention_days = settings.WEBHOOK_LOG_RETENTION_DAYS
+ recent = _make_webhook_log(workspace, timezone.now() - timedelta(days=retention_days - 1))
+
+ delete_webhook_logs()
+
+ assert WebhookLog.all_objects.filter(pk=recent.pk).exists()
+
+
+class TestDeleteEmailLogs:
+ def test_expired_logs_are_hard_deleted(self):
+ user = UserFactory()
+ retention_days = settings.EMAIL_LOG_RETENTION_DAYS
+ expired = _make_email_log(user, timezone.now() - timedelta(days=retention_days + 1))
+
+ delete_email_notification_logs()
+
+ assert not EmailNotificationLog.all_objects.filter(pk=expired.pk).exists()
+
+ def test_recent_logs_are_retained(self):
+ user = UserFactory()
+ retention_days = settings.EMAIL_LOG_RETENTION_DAYS
+ recent = _make_email_log(user, timezone.now() - timedelta(days=retention_days - 1))
+
+ delete_email_notification_logs()
+
+ assert EmailNotificationLog.all_objects.filter(pk=recent.pk).exists()
+
+
+class TestProcessCleanupTaskErrorHandling:
+ def test_batch_delete_failure_is_swallowed(self):
+ """A failing batch is logged and skipped; the run does not raise."""
+
+ class _BoomManager:
+ @staticmethod
+ def filter(**kwargs):
+ raise RuntimeError("db unavailable")
+
+ class _BoomModel:
+ all_objects = _BoomManager()
+
+ # Should not raise even though the delete blows up.
+ process_cleanup_task(lambda: iter([1, 2, 3]), _BoomModel, "Boom")
diff --git a/apps/api/plane/tests/unit/middleware/test_logger.py b/apps/api/plane/tests/unit/middleware/test_logger.py
new file mode 100644
index 0000000000..5c13f53f6f
--- /dev/null
+++ b/apps/api/plane/tests/unit/middleware/test_logger.py
@@ -0,0 +1,79 @@
+# Copyright (c) 2023-present Plane Software, Inc. and contributors
+# SPDX-License-Identifier: AGPL-3.0-only
+# See the LICENSE file for details.
+
+"""
+Unit tests for APITokenLogMiddleware.
+
+Covers the credential-hygiene guarantees of the external API request logger:
+- the raw API key is never persisted (a non-reversible hash is stored instead)
+- sensitive request headers are redacted before being logged
+"""
+
+import hashlib
+import hmac
+from unittest.mock import Mock, patch
+
+import pytest
+from django.conf import settings
+from django.contrib.auth.models import AnonymousUser
+from django.http import HttpResponse
+from django.test import RequestFactory
+
+from plane.middleware.logger import APITokenLogMiddleware
+
+
+def request_factory():
+ return RequestFactory()
+
+
+def middleware():
+ return APITokenLogMiddleware(Mock(return_value=HttpResponse(b"{}")))
+
+
+class TestAPITokenLogMiddleware:
+ API_KEY = "plane_api_supersecretvalue"
+ AUTHORIZATION = "Bearer secret-bearer-token"
+ COOKIE = "sessionid=secret-session-value"
+
+ def _captured_log_data(self, middleware, request_factory):
+ request = request_factory.get(
+ "/api/v1/workspaces/",
+ HTTP_X_API_KEY=self.API_KEY,
+ HTTP_AUTHORIZATION=self.AUTHORIZATION,
+ HTTP_COOKIE=self.COOKIE,
+ )
+ request.user = AnonymousUser()
+ response = HttpResponse(b"{}")
+ with patch("plane.middleware.logger.process_logs") as process_logs:
+ middleware.process_request(request, response, request_body=b"")
+ assert process_logs.delay.called
+ return process_logs.delay.call_args.kwargs["log_data"]
+
+ def test_token_identifier_is_hashed_not_plaintext(self, middleware, request_factory):
+ log_data = self._captured_log_data(middleware, request_factory)
+
+ expected_hash = hmac.new(
+ settings.SECRET_KEY.encode(), self.API_KEY.encode(), hashlib.sha256
+ ).hexdigest()
+ assert log_data["token_identifier"] == expected_hash
+ assert self.API_KEY not in log_data["token_identifier"]
+
+ def test_sensitive_headers_are_redacted(self, middleware, request_factory):
+ log_data = self._captured_log_data(middleware, request_factory)
+
+ # None of the sensitive header values may appear in the logged headers.
+ assert self.API_KEY not in log_data["headers"]
+ assert self.AUTHORIZATION not in log_data["headers"]
+ assert self.COOKIE not in log_data["headers"]
+ assert "[REDACTED]" in log_data["headers"]
+
+ def test_no_log_without_api_key(self, middleware, request_factory):
+ request = request_factory.get("/api/v1/workspaces/")
+ request.user = AnonymousUser()
+ with patch("plane.middleware.logger.process_logs") as process_logs:
+ middleware.process_request(request, HttpResponse(b"{}"), request_body=b"")
+ assert not process_logs.delay.called
diff --git a/apps/api/plane/tests/unit/settings/test_retention.py b/apps/api/plane/tests/unit/settings/test_retention.py
new file mode 100644
index 0000000000..10bc79192d
--- /dev/null
+++ b/apps/api/plane/tests/unit/settings/test_retention.py
@@ -0,0 +1,34 @@
+# Copyright (c) 2023-present Plane Software, Inc. and contributors
+# SPDX-License-Identifier: AGPL-3.0-only
+# See the LICENSE file for details.
+
+"""Unit tests for the log-retention env parsing helper."""
+
+import pytest
+
+from plane.settings.common import _retention_days
+
+ENV_VAR = "TEST_RETENTION_DAYS"
+
+
+class TestRetentionDays:
+ def test_uses_default_when_unset(self, monkeypatch):
+ monkeypatch.delenv(ENV_VAR, raising=False)
+ assert _retention_days(ENV_VAR, 14) == 14
+
+ def test_uses_env_value_when_valid(self, monkeypatch):
+ monkeypatch.setenv(ENV_VAR, "30")
+ assert _retention_days(ENV_VAR, 14) == 30
+
+ def test_zero_is_allowed(self, monkeypatch):
+ monkeypatch.setenv(ENV_VAR, "0")
+ assert _retention_days(ENV_VAR, 14) == 0
+
+ def test_negative_falls_back_to_default(self, monkeypatch):
+ monkeypatch.setenv(ENV_VAR, "-5")
+ assert _retention_days(ENV_VAR, 14) == 14
+
+ def test_unparseable_falls_back_to_default(self, monkeypatch):
+ monkeypatch.setenv(ENV_VAR, "abc")
+ assert _retention_days(ENV_VAR, 7) == 7
diff --git a/apps/api/requirements/base.txt b/apps/api/requirements/base.txt
index a34bf6de11..68c4a4a43c 100644
--- a/apps/api/requirements/base.txt
+++ b/apps/api/requirements/base.txt
@@ -9,8 +9,6 @@ psycopg==3.3.0
psycopg-binary==3.3.0
psycopg-c==3.3.0
dj-database-url==2.1.0
-# mongo
-pymongo==4.6.3
# redis
redis==5.0.4
django-redis==5.4.0