π₯π₯π₯ Open-source Jira, Linear, Monday, and ClickUp alternative. Plane is a modern project management platform to manage tasks, sprints, docs, and triage.
TypeScript boardsbug-trackerdjangodockerganttissue-trackerjirajira-alternativekanbanlinearpostgresqlproduct-managementproject-managementproject-planningpythonreactredistypesciptvitework-management
refactor: logging with retention + API token hardening (#9148)
sriram veeraghantaedf247541301
diff --git a/.gitignore b/.gitignoreindex 6f329c7393..61351715a1 100644--- a/.gitignore+++ b/.gitignore@@ -114,3 +114,6 @@ scripts/# i18n auto-generated types (regenerated on every build)packages/i18n/src/types/keys.generated.ts++# Local security advisory notes (not for version control)+/advisories.mddiff --git a/apps/api/plane/app/serializers/api.py b/apps/api/plane/app/serializers/api.pyindex 05c6198f59..71ffd442bf 100644--- a/apps/api/plane/app/serializers/api.py+++ b/apps/api/plane/app/serializers/api.py@@ -22,6 +22,7 @@ class APITokenSerializer(BaseSerializer):"is_active","last_used","user_type",+ "allowed_rate_limit",]diff --git a/apps/api/plane/bgtasks/cleanup_task.py b/apps/api/plane/bgtasks/cleanup_task.pyindex 407a67ca69..3414438ccf 100644--- a/apps/api/plane/bgtasks/cleanup_task.py+++ b/apps/api/plane/bgtasks/cleanup_task.py@@ -5,19 +5,16 @@# Python importsfrom datetime import timedeltaimport logging-from typing import List, Dict, Any, Callable, Optional-import os+from typing import Callable, Iterable# Django imports+from django.conf import settingsfrom django.utils import timezonefrom django.db.models import F, Window, Subqueryfrom django.db.models.functions import RowNumber# Third party importsfrom celery import shared_task-from pymongo.errors import BulkWriteError-from pymongo.collection import Collection-from pymongo.operations import InsertOne# Module importsfrom plane.db.models import (@@ -27,7 +24,6 @@ from plane.db.models import (IssueDescriptionVersion,WebhookLog,)-from plane.settings.mongo import MongoConnectionfrom plane.utils.exception_logger import log_exception@@ -35,285 +31,75 @@ logger = logging.getLogger("plane.worker")BATCH_SIZE = 500-def get_mongo_collection(collection_name: str) -> Optional[Collection]:- """Get MongoDB collection if available, otherwise return None."""- if not MongoConnection.is_configured():- logger.info("MongoDB not configured")- return None-- try:- mongo_collection = MongoConnection.get_collection(collection_name)- logger.info(f"MongoDB collection '{collection_name}' connected successfully")- return mongo_collection- except Exception as e:- logger.error(f"Failed to get MongoDB collection: {str(e)}")- log_exception(e)- return None---def flush_to_mongo_and_delete(- mongo_collection: Optional[Collection],- buffer: List[Dict[str, Any]],- ids_to_delete: List[int],- model,- mongo_available: bool,-) -> None:- """- Inserts a batch of records into MongoDB and deletes the corresponding rows from PostgreSQL.- """- if not buffer:- logger.debug("No records to flush - buffer is empty")- return-- logger.info(f"Starting batch flush: {len(buffer)} records, {len(ids_to_delete)} IDs to delete")-- mongo_archival_failed = False-- # Try to insert into MongoDB if available- if mongo_collection is not None and mongo_available:- try:- mongo_collection.bulk_write([InsertOne(doc) for doc in buffer])- except BulkWriteError as bwe:- logger.error(f"MongoDB bulk write error: {str(bwe)}")- log_exception(bwe)- mongo_archival_failed = True-- # If MongoDB is available and archival failed, log the error and return- if mongo_available and mongo_archival_failed:- logger.error(f"MongoDB archival failed for {len(buffer)} records")- return-- # Delete from PostgreSQL - delete() returns (count, {model: count})- delete_result = model.all_objects.filter(id__in=ids_to_delete).delete()- deleted_count = delete_result[0] if delete_result and isinstance(delete_result, tuple) else 0- logger.info(f"Batch flush completed: {deleted_count} records deleted")--def process_cleanup_task(- queryset_func: Callable,- transform_func: Callable[[Dict], Dict],+ queryset_func: Callable[[], Iterable],model,task_name: str,- collection_name: str,):"""- Generic function to process cleanup tasks.+ Batch-delete expired rows for the given model from PostgreSQL.Args:- queryset_func: Function that returns the queryset to process- transform_func: Function to transform each record for MongoDB- model: Django model class- task_name: Name of the task for logging- collection_name: MongoDB collection name+ queryset_func: Callable returning an iterable of primary keys to delete.+ model: Django model class.+ task_name: Name of the task for logging."""logger.info(f"Starting {task_name} cleanup task")- # Get MongoDB collection- mongo_collection = get_mongo_collection(collection_name)- mongo_available = mongo_collection is not None-- # Get queryset- queryset = queryset_func()-- # Process records in batches- buffer: List[Dict[str, Any]] = []- ids_to_delete: List[int] = []- total_processed = 0+ total_deleted = 0total_batches = 0+ batch: list = []- for record in queryset:- # Transform record for MongoDB- buffer.append(transform_func(record))- ids_to_delete.append(record["id"])-- # Flush batch when it reaches BATCH_SIZE- if len(buffer) >= BATCH_SIZE:- total_batches += 1- flush_to_mongo_and_delete(- mongo_collection=mongo_collection,- buffer=buffer,- ids_to_delete=ids_to_delete,- model=model,- mongo_available=mongo_available,- )- total_processed += len(buffer)- buffer.clear()- ids_to_delete.clear()-- # Process final batch if any records remain- if buffer:+ def flush(ids: list) -> None:+ nonlocal total_deleted, total_batches+ if not ids:+ returntotal_batches += 1- flush_to_mongo_and_delete(- mongo_collection=mongo_collection,- buffer=buffer,- ids_to_delete=ids_to_delete,- model=model,- mongo_available=mongo_available,- )- total_processed += len(buffer)+ try:+ # `all_objects` is a plain manager, so this is a hard delete β rows+ # are removed from PostgreSQL immediately rather than soft-deleted.+ delete_result = model.all_objects.filter(id__in=ids).delete()+ deleted = delete_result[0] if isinstance(delete_result, tuple) else 0+ total_deleted += deleted+ except Exception as e:+ # Log and skip a failed batch rather than aborting the whole run, so+ # a single bad batch doesn't block cleanup of the remaining rows.+ log_exception(e)++ for record_id in queryset_func():+ batch.append(record_id)+ if len(batch) >= BATCH_SIZE:+ flush(batch)+ batch = []++ # Flush the final partial batch+ flush(batch)logger.info(f"{task_name} cleanup task completed",- extra={- "total_records_processed": total_processed,- "total_batches": total_batches,- "mongo_available": mongo_available,- "collection_name": collection_name,- },+ extra={"total_records_deleted": total_deleted, "total_batches": total_batches},)-# Transform functions for each model-def transform_api_log(record: Dict) -> Dict:- """Transform API activity log record."""- return {- "id": str(record["id"]),- "created_at": str(record["created_at"]) if record.get("created_at") else None,- "token_identifier": str(record["token_identifier"]),- "path": record["path"],- "method": record["method"],- "query_params": record.get("query_params"),- "headers": record.get("headers"),- "body": record.get("body"),- "response_code": record["response_code"],- "response_body": record["response_body"],- "ip_address": record["ip_address"],- "user_agent": record["user_agent"],- "created_by_id": str(record["created_by_id"]),- }---def transform_email_log(record: Dict) -> Dict:- """Transform email notification log record."""- return {- "id": str(record["id"]),- "created_at": str(record["created_at"]) if record.get("created_at") else None,- "receiver_id": str(record["receiver_id"]),- "triggered_by_id": str(record["triggered_by_id"]),- "entity_identifier": str(record["entity_identifier"]),- "entity_name": record["entity_name"],- "data": record["data"],- "processed_at": (str(record["processed_at"]) if record.get("processed_at") else None),- "sent_at": str(record["sent_at"]) if record.get("sent_at") else None,- "entity": record["entity"],- "old_value": str(record["old_value"]),- "new_value": str(record["new_value"]),- "created_by_id": str(record["created_by_id"]),- }---def transform_page_version(record: Dict) -> Dict:- """Transform page version record."""- return {- "id": str(record["id"]),- "created_at": str(record["created_at"]) if record.get("created_at") else None,- "page_id": str(record["page_id"]),- "workspace_id": str(record["workspace_id"]),- "owned_by_id": str(record["owned_by_id"]),- "description_html": record["description_html"],- "description_binary": record["description_binary"],- "description_stripped": record["description_stripped"],- "description_json": record["description_json"],- "sub_pages_data": record["sub_pages_data"],- "created_by_id": str(record["created_by_id"]),- "updated_by_id": str(record["updated_by_id"]),- "deleted_at": str(record["deleted_at"]) if record.get("deleted_at") else None,- "last_saved_at": (str(record["last_saved_at"]) if record.get("last_saved_at") else None),- }---def transform_issue_description_version(record: Dict) -> Dict:- """Transform issue description version record."""- return {- "id": str(record["id"]),- "created_at": str(record["created_at"]) if record.get("created_at") else None,- "issue_id": str(record["issue_id"]),- "workspace_id": str(record["workspace_id"]),- "project_id": str(record["project_id"]),- "created_by_id": str(record["created_by_id"]),- "updated_by_id": str(record["updated_by_id"]),- "owned_by_id": str(record["owned_by_id"]),- "last_saved_at": (str(record["last_saved_at"]) if record.get("last_saved_at") else None),- "description_binary": record["description_binary"],- "description_html": record["description_html"],- "description_stripped": record["description_stripped"],- "description_json": record["description_json"],- "deleted_at": str(record["deleted_at"]) if record.get("deleted_at") else None,- }---def transform_webhook_log(record: Dict):- """Transfer webhook logs to a new destination."""- return {- "id": str(record["id"]),- "created_at": str(record["created_at"]) if record.get("created_at") else None,- "workspace_id": str(record["workspace_id"]),- "webhook": str(record["webhook"]),- # Request- "event_type": str(record["event_type"]),- "request_method": str(record["request_method"]),- "request_headers": str(record["request_headers"]),- "request_body": str(record["request_body"]),- # Response- "response_status": str(record["response_status"]),- "response_body": str(record["response_body"]),- "response_headers": str(record["response_headers"]),- # retry count- "retry_count": str(record["retry_count"]),- }---# Queryset functions for each cleanup task+# Queryset functions for each cleanup task β each yields primary keys to deletedef get_api_logs_queryset():- """Get API logs older than cutoff days."""- cutoff_days = int(os.environ.get("HARD_DELETE_AFTER_DAYS", 30))- cutoff_time = timezone.now() - timedelta(days=cutoff_days)+ """Get API activity logs older than the API retention window."""+ cutoff_time = timezone.now() - timedelta(days=settings.API_ACTIVITY_LOG_RETENTION_DAYS)logger.info(f"API logs cutoff time: {cutoff_time}")-return (APIActivityLog.all_objects.filter(created_at__lte=cutoff_time)- .values(- "id",- "created_at",- "token_identifier",- "path",- "method",- "query_params",- "headers",- "body",- "response_code",- "response_body",- "ip_address",- "user_agent",- "created_by_id",- )+ .values_list("id", flat=True).iterator(chunk_size=BATCH_SIZE))def get_email_logs_queryset():- """Get email logs older than cutoff days."""- cutoff_days = int(os.environ.get("HARD_DELETE_AFTER_DAYS", 30))- cutoff_time = timezone.now() - timedelta(days=cutoff_days)+ """Get email logs older than the email retention window."""+ cutoff_time = timezone.now() - timedelta(days=settings.EMAIL_LOG_RETENTION_DAYS)logger.info(f"Email logs cutoff time: {cutoff_time}")-return (EmailNotificationLog.all_objects.filter(sent_at__lte=cutoff_time)- .values(- "id",- "created_at",- "receiver_id",- "triggered_by_id",- "entity_identifier",- "entity_name",- "data",- "processed_at",- "sent_at",- "entity",- "old_value",- "new_value",- "created_by_id",- )+ .values_list("id", flat=True).iterator(chunk_size=BATCH_SIZE))@@ -334,22 +120,7 @@ def get_page_versions_queryset():return (PageVersion.all_objects.filter(id__in=Subquery(subq))- .values(- "id",- "created_at",- "page_id",- "workspace_id",- "owned_by_id",- "description_html",- "description_binary",- "description_stripped",- "description_json",- "sub_pages_data",- "created_by_id",- "updated_by_id",- "deleted_at",- "last_saved_at",- )+ .values_list("id", flat=True).iterator(chunk_size=BATCH_SIZE))@@ -370,52 +141,20 @@ def get_issue_description_versions_queryset():return (IssueDescriptionVersion.all_objects.filter(id__in=Subquery(subq))- .values(- "id",- "created_at",- "issue_id",- "workspace_id",- "project_id",- "created_by_id",- "updated_by_id",- "owned_by_id",- "last_saved_at",- "description_binary",- "description_html",- "description_stripped",- "description_json",- "deleted_at",- )+ .values_list("id", flat=True).iterator(chunk_size=BATCH_SIZE))def get_webhook_logs_queryset():- """Get email logs older than cutoff days."""- cutoff_days = int(os.environ.get("HARD_DELETE_AFTER_DAYS", 30))- cutoff_time = timezone.now() - timedelta(days=cutoff_days)+ """Get webhook logs older than the webhook retention window."""+ cutoff_time = timezone.now() - timedelta(days=settings.WEBHOOK_LOG_RETENTION_DAYS)logger.info(f"Webhook logs cutoff time: {cutoff_time}")-return (WebhookLog.all_objects.filter(created_at__lte=cutoff_time)- .values(- "id",- "created_at",- "workspace_id",- "webhook",- "event_type",- # Request- "request_method",- "request_headers",- "request_body",- # Response- "response_status",- "response_body",- "response_headers",- "retry_count",- ).order_by("created_at")- .iterator(chunk_size=100)+ .values_list("id", flat=True)+ .iterator(chunk_size=BATCH_SIZE))@@ -424,10 +163,8 @@ def delete_api_logs():"""Delete old API activity logs."""process_cleanup_task(queryset_func=get_api_logs_queryset,- transform_func=transform_api_log,model=APIActivityLog,task_name="API Activity Log",- collection_name="api_activity_logs",)@@ -436,10 +173,8 @@ def delete_email_notification_logs():"""Delete old email notification logs."""process_cleanup_task(queryset_func=get_email_logs_queryset,- transform_func=transform_email_log,model=EmailNotificationLog,task_name="Email Notification Log",- collection_name="email_notification_logs",)@@ -448,10 +183,8 @@ def delete_page_versions():"""Delete excess page versions."""process_cleanup_task(queryset_func=get_page_versions_queryset,- transform_func=transform_page_version,model=PageVersion,task_name="Page Version",- collection_name="page_versions",)@@ -460,20 +193,16 @@ def delete_issue_description_versions():"""Delete excess issue description versions."""process_cleanup_task(queryset_func=get_issue_description_versions_queryset,- transform_func=transform_issue_description_version,model=IssueDescriptionVersion,task_name="Issue Description Version",- collection_name="issue_description_versions",)@shared_taskdef delete_webhook_logs():- """Delete old webhook logs"""+ """Delete old webhook logs."""process_cleanup_task(queryset_func=get_webhook_logs_queryset,- transform_func=transform_webhook_log,model=WebhookLog,task_name="Webhook Log",- collection_name="webhook_logs",)diff --git a/apps/api/plane/bgtasks/logger_task.py b/apps/api/plane/bgtasks/logger_task.pyindex 4a74e54bc5..3285e544a3 100644--- a/apps/api/plane/bgtasks/logger_task.py+++ b/apps/api/plane/bgtasks/logger_task.py@@ -4,14 +4,12 @@# Python importsimport logging-from typing import Optional, Dict, Any+from typing import Dict, Any# Third party imports-from pymongo.collection import Collectionfrom celery import shared_task# Django imports-from plane.settings.mongo import MongoConnectionfrom plane.utils.exception_logger import log_exceptionfrom plane.db.models import APIActivityLog@@ -19,66 +17,9 @@ from plane.db.models import APIActivityLoglogger = logging.getLogger("plane.worker")-def get_mongo_collection() -> Optional[Collection]:- """- Returns the MongoDB collection for external API activity logs.- """- if not MongoConnection.is_configured():- logger.info("MongoDB not configured")- return None-- try:- return MongoConnection.get_collection("api_activity_logs")- except Exception as e:- logger.error(f"Error getting MongoDB collection: {str(e)}")- log_exception(e)- return None---def safe_decode_body(content: bytes) -> Optional[str]:- """- Safely decodes request/response body content, handling binary data.- Returns "[Binary Content]" if the content is binary, or a string representation of the content.- Returns None if the content is None or empty.- """- # If the content is None, return None- if content is None:- return None-- # If the content is an empty bytes object, return None- if content == b"":- return None-- # Check if content is binary by looking for common binary file signatures- if content.startswith(b"\x89PNG") or content.startswith(b"\xff\xd8\xff") or content.startswith(b"%PDF"):- return "[Binary Content]"-- try:- return content.decode("utf-8")- except UnicodeDecodeError:- return "[Could not decode content]"---def log_to_mongo(log_document: Dict[str, Any]) -> bool:- """- Logs the request to MongoDB if available.- """- mongo_collection = get_mongo_collection()- if mongo_collection is None:- logger.error("MongoDB not configured")- return False-- try:- mongo_collection.insert_one(log_document)- return True- except Exception as e:- log_exception(e)- return False--def log_to_postgres(log_data: Dict[str, Any]) -> bool:"""- Fallback to logging to PostgreSQL if MongoDB is unavailable.+ Persist an external API request log to PostgreSQL."""try:APIActivityLog.objects.create(**log_data)@@ -89,12 +30,12 @@ def log_to_postgres(log_data: Dict[str, Any]) -> bool:@shared_task-def process_logs(log_data: Dict[str, Any], mongo_log: Dict[str, Any]) -> None:- """- Process logs to save to MongoDB or Postgres based on the configuration+def process_logs(log_data: Dict[str, Any], **_: Any) -> None:"""+ Persist external API request logs to PostgreSQL.- if MongoConnection.is_configured():- log_to_mongo(mongo_log)- else:- log_to_postgres(log_data)+ The catch-all kwargs keep this task signature compatible with jobs enqueued+ by an older release (which passed a `mongo_log` argument), so in-flight tasks+ don't fail during a rolling deploy. It can be dropped once no such jobs remain.+ """+ log_to_postgres(log_data)diff --git a/apps/api/plane/bgtasks/webhook_task.py b/apps/api/plane/bgtasks/webhook_task.pyindex ddb5299fb5..02e6915d4e 100644--- a/apps/api/plane/bgtasks/webhook_task.py+++ b/apps/api/plane/bgtasks/webhook_task.py@@ -53,7 +53,6 @@ from plane.license.utils.instance_value import get_email_configurationfrom plane.utils.email import generate_plain_text_from_htmlfrom plane.utils.exception_logger import log_exceptionfrom plane.utils.ip_address import validate_url-from plane.settings.mongo import MongoConnectionSERIALIZER_MAPPER = {@@ -102,9 +101,6 @@ def save_webhook_log(retry_count: int,event_type: str,) -> None:- # webhook_logs- mongo_collection = MongoConnection.get_collection("webhook_logs")-log_data = {"workspace_id": str(webhook.workspace_id),"webhook": str(webhook.id),@@ -118,27 +114,12 @@ def save_webhook_log("retry_count": retry_count,}- mongo_save_success = False- if mongo_collection is not None:- try:- # insert the log data into the mongo collection- mongo_collection.insert_one(log_data)- logger.info("Webhook log saved successfully to mongo")- mongo_save_success = True- except Exception as e:- log_exception(e, warning=True)- logger.error(f"Failed to save webhook log: {e}")- mongo_save_success = False-- # if the mongo save is not successful, save the log data into the database- if not mongo_save_success:- try:- # insert the log data into the database- WebhookLog.objects.create(**log_data)- logger.info("Webhook log saved successfully to database")- except Exception as e:- log_exception(e, warning=True)- logger.error(f"Failed to save webhook log: {e}")+ try:+ WebhookLog.objects.create(**log_data)+ logger.info("Webhook log saved successfully to database")+ except Exception as e:+ log_exception(e, warning=True)+ logger.error(f"Failed to save webhook log: {e}")def get_model_data(event: str, event_id: Union[str, List[str]], many: bool = False) -> Dict[str, Any]:diff --git a/apps/api/plane/middleware/logger.py b/apps/api/plane/middleware/logger.pyindex b8cf6f9c04..343409c4e1 100644--- a/apps/api/plane/middleware/logger.py+++ b/apps/api/plane/middleware/logger.py@@ -3,12 +3,14 @@# See the LICENSE file for details.# Python imports+import hashlib+import hmacimport loggingimport time# Django imports+from django.conf import settingsfrom django.http import HttpRequest-from django.utils import timezone# Third party importsfrom rest_framework.request import Request@@ -77,7 +79,7 @@ class RequestLoggerMiddleware:class APITokenLogMiddleware:"""- Middleware to log External API requests to MongoDB or PostgreSQL.+ Middleware to log External API requests to PostgreSQL."""def __init__(self, get_response):@@ -111,6 +113,20 @@ class APITokenLogMiddleware:except UnicodeDecodeError:return "[Could not decode content]"+ # Headers whose values must never be persisted in plaintext logs+ SENSITIVE_HEADERS = frozenset({"x-api-key", "authorization", "cookie"})++ def _redacted_headers(self, request):+ """+ Returns the request headers as a string with sensitive values redacted,+ so that credentials such as the API key are never stored in plaintext.+ """+ redacted = {+ key: ("[REDACTED]" if key.lower() in self.SENSITIVE_HEADERS else value)+ for key, value in request.headers.items()+ }+ return str(redacted)+def process_request(self, request, response, request_body):api_key_header = "X-Api-Key"api_key = request.headers.get(api_key_header)@@ -121,32 +137,25 @@ class APITokenLogMiddleware:try:log_data = {- "token_identifier": api_key,+ # Tokenize the (high-entropy) API key into a stable, non-reversible+ # identifier so logs can be correlated to a token without ever+ # persisting the raw key. A keyed HMAC is used rather than a bare+ # hash so the digest cannot be precomputed from a known key value.+ "token_identifier": hmac.new(+ settings.SECRET_KEY.encode(), api_key.encode(), hashlib.sha256+ ).hexdigest(),"path": request.path,"method": request.method,"query_params": request.META.get("QUERY_STRING", ""),- "headers": str(request.headers),+ "headers": self._redacted_headers(request),"body": self._safe_decode_body(request_body) if request_body else None,"response_body": self._safe_decode_body(response.content) if response.content else None,"response_code": response.status_code,"ip_address": get_client_ip(request=request),"user_agent": request.META.get("HTTP_USER_AGENT", None),}- user_id = (- str(request.user.id)- if getattr(request, "user") and getattr(request.user, "is_authenticated", False)- else None- )- # Additional fields for MongoDB- mongo_log = {- **log_data,- "created_at": timezone.now(),- "updated_at": timezone.now(),- "created_by": user_id,- "updated_by": user_id,- }- process_logs.delay(log_data=log_data, mongo_log=mongo_log)+ process_logs.delay(log_data=log_data)except Exception as e:log_exception(e)diff --git a/apps/api/plane/settings/common.py b/apps/api/plane/settings/common.pyindex 9a6ec57c93..59140c070d 100644--- a/apps/api/plane/settings/common.py+++ b/apps/api/plane/settings/common.py@@ -402,6 +402,34 @@ WEB_URL = os.environ.get("WEB_URL")HARD_DELETE_AFTER_DAYS = int(os.environ.get("HARD_DELETE_AFTER_DAYS", 60))++def _retention_days(env_var, default):+ """+ Read a retention window (in days) from the environment, falling back to the+ default when the variable is unset, unparseable, or negative β a negative+ window would otherwise select rows with a future cutoff and delete everything.+ """+ raw = os.environ.get(env_var)+ if raw is None:+ return default+ try:+ days = int(raw)+ except ValueError:+ return default+ return days if days >= 0 else default+++# API activity logs hold request/response payloads, so they are retained for a+# shorter window than other logs.+API_ACTIVITY_LOG_RETENTION_DAYS = _retention_days("API_ACTIVITY_LOG_RETENTION_DAYS", 14)++# Webhook delivery logs are retained on their own window, independent of the+# generic HARD_DELETE_AFTER_DAYS.+WEBHOOK_LOG_RETENTION_DAYS = _retention_days("WEBHOOK_LOG_RETENTION_DAYS", 14)++# Email notification logs are retained on their own window.+EMAIL_LOG_RETENTION_DAYS = _retention_days("EMAIL_LOG_RETENTION_DAYS", 7)+# Instance Changelog URLINSTANCE_CHANGELOG_URL = os.environ.get("INSTANCE_CHANGELOG_URL", "")@@ -504,7 +532,3 @@ if ENABLE_DRF_SPECTACULAR:REST_FRAMEWORK["DEFAULT_SCHEMA_CLASS"] = "drf_spectacular.openapi.AutoSchema"INSTALLED_APPS.append("drf_spectacular")from .openapi import SPECTACULAR_SETTINGS # noqa: F401--# MongoDB Settings-MONGO_DB_URL = os.environ.get("MONGO_DB_URL", False)-MONGO_DB_DATABASE = os.environ.get("MONGO_DB_DATABASE", False)diff --git a/apps/api/plane/settings/local.py b/apps/api/plane/settings/local.pyindex e71f40bf14..a346eca141 100644--- a/apps/api/plane/settings/local.py+++ b/apps/api/plane/settings/local.py@@ -75,11 +75,6 @@ LOGGING = {"handlers": ["console"],"propagate": False,},- "plane.mongo": {- "level": "INFO",- "handlers": ["console"],- "propagate": False,- },"plane.authentication": {"level": "INFO","handlers": ["console"],diff --git a/apps/api/plane/settings/mongo.py b/apps/api/plane/settings/mongo.pydeleted file mode 100644index 7855a52d51..0000000000--- a/apps/api/plane/settings/mongo.py+++ /dev/null@@ -1,126 +0,0 @@-# Copyright (c) 2023-present Plane Software, Inc. and contributors-# SPDX-License-Identifier: AGPL-3.0-only-# See the LICENSE file for details.--# Django imports-from django.conf import settings-import logging--# Third party imports-from pymongo import MongoClient-from pymongo.database import Database-from pymongo.collection import Collection-from typing import Optional, TypeVar, Type---T = TypeVar("T", bound="MongoConnection")--# Set up logger-logger = logging.getLogger("plane.mongo")---class MongoConnection:- """- A singleton class that manages MongoDB connections.-- This class ensures only one MongoDB connection is maintained throughout the application.- It provides methods to access the MongoDB client, database, and collections.-- Attributes:- _instance (Optional[MongoConnection]): The singleton instance of this class- _client (Optional[MongoClient]): The MongoDB client instance- _db (Optional[Database]): The MongoDB database instance- """-- _instance: Optional["MongoConnection"] = None- _client: Optional[MongoClient] = None- _db: Optional[Database] = None-- def __new__(cls: Type[T]) -> T:- """- Creates a new instance of MongoConnection if one doesn't exist.-- Returns:- MongoConnection: The singleton instance- """- if cls._instance is None:- cls._instance = super(MongoConnection, cls).__new__(cls)- try:- mongo_url = getattr(settings, "MONGO_DB_URL", None)- mongo_db_database = getattr(settings, "MONGO_DB_DATABASE", None)-- if not mongo_url or not mongo_db_database:- logger.warning(- "MongoDB connection parameters not configured. MongoDB functionality will be disabled."- )- return cls._instance-- cls._client = MongoClient(mongo_url)- cls._db = cls._client[mongo_db_database]-- # Test the connection- cls._client.server_info()- logger.info("MongoDB connection established successfully")- except Exception as e:- logger.warning(- f"Failed to initialize MongoDB connection: {str(e)}. MongoDB functionality will be disabled."- )- return cls._instance-- @classmethod- def get_client(cls) -> Optional[MongoClient]:- """- Returns the MongoDB client instance.-- Returns:- Optional[MongoClient]: The MongoDB client instance or None if not configured- """- if cls._client is None:- cls._instance = cls()- return cls._client-- @classmethod- def get_db(cls) -> Optional[Database]:- """- Returns the MongoDB database instance.-- Returns:- Optional[Database]: The MongoDB database instance or None if not configured- """- if cls._db is None:- cls._instance = cls()- return cls._db-- @classmethod- def get_collection(cls, collection_name: str) -> Optional[Collection]:- """- Returns a MongoDB collection by name.-- Args:- collection_name (str): The name of the collection to retrieve-- Returns:- Optional[Collection]: The MongoDB collection instance or None if not configured- """- try:- db = cls.get_db()- if db is None:- logger.warning(f"Cannot access collection '{collection_name}': MongoDB not configured")- return None- return db[collection_name]- except Exception as e:- logger.warning(f"Failed to access collection '{collection_name}': {str(e)}")- return None-- @classmethod- def is_configured(cls) -> bool:- """- Check if MongoDB is properly configured and connected.-- Returns:- bool: True if MongoDB is configured and connected, False otherwise- """-- if cls._client is None:- cls._instance = cls()- return cls._client is not None and cls._db is not Nonediff --git a/apps/api/plane/settings/production.py b/apps/api/plane/settings/production.pyindex f58450a9da..1dbf43196a 100644--- a/apps/api/plane/settings/production.py+++ b/apps/api/plane/settings/production.py@@ -85,11 +85,6 @@ LOGGING = {"handlers": ["console"],"propagate": False,},- "plane.mongo": {- "level": "INFO",- "handlers": ["console"],- "propagate": False,- },"plane.authentication": {"level": "DEBUG" if DEBUG else "INFO","handlers": ["console"],diff --git a/apps/api/plane/tests/README.md b/apps/api/plane/tests/README.mdindex df9aba6da1..3d6936d785 100644--- a/apps/api/plane/tests/README.md+++ b/apps/api/plane/tests/README.md@@ -91,7 +91,7 @@ When writing tests, follow these guidelines:- For web app API (`/api/`), use `session_client`- For smoke tests with real HTTP, use `plane_server`3. Use the correct URL namespace when reverse-resolving URLs:- - For external API, use `reverse("api:endpoint_name")`+ - For external API, use `reverse("api:endpoint_name")`- For web app API, use `reverse("endpoint_name")`4. Add the `@pytest.mark.django_db` decorator to tests that interact with the database.5. Add the appropriate markers (`@pytest.mark.contract`, etc.) to categorize tests.@@ -101,7 +101,7 @@ When writing tests, follow these guidelines:Common fixtures are defined in:- `conftest.py`: General fixtures for authentication, database access, etc.-- `conftest_external.py`: Fixtures for external services (Redis, Elasticsearch, Celery, MongoDB)+- `conftest_external.py`: Fixtures for external services (Redis, Elasticsearch, Celery)- `factories.py`: Test factories for easy model instance creation## Best Practices@@ -125,7 +125,7 @@ When writing tests, follow these guidelines:Tests for components that interact with external services should:-1. Use the `mock_redis`, `mock_elasticsearch`, `mock_mongodb`, and `mock_celery` fixtures for unit and most contract tests.+1. Use the `mock_redis`, `mock_elasticsearch`, and `mock_celery` fixtures for unit and most contract tests.2. For more comprehensive contract tests, use Docker-based test containers (optional).## Coverage Reports@@ -140,4 +140,4 @@ This creates an HTML report in the `htmlcov/` directory.## Migration from Old Tests-Some tests are still in the old format in the `api/` directory. These need to be migrated to the new contract test structure in the appropriate directories.\ No newline at end of file+Some tests are still in the old format in the `api/` directory. These need to be migrated to the new contract test structure in the appropriate directories.diff --git a/apps/api/plane/tests/conftest_external.py b/apps/api/plane/tests/conftest_external.pyindex cd5469caa6..16bc31a430 100644--- a/apps/api/plane/tests/conftest_external.py+++ b/apps/api/plane/tests/conftest_external.py@@ -51,41 +51,6 @@ def mock_elasticsearch():yield mock_es_client-def mock_mongodb():- """- Mock MongoDB for testing without actual MongoDB connection.-- This fixture patches PyMongo to return a MagicMock that behaves like a MongoDB client.- """- # Create mock MongoDB clients and collections- mock_mongo_client = MagicMock()- mock_mongo_db = MagicMock()- mock_mongo_collection = MagicMock()-- # Set up the chain: client -> database -> collection- mock_mongo_client.__getitem__.return_value = mock_mongo_db- mock_mongo_client.get_database.return_value = mock_mongo_db- mock_mongo_db.__getitem__.return_value = mock_mongo_collection-- # Configure common MongoDB collection operations- mock_mongo_collection.find_one.return_value = None- mock_mongo_collection.find.return_value = MagicMock(__iter__=lambda x: iter([]), count=lambda: 0)- mock_mongo_collection.insert_one.return_value = MagicMock(inserted_id="mock_id_123", acknowledged=True)- mock_mongo_collection.insert_many.return_value = MagicMock(- inserted_ids=["mock_id_123", "mock_id_456"], acknowledged=True- )- mock_mongo_collection.update_one.return_value = MagicMock(modified_count=1, matched_count=1, acknowledged=True)- mock_mongo_collection.update_many.return_value = MagicMock(modified_count=2, matched_count=2, acknowledged=True)- mock_mongo_collection.delete_one.return_value = MagicMock(deleted_count=1, acknowledged=True)- mock_mongo_collection.delete_many.return_value = MagicMock(deleted_count=2, acknowledged=True)- mock_mongo_collection.count_documents.return_value = 0-- # Start the patch- with patch("pymongo.MongoClient", return_value=mock_mongo_client):- yield mock_mongo_client--@pytest.fixturedef mock_celery():"""diff --git a/apps/api/plane/tests/contract/app/test_api_token.py b/apps/api/plane/tests/contract/app/test_api_token.pyindex ed071b98cb..60ebd8fde8 100644--- a/apps/api/plane/tests/contract/app/test_api_token.py+++ b/apps/api/plane/tests/contract/app/test_api_token.py@@ -366,6 +366,23 @@ class TestApiTokenEndpoint:create_api_token_for_user.refresh_from_db()assert create_api_token_for_user.user_type == 0+ @pytest.mark.django_db+ def test_patch_cannot_modify_allowed_rate_limit(self, session_client, create_user, create_api_token_for_user):+ """Test that allowed_rate_limit cannot be modified via PATCH"""+ # Arrange+ session_client.force_authenticate(user=create_user)+ url = reverse("api-tokens-details", kwargs={"pk": create_api_token_for_user.pk})+ original_rate_limit = create_api_token_for_user.allowed_rate_limit+ update_data = {"allowed_rate_limit": "100000/min"}++ # Act+ response = session_client.patch(url, update_data, format="json")++ # Assert+ assert response.status_code == status.HTTP_200_OK+ create_api_token_for_user.refresh_from_db()+ assert create_api_token_for_user.allowed_rate_limit == original_rate_limit+@pytest.mark.django_dbdef test_patch_cannot_modify_service_token(self, session_client, create_user):"""Test that service tokens cannot be modified through user token endpoint"""diff --git a/apps/api/plane/tests/unit/bg_tasks/test_cleanup_task.py b/apps/api/plane/tests/unit/bg_tasks/test_cleanup_task.pynew file mode 100644index 0000000000..d0a85a19bc--- /dev/null+++ b/apps/api/plane/tests/unit/bg_tasks/test_cleanup_task.py@@ -0,0 +1,143 @@+# Copyright (c) 2023-present Plane Software, Inc. and contributors+# SPDX-License-Identifier: AGPL-3.0-only+# See the LICENSE file for details.++"""+Unit tests for the log cleanup tasks.++Verifies that API activity logs past the retention window are hard-deleted+(removed from PostgreSQL, not soft-deleted) and that fresh logs are retained.+"""++from datetime import timedelta++import pytest+from django.conf import settings+from django.utils import timezone++from uuid import uuid4++from plane.bgtasks.cleanup_task import (+ delete_api_logs,+ delete_email_notification_logs,+ delete_webhook_logs,+ process_cleanup_task,+)+from plane.db.models import APIActivityLog, EmailNotificationLog, WebhookLog+from plane.tests.factories import UserFactory, WorkspaceFactory+++def _make_api_log(created_at):+ log = APIActivityLog.objects.create(+ token_identifier="hashed-token",+ path="/api/v1/workspaces/",+ method="GET",+ response_code=200,+ )+ # created_at is auto-set on insert, so backdate it explicitly afterwards.+ APIActivityLog.all_objects.filter(pk=log.pk).update(created_at=created_at)+ return log+++def _make_webhook_log(workspace, created_at):+ log = WebhookLog.objects.create(+ workspace=workspace,+ webhook=uuid4(),+ event_type="issue",+ request_method="POST",+ response_status="200",+ )+ WebhookLog.all_objects.filter(pk=log.pk).update(created_at=created_at)+ return log+++def _make_email_log(user, sent_at):+ return EmailNotificationLog.objects.create(+ receiver=user,+ triggered_by=user,+ entity_name="issue",+ entity="issue",+ sent_at=sent_at,+ )+++class TestDeleteApiLogs:+ def test_expired_logs_are_hard_deleted(self):+ retention_days = settings.API_ACTIVITY_LOG_RETENTION_DAYS+ expired = _make_api_log(timezone.now() - timedelta(days=retention_days + 1))++ delete_api_logs()++ # Hard delete: the row must be gone even from the unfiltered manager.+ assert not APIActivityLog.all_objects.filter(pk=expired.pk).exists()++ def test_recent_logs_are_retained(self):+ retention_days = settings.API_ACTIVITY_LOG_RETENTION_DAYS+ recent = _make_api_log(timezone.now() - timedelta(days=retention_days - 1))++ delete_api_logs()++ assert APIActivityLog.all_objects.filter(pk=recent.pk).exists()+++class TestDeleteWebhookLogs:+ def test_expired_logs_are_hard_deleted(self):+ workspace = WorkspaceFactory()+ retention_days = settings.WEBHOOK_LOG_RETENTION_DAYS+ expired = _make_webhook_log(workspace, timezone.now() - timedelta(days=retention_days + 1))++ delete_webhook_logs()++ assert not WebhookLog.all_objects.filter(pk=expired.pk).exists()++ def test_recent_logs_are_retained(self):+ workspace = WorkspaceFactory()+ retention_days = settings.WEBHOOK_LOG_RETENTION_DAYS+ recent = _make_webhook_log(workspace, timezone.now() - timedelta(days=retention_days - 1))++ delete_webhook_logs()++ assert WebhookLog.all_objects.filter(pk=recent.pk).exists()+++class TestDeleteEmailLogs:+ def test_expired_logs_are_hard_deleted(self):+ user = UserFactory()+ retention_days = settings.EMAIL_LOG_RETENTION_DAYS+ expired = _make_email_log(user, timezone.now() - timedelta(days=retention_days + 1))++ delete_email_notification_logs()++ assert not EmailNotificationLog.all_objects.filter(pk=expired.pk).exists()++ def test_recent_logs_are_retained(self):+ user = UserFactory()+ retention_days = settings.EMAIL_LOG_RETENTION_DAYS+ recent = _make_email_log(user, timezone.now() - timedelta(days=retention_days - 1))++ delete_email_notification_logs()++ assert EmailNotificationLog.all_objects.filter(pk=recent.pk).exists()+++class TestProcessCleanupTaskErrorHandling:+ def test_batch_delete_failure_is_swallowed(self):+ """A failing batch is logged and skipped; the run does not raise."""++ class _BoomManager:+ @staticmethod+ def filter(**kwargs):+ raise RuntimeError("db unavailable")++ class _BoomModel:+ all_objects = _BoomManager()++ # Should not raise even though the delete blows up.+ process_cleanup_task(lambda: iter([1, 2, 3]), _BoomModel, "Boom")diff --git a/apps/api/plane/tests/unit/middleware/test_logger.py b/apps/api/plane/tests/unit/middleware/test_logger.pynew file mode 100644index 0000000000..5c13f53f6f--- /dev/null+++ b/apps/api/plane/tests/unit/middleware/test_logger.py@@ -0,0 +1,79 @@+# Copyright (c) 2023-present Plane Software, Inc. and contributors+# SPDX-License-Identifier: AGPL-3.0-only+# See the LICENSE file for details.++"""+Unit tests for APITokenLogMiddleware.++Covers the credential-hygiene guarantees of the external API request logger:+- the raw API key is never persisted (a non-reversible hash is stored instead)+- sensitive request headers are redacted before being logged+"""++import hashlib+import hmac+from unittest.mock import Mock, patch++import pytest+from django.conf import settings+from django.contrib.auth.models import AnonymousUser+from django.http import HttpResponse+from django.test import RequestFactory++from plane.middleware.logger import APITokenLogMiddleware+++def request_factory():+ return RequestFactory()+++def middleware():+ return APITokenLogMiddleware(Mock(return_value=HttpResponse(b"{}")))+++class TestAPITokenLogMiddleware:+ API_KEY = "plane_api_supersecretvalue"+ AUTHORIZATION = "Bearer secret-bearer-token"+ COOKIE = "sessionid=secret-session-value"++ def _captured_log_data(self, middleware, request_factory):+ request = request_factory.get(+ "/api/v1/workspaces/",+ HTTP_X_API_KEY=self.API_KEY,+ HTTP_AUTHORIZATION=self.AUTHORIZATION,+ HTTP_COOKIE=self.COOKIE,+ )+ request.user = AnonymousUser()+ response = HttpResponse(b"{}")+ with patch("plane.middleware.logger.process_logs") as process_logs:+ middleware.process_request(request, response, request_body=b"")+ assert process_logs.delay.called+ return process_logs.delay.call_args.kwargs["log_data"]++ def test_token_identifier_is_hashed_not_plaintext(self, middleware, request_factory):+ log_data = self._captured_log_data(middleware, request_factory)++ expected_hash = hmac.new(+ settings.SECRET_KEY.encode(), self.API_KEY.encode(), hashlib.sha256+ ).hexdigest()+ assert log_data["token_identifier"] == expected_hash+ assert self.API_KEY not in log_data["token_identifier"]++ def test_sensitive_headers_are_redacted(self, middleware, request_factory):+ log_data = self._captured_log_data(middleware, request_factory)++ # None of the sensitive header values may appear in the logged headers.+ assert self.API_KEY not in log_data["headers"]+ assert self.AUTHORIZATION not in log_data["headers"]+ assert self.COOKIE not in log_data["headers"]+ assert "[REDACTED]" in log_data["headers"]++ def test_no_log_without_api_key(self, middleware, request_factory):+ request = request_factory.get("/api/v1/workspaces/")+ request.user = AnonymousUser()+ with patch("plane.middleware.logger.process_logs") as process_logs:+ middleware.process_request(request, HttpResponse(b"{}"), request_body=b"")+ assert not process_logs.delay.calleddiff --git a/apps/api/plane/tests/unit/settings/test_retention.py b/apps/api/plane/tests/unit/settings/test_retention.pynew file mode 100644index 0000000000..10bc79192d--- /dev/null+++ b/apps/api/plane/tests/unit/settings/test_retention.py@@ -0,0 +1,34 @@+# Copyright (c) 2023-present Plane Software, Inc. and contributors+# SPDX-License-Identifier: AGPL-3.0-only+# See the LICENSE file for details.++"""Unit tests for the log-retention env parsing helper."""++import pytest++from plane.settings.common import _retention_days++ENV_VAR = "TEST_RETENTION_DAYS"+++class TestRetentionDays:+ def test_uses_default_when_unset(self, monkeypatch):+ monkeypatch.delenv(ENV_VAR, raising=False)+ assert _retention_days(ENV_VAR, 14) == 14++ def test_uses_env_value_when_valid(self, monkeypatch):+ monkeypatch.setenv(ENV_VAR, "30")+ assert _retention_days(ENV_VAR, 14) == 30++ def test_zero_is_allowed(self, monkeypatch):+ monkeypatch.setenv(ENV_VAR, "0")+ assert _retention_days(ENV_VAR, 14) == 0++ def test_negative_falls_back_to_default(self, monkeypatch):+ monkeypatch.setenv(ENV_VAR, "-5")+ assert _retention_days(ENV_VAR, 14) == 14++ def test_unparseable_falls_back_to_default(self, monkeypatch):+ monkeypatch.setenv(ENV_VAR, "abc")+ assert _retention_days(ENV_VAR, 7) == 7diff --git a/apps/api/requirements/base.txt b/apps/api/requirements/base.txtindex a34bf6de11..68c4a4a43c 100644--- a/apps/api/requirements/base.txt+++ b/apps/api/requirements/base.txt@@ -9,8 +9,6 @@ psycopg==3.3.0psycopg-binary==3.3.0psycopg-c==3.3.0dj-database-url==2.1.0-# mongo-pymongo==4.6.3# redisredis==5.0.4django-redis==5.4.0