π₯π₯π₯ Open-source Jira, Linear, Monday, and ClickUp alternative. Plane is a modern project management platform to manage tasks, sprints, docs, and triage.
TypeScript boardsbug-trackerdjangodockerganttissue-trackerjirajira-alternativekanbanlinearpostgresqlproduct-managementproject-managementproject-planningpythonreactredistypesciptvitework-management
[WEB-4873]: Add webhook log cleanup task and update Celery schedule (#7772)
Nikhil69c688b017de
diff --git a/apps/api/plane/bgtasks/cleanup_task.py b/apps/api/plane/bgtasks/cleanup_task.pyindex c9d86b6398..e5cacd6371 100644--- a/apps/api/plane/bgtasks/cleanup_task.py+++ b/apps/api/plane/bgtasks/cleanup_task.py@@ -21,13 +21,14 @@ from plane.db.models import (PageVersion,APIActivityLog,IssueDescriptionVersion,+ WebhookLog,)from plane.settings.mongo import MongoConnectionfrom plane.utils.exception_logger import log_exceptionlogger = logging.getLogger("plane.worker")-BATCH_SIZE = 1000+BATCH_SIZE = 500def get_mongo_collection(collection_name: str) -> Optional[Collection]:@@ -247,6 +248,27 @@ def transform_issue_description_version(record: Dict) -> Dict:}+def transform_webhook_log(record: Dict):+ """Transfer webhook logs to a new destination."""+ return {+ "id": str(record["id"]),+ "created_at": str(record["created_at"]) if record.get("created_at") else None,+ "workspace_id": str(record["workspace_id"]),+ "webhook": str(record["webhook"]),+ # Request+ "event_type": str(record["event_type"]),+ "request_method": str(record["request_method"]),+ "request_headers": str(record["request_headers"]),+ "request_body": str(record["request_body"]),+ # Response+ "response_status": str(record["response_status"]),+ "response_body": str(record["response_body"]),+ "response_headers": str(record["response_headers"]),+ # retry count+ "retry_count": str(record["retry_count"]),+ }++# Queryset functions for each cleanup taskdef get_api_logs_queryset():"""Get API logs older than cutoff days."""@@ -374,7 +396,34 @@ def get_issue_description_versions_queryset():)-# Celery tasks - now much simpler!+def get_webhook_logs_queryset():+ """Get email logs older than cutoff days."""+ cutoff_days = int(os.environ.get("HARD_DELETE_AFTER_DAYS", 30))+ cutoff_time = timezone.now() - timedelta(days=cutoff_days)+ logger.info(f"Webhook logs cutoff time: {cutoff_time}")++ return (+ WebhookLog.all_objects.filter(created_at__lte=cutoff_time)+ .values(+ "id",+ "created_at",+ "workspace_id",+ "webhook",+ "event_type",+ # Request+ "request_method",+ "request_headers",+ "request_body",+ # Response+ "response_status",+ "response_body",+ "response_headers",+ "retry_count",+ )+ .iterator(chunk_size=BATCH_SIZE)+ )++@shared_taskdef delete_api_logs():"""Delete old API activity logs."""@@ -421,3 +470,15 @@ def delete_issue_description_versions():task_name="Issue Description Version",collection_name="issue_description_versions",)+++@shared_task+def delete_webhook_logs():+ """Delete old webhook logs"""+ process_cleanup_task(+ queryset_func=get_webhook_logs_queryset,+ transform_func=transform_webhook_log,+ model=WebhookLog,+ task_name="Webhook Log",+ collection_name="webhook_logs",+ )diff --git a/apps/api/plane/celery.py b/apps/api/plane/celery.pyindex 2eeac358c6..5fb136f23f 100644--- a/apps/api/plane/celery.py+++ b/apps/api/plane/celery.py@@ -55,15 +55,23 @@ app.conf.beat_schedule = {},"check-every-day-to-delete-email-notification-logs": {"task": "plane.bgtasks.cleanup_task.delete_email_notification_logs",- "schedule": crontab(hour=3, minute=0), # UTC 03:00+ "schedule": crontab(hour=2, minute=45), # UTC 02:45},"check-every-day-to-delete-page-versions": {"task": "plane.bgtasks.cleanup_task.delete_page_versions",- "schedule": crontab(hour=3, minute=30), # UTC 03:30+ "schedule": crontab(hour=3, minute=0), # UTC 03:00},"check-every-day-to-delete-issue-description-versions": {"task": "plane.bgtasks.cleanup_task.delete_issue_description_versions",- "schedule": crontab(hour=4, minute=0), # UTC 04:00+ "schedule": crontab(hour=3, minute=15), # UTC 03:15+ },+ "check-every-day-to-delete-webhook-logs": {+ "task": "plane.bgtasks.cleanup_task.delete_webhook_logs",+ "schedule": crontab(hour=3, minute=30), # UTC 03:30+ },+ "check-every-day-to-delete-exporter-history": {+ "task": "plane.bgtasks.exporter_expired_task.delete_old_s3_link",+ "schedule": crontab(hour=3, minute=45), # UTC 03:45},}