π₯π₯π₯ Open-source Jira, Linear, Monday, and ClickUp alternative. Plane is a modern project management platform to manage tasks, sprints, docs, and triage.
TypeScript boardsbug-trackerdjangodockerganttissue-trackerjirajira-alternativekanbanlinearpostgresqlproduct-managementproject-managementproject-planningpythonreactredistypesciptvitework-management
chore: add sync jobs for issue_version and issue_description_version tables (#6199)
guru_sainathdf6a80e7aeef
diff --git a/apiserver/plane/bgtasks/issue_description_version_sync.py b/apiserver/plane/bgtasks/issue_description_version_sync.pynew file mode 100644index 0000000000..14956cb50c--- /dev/null+++ b/apiserver/plane/bgtasks/issue_description_version_sync.py@@ -0,0 +1,125 @@+# Python imports+from typing import Optional+import logging++# Django imports+from django.utils import timezone+from django.db import transaction++# Third party imports+from celery import shared_task++# Module imports+from plane.db.models import Issue, IssueDescriptionVersion, ProjectMember+from plane.utils.exception_logger import log_exception+++def get_owner_id(issue: Issue) -> Optional[int]:+ """Get the owner ID of the issue"""++ if issue.updated_by_id:+ return issue.updated_by_id++ if issue.created_by_id:+ return issue.created_by_id++ # Find project admin as fallback+ project_member = ProjectMember.objects.filter(+ project_id=issue.project_id,+ role=20, # Admin role+ ).first()++ return project_member.member_id if project_member else None+++@shared_task+def sync_issue_description_version(batch_size=5000, offset=0, countdown=300):+ """Task to create IssueDescriptionVersion records for existing Issues in batches"""+ try:+ with transaction.atomic():+ base_query = Issue.objects+ total_issues_count = base_query.count()++ if total_issues_count == 0:+ return++ # Calculate batch range+ end_offset = min(offset + batch_size, total_issues_count)++ # Fetch issues with related data+ issues_batch = (+ base_query.order_by("created_at")+ .select_related("workspace", "project")+ .only(+ "id",+ "workspace_id",+ "project_id",+ "created_by_id",+ "updated_by_id",+ "description_binary",+ "description_html",+ "description_stripped",+ "description",+ )[offset:end_offset]+ )++ if not issues_batch:+ return++ version_objects = []+ for issue in issues_batch:+ # Validate required fields+ if not issue.workspace_id or not issue.project_id:+ logging.warning(+ f"Skipping {issue.id} - missing workspace_id or project_id"+ )+ continue++ # Determine owned_by_id+ owned_by_id = get_owner_id(issue)+ if owned_by_id is None:+ logging.warning(f"Skipping issue {issue.id} - missing owned_by")+ continue++ # Create version object+ version_objects.append(+ IssueDescriptionVersion(+ workspace_id=issue.workspace_id,+ project_id=issue.project_id,+ created_by_id=issue.created_by_id,+ updated_by_id=issue.updated_by_id,+ owned_by_id=owned_by_id,+ last_saved_at=timezone.now(),+ issue_id=issue.id,+ description_binary=issue.description_binary,+ description_html=issue.description_html,+ description_stripped=issue.description_stripped,+ description_json=issue.description,+ )+ )++ # Bulk create version objects+ if version_objects:+ IssueDescriptionVersion.objects.bulk_create(version_objects)++ # Schedule next batch if needed+ if end_offset < total_issues_count:+ sync_issue_description_version.apply_async(+ kwargs={+ "batch_size": batch_size,+ "offset": end_offset,+ "countdown": countdown,+ },+ countdown=countdown,+ )+ return+ except Exception as e:+ log_exception(e)+ return+++@shared_task+def schedule_issue_description_version(batch_size=5000, countdown=300):+ sync_issue_description_version.delay(+ batch_size=int(batch_size), countdown=countdown+ )diff --git a/apiserver/plane/bgtasks/issue_version_sync.py b/apiserver/plane/bgtasks/issue_version_sync.pynew file mode 100644index 0000000000..698cedf155--- /dev/null+++ b/apiserver/plane/bgtasks/issue_version_sync.py@@ -0,0 +1,254 @@+# Python imports+import json+from typing import Optional, List, Dict+from uuid import UUID+from itertools import groupby+import logging++# Django imports+from django.utils import timezone+from django.db import transaction++# Third party imports+from celery import shared_task++# Module imports+from plane.db.models import (+ Issue,+ IssueVersion,+ ProjectMember,+ CycleIssue,+ ModuleIssue,+ IssueActivity,+ IssueAssignee,+ IssueLabel,+)+from plane.utils.exception_logger import log_exception+++@shared_task+def issue_task(updated_issue, issue_id, user_id):+ try:+ current_issue = json.loads(updated_issue) if updated_issue else {}+ issue = Issue.objects.get(id=issue_id)++ updated_current_issue = {}+ for key, value in current_issue.items():+ if getattr(issue, key) != value:+ updated_current_issue[key] = value++ if updated_current_issue:+ issue_version = (+ IssueVersion.objects.filter(issue_id=issue_id)+ .order_by("-last_saved_at")+ .first()+ )++ if (+ issue_version+ and str(issue_version.owned_by) == str(user_id)+ and (timezone.now() - issue_version.last_saved_at).total_seconds()+ <= 600+ ):+ for key, value in updated_current_issue.items():+ setattr(issue_version, key, value)+ issue_version.last_saved_at = timezone.now()+ issue_version.save(+ update_fields=list(updated_current_issue.keys()) + ["last_saved_at"]+ )+ else:+ IssueVersion.log_issue_version(issue, user_id)++ return+ except Issue.DoesNotExist:+ return+ except Exception as e:+ log_exception(e)+ return+++def get_owner_id(issue: Issue) -> Optional[int]:+ """Get the owner ID of the issue"""++ if issue.updated_by_id:+ return issue.updated_by_id++ if issue.created_by_id:+ return issue.created_by_id++ # Find project admin as fallback+ project_member = ProjectMember.objects.filter(+ project_id=issue.project_id,+ role=20, # Admin role+ ).first()++ return project_member.member_id if project_member else None+++def get_related_data(issue_ids: List[UUID]) -> Dict:+ """Get related data for the given issue IDs"""++ cycle_issues = {+ ci.issue_id: ci.cycle_id+ for ci in CycleIssue.objects.filter(issue_id__in=issue_ids)+ }++ # Get assignees with proper grouping+ assignee_records = list(+ IssueAssignee.objects.filter(issue_id__in=issue_ids)+ .values_list("issue_id", "assignee_id")+ .order_by("issue_id")+ )+ assignees = {}+ for issue_id, group in groupby(assignee_records, key=lambda x: x[0]):+ assignees[issue_id] = [str(g[1]) for g in group]++ # Get labels with proper grouping+ label_records = list(+ IssueLabel.objects.filter(issue_id__in=issue_ids)+ .values_list("issue_id", "label_id")+ .order_by("issue_id")+ )+ labels = {}+ for issue_id, group in groupby(label_records, key=lambda x: x[0]):+ labels[issue_id] = [str(g[1]) for g in group]++ # Get modules with proper grouping+ module_records = list(+ ModuleIssue.objects.filter(issue_id__in=issue_ids)+ .values_list("issue_id", "module_id")+ .order_by("issue_id")+ )+ modules = {}+ for issue_id, group in groupby(module_records, key=lambda x: x[0]):+ modules[issue_id] = [str(g[1]) for g in group]++ # Get latest activities+ latest_activities = {}+ activities = IssueActivity.objects.filter(issue_id__in=issue_ids).order_by(+ "issue_id", "-created_at"+ )+ for issue_id, activities_group in groupby(activities, key=lambda x: x.issue_id):+ first_activity = next(activities_group, None)+ if first_activity:+ latest_activities[issue_id] = first_activity.id++ return {+ "cycle_issues": cycle_issues,+ "assignees": assignees,+ "labels": labels,+ "modules": modules,+ "activities": latest_activities,+ }+++def create_issue_version(issue: Issue, related_data: Dict) -> Optional[IssueVersion]:+ """Create IssueVersion object from the given issue and related data"""++ try:+ if not issue.workspace_id or not issue.project_id:+ logging.warning(+ f"Skipping issue {issue.id} - missing workspace_id or project_id"+ )+ return None++ owned_by_id = get_owner_id(issue)+ if owned_by_id is None:+ logging.warning(f"Skipping issue {issue.id} - missing owned_by")+ return None++ return IssueVersion(+ workspace_id=issue.workspace_id,+ project_id=issue.project_id,+ created_by_id=issue.created_by_id,+ updated_by_id=issue.updated_by_id,+ owned_by_id=owned_by_id,+ last_saved_at=timezone.now(),+ activity_id=related_data["activities"].get(issue.id),+ properties=getattr(issue, "properties", {}),+ meta=getattr(issue, "meta", {}),+ issue_id=issue.id,+ parent=issue.parent_id,+ state=issue.state_id,+ estimate_point=issue.estimate_point_id,+ name=issue.name,+ priority=issue.priority,+ start_date=issue.start_date,+ target_date=issue.target_date,+ assignees=related_data["assignees"].get(issue.id, []),+ sequence_id=issue.sequence_id,+ labels=related_data["labels"].get(issue.id, []),+ sort_order=issue.sort_order,+ completed_at=issue.completed_at,+ archived_at=issue.archived_at,+ is_draft=issue.is_draft,+ external_source=issue.external_source,+ external_id=issue.external_id,+ type=issue.type_id,+ cycle=related_data["cycle_issues"].get(issue.id),+ modules=related_data["modules"].get(issue.id, []),+ )+ except Exception as e:+ log_exception(e)+ return None+++@shared_task+def sync_issue_version(batch_size=5000, offset=0, countdown=300):+ """Task to create IssueVersion records for existing Issues in batches"""++ try:+ with transaction.atomic():+ base_query = Issue.objects+ total_issues_count = base_query.count()++ if total_issues_count == 0:+ return++ end_offset = min(offset + batch_size, total_issues_count)++ # Get issues batch with optimized queries+ issues_batch = list(+ base_query.order_by("created_at")+ .select_related("workspace", "project")+ .all()[offset:end_offset]+ )++ if not issues_batch:+ return++ # Get all related data in bulk+ issue_ids = [issue.id for issue in issues_batch]+ related_data = get_related_data(issue_ids)++ issue_versions = []+ for issue in issues_batch:+ version = create_issue_version(issue, related_data)+ if version:+ issue_versions.append(version)++ # Bulk create versions+ if issue_versions:+ IssueVersion.objects.bulk_create(issue_versions, batch_size=1000)++ # Schedule the next batch if there are more workspaces to process+ if end_offset < total_issues_count:+ sync_issue_version.apply_async(+ kwargs={+ "batch_size": batch_size,+ "offset": end_offset,+ "countdown": countdown,+ },+ countdown=countdown,+ )++ logging.info(f"Processed Issues: {end_offset}")+ return+ except Exception as e:+ log_exception(e)+ return+++@shared_task+def schedule_issue_version(batch_size=5000, countdown=300):+ sync_issue_version.delay(batch_size=int(batch_size), countdown=countdown)diff --git a/apiserver/plane/db/management/commands/sync_issue_description_version.py b/apiserver/plane/db/management/commands/sync_issue_description_version.pynew file mode 100644index 0000000000..7ff2fc3914--- /dev/null+++ b/apiserver/plane/db/management/commands/sync_issue_description_version.py@@ -0,0 +1,23 @@+# Django imports+from django.core.management.base import BaseCommand++# Module imports+from plane.bgtasks.issue_description_version_sync import (+ schedule_issue_description_version,+)+++class Command(BaseCommand):+ help = "Creates IssueDescriptionVersion records for existing Issues in batches"++ def handle(self, *args, **options):+ batch_size = input("Enter the batch size: ")+ batch_countdown = input("Enter the batch countdown: ")++ schedule_issue_description_version.delay(+ batch_size=batch_size, countdown=int(batch_countdown)+ )++ self.stdout.write(+ self.style.SUCCESS("Successfully created issue description version task")+ )diff --git a/apiserver/plane/db/management/commands/sync_issue_version.py b/apiserver/plane/db/management/commands/sync_issue_version.pynew file mode 100644index 0000000000..2b6632f261--- /dev/null+++ b/apiserver/plane/db/management/commands/sync_issue_version.py@@ -0,0 +1,19 @@+# Django imports+from django.core.management.base import BaseCommand++# Module imports+from plane.bgtasks.issue_version_sync import schedule_issue_version+++class Command(BaseCommand):+ help = "Creates IssueVersion records for existing Issues in batches"++ def handle(self, *args, **options):+ batch_size = input("Enter the batch size: ")+ batch_countdown = input("Enter the batch countdown: ")++ schedule_issue_version.delay(+ batch_size=batch_size, countdown=int(batch_countdown)+ )++ self.stdout.write(self.style.SUCCESS("Successfully created issue version task"))