π₯π₯π₯ Open-source Jira, Linear, Monday, and ClickUp alternative. Plane is a modern project management platform to manage tasks, sprints, docs, and triage.
TypeScript boardsbug-trackerdjangodockerganttissue-trackerjirajira-alternativekanbanlinearpostgresqlproduct-managementproject-managementproject-planningpythonreactredistypesciptvitework-management
[WEB-4346] chore: refactor copy_s3_object bg task (#7298)
Sangeethae313aee3df02
diff --git a/apps/api/plane/app/views/page/base.py b/apps/api/plane/app/views/page/base.pyindex 26e9223b82..cb9b0e0923 100644--- a/apps/api/plane/app/views/page/base.py+++ b/apps/api/plane/app/views/page/base.py@@ -40,7 +40,7 @@ from ..base import BaseAPIView, BaseViewSetfrom plane.bgtasks.page_transaction_task import page_transactionfrom plane.bgtasks.page_version_task import page_versionfrom plane.bgtasks.recent_visited_task import recent_visited_task-from plane.bgtasks.copy_s3_object import copy_s3_objects+from plane.bgtasks.copy_s3_object import copy_s3_objects_of_description_and_assetsdef unarchive_archive_page_and_descendants(page_id, archived_at):@@ -606,7 +606,7 @@ class PageDuplicateEndpoint(BaseAPIView):)# Copy the s3 objects uploaded in the page- copy_s3_objects.delay(+ copy_s3_objects_of_description_and_assets.delay(entity_name="PAGE",entity_identifier=page.id,project_id=project_id,diff --git a/apps/api/plane/bgtasks/copy_s3_object.py b/apps/api/plane/bgtasks/copy_s3_object.pyindex a92d7fe4e7..c8d9fc480a 100644--- a/apps/api/plane/bgtasks/copy_s3_object.py+++ b/apps/api/plane/bgtasks/copy_s3_object.py@@ -83,8 +83,52 @@ def sync_with_external_service(entity_name, description_html):return {}+def copy_assets(entity, entity_identifier, project_id, asset_ids, user_id):+ duplicated_assets = []+ workspace = entity.workspace+ storage = S3Storage()+ original_assets = FileAsset.objects.filter(+ workspace=workspace, project_id=project_id, id__in=asset_ids+ )++ for original_asset in original_assets:+ destination_key = (+ f"{workspace.id}/{uuid.uuid4().hex}-{original_asset.attributes.get('name')}"+ )+ duplicated_asset = FileAsset.objects.create(+ attributes={+ "name": original_asset.attributes.get("name"),+ "type": original_asset.attributes.get("type"),+ "size": original_asset.attributes.get("size"),+ },+ asset=destination_key,+ size=original_asset.size,+ workspace=workspace,+ created_by_id=user_id,+ entity_type=original_asset.entity_type,+ project_id=project_id,+ storage_metadata=original_asset.storage_metadata,+ **get_entity_id_field(original_asset.entity_type, entity_identifier),+ )+ storage.copy_object(original_asset.asset, destination_key)+ duplicated_assets.append(+ {+ "new_asset_id": str(duplicated_asset.id),+ "old_asset_id": str(original_asset.id),+ }+ )+ if duplicated_assets:+ FileAsset.objects.filter(+ pk__in=[item["new_asset_id"] for item in duplicated_assets]+ ).update(is_uploaded=True)++ return duplicated_assets++@shared_task-def copy_s3_objects(entity_name, entity_identifier, project_id, slug, user_id):+def copy_s3_objects_of_description_and_assets(+ entity_name, entity_identifier, project_id, slug, user_id+):"""Step 1: Extract asset ids from the description_html of the entityStep 2: Duplicate the assets@@ -100,53 +144,20 @@ def copy_s3_objects(entity_name, entity_identifier, project_id, slug, user_id):entity = model_class.objects.get(id=entity_identifier)asset_ids = extract_asset_ids(entity.description_html, "image-component")- duplicated_assets = []- workspace = entity.workspace- storage = S3Storage()- original_assets = FileAsset.objects.filter(- workspace=workspace, project_id=project_id, id__in=asset_ids+ duplicated_assets = copy_assets(+ entity, entity_identifier, project_id, asset_ids, user_id)- for original_asset in original_assets:- destination_key = f"{workspace.id}/{uuid.uuid4().hex}-{original_asset.attributes.get('name')}"- duplicated_asset = FileAsset.objects.create(- attributes={- "name": original_asset.attributes.get("name"),- "type": original_asset.attributes.get("type"),- "size": original_asset.attributes.get("size"),- },- asset=destination_key,- size=original_asset.size,- workspace=workspace,- created_by_id=user_id,- entity_type=original_asset.entity_type,- project_id=project_id,- storage_metadata=original_asset.storage_metadata,- **get_entity_id_field(original_asset.entity_type, entity_identifier),- )- storage.copy_object(original_asset.asset, destination_key)- duplicated_assets.append(- {- "new_asset_id": str(duplicated_asset.id),- "old_asset_id": str(original_asset.id),- }- )+ updated_html = update_description(entity, duplicated_assets, "image-component")++ external_data = sync_with_external_service(entity_name, updated_html)- if duplicated_assets:- FileAsset.objects.filter(- pk__in=[item["new_asset_id"] for item in duplicated_assets]- ).update(is_uploaded=True)- updated_html = update_description(- entity, duplicated_assets, "image-component"+ if external_data:+ entity.description = external_data.get("description")+ entity.description_binary = base64.b64decode(+ external_data.get("description_binary"))- external_data = sync_with_external_service(entity_name, updated_html)-- if external_data:- entity.description = external_data.get("description")- entity.description_binary = base64.b64decode(- external_data.get("description_binary")- )- entity.save()+ entity.save()returnexcept Exception as e:diff --git a/apps/api/plane/tests/unit/bg_tasks/test_copy_s3_objects.py b/apps/api/plane/tests/unit/bg_tasks/test_copy_s3_objects.pynew file mode 100644index 0000000000..bbb98e6b1c--- /dev/null+++ b/apps/api/plane/tests/unit/bg_tasks/test_copy_s3_objects.py@@ -0,0 +1,182 @@+import pytest+from plane.db.models import Project, ProjectMember, Issue, FileAsset+from unittest.mock import patch, MagicMock+from plane.bgtasks.copy_s3_object import (+ copy_s3_objects_of_description_and_assets,+ copy_assets,+)+import base64+++class TestCopyS3Objects:+ """Test the copy_s3_objects_of_description_and_assets function"""++ @pytest.fixture+ def project(self, create_user, workspace):+ project = Project.objects.create(+ name="Test Project", identifier="test-project", workspace=workspace+ )++ ProjectMember.objects.create(project=project, member=create_user)+ return project++ @pytest.fixture+ def issue(self, workspace, project):+ return Issue.objects.create(+ name="Test Issue",+ workspace=workspace,+ project_id=project.id,+ description_html=f'<div><image-component src="35e8b958-6ee5-43ce-ae56-fb0e776f421e"></image-component><image-component src="97988198-274f-4dfe-aa7a-4c0ffc684214"></image-component></div>',+ )++ @pytest.fixture+ def file_asset(self, workspace, project, issue):+ return FileAsset.objects.create(+ issue=issue,+ workspace=workspace,+ project=project,+ asset="workspace1/test-asset-1.jpg",+ attributes={+ "name": "test-asset-1.jpg",+ "size": 100,+ "type": "image/jpeg",+ },+ id="35e8b958-6ee5-43ce-ae56-fb0e776f421e",+ entity_type="ISSUE_DESCRIPTION",+ )++ @pytest.mark.django_db+ @patch("plane.bgtasks.copy_s3_object.S3Storage")+ def test_copy_s3_objects_of_description_and_assets(+ self, mock_s3_storage, create_user, workspace, project, issue, file_asset+ ):+ FileAsset.objects.create(+ issue=issue,+ workspace=workspace,+ project=project,+ asset="workspace1/test-asset-2.pdf",+ attributes={+ "name": "test-asset-2.pdf",+ "size": 100,+ "type": "application/pdf",+ },+ id="97988198-274f-4dfe-aa7a-4c0ffc684214",+ entity_type="ISSUE_DESCRIPTION",+ )++ issue.save()++ # Set up mock S3 storage+ mock_storage_instance = MagicMock()+ mock_s3_storage.return_value = mock_storage_instance++ # Mock the external service call to avoid actual HTTP requests+ with patch(+ "plane.bgtasks.copy_s3_object.sync_with_external_service"+ ) as mock_sync:+ mock_sync.return_value = {+ "description": "test description",+ "description_binary": base64.b64encode(b"test binary").decode(),+ }++ # Call the actual function (not .delay())+ copy_s3_objects_of_description_and_assets(+ "ISSUE", issue.id, project.id, "test-workspace", create_user.id+ )++ # Assert that copy_object was called for each asset+ assert mock_storage_instance.copy_object.call_count == 2++ # Get the updated issue and its new assets+ updated_issue = Issue.objects.get(id=issue.id)+ new_assets = FileAsset.objects.filter(+ issue=updated_issue,+ entity_type="ISSUE_DESCRIPTION",+ )++ # Verify new assets were created+ assert new_assets.count() == 4 # 2 original + 2 copied++ @pytest.mark.django_db+ @patch("plane.bgtasks.copy_s3_object.S3Storage")+ def test_copy_assets_successful(+ self, mock_s3_storage, workspace, project, issue, file_asset+ ):+ """Test successful copying of assets"""+ # Arrange+ mock_storage_instance = MagicMock()+ mock_s3_storage.return_value = mock_storage_instance++ # Act+ result = copy_assets(+ entity=issue,+ entity_identifier=issue.id,+ project_id=project.id,+ asset_ids=[file_asset.id],+ user_id=issue.created_by_id,+ )++ # Assert+ # Verify S3 copy was called+ mock_storage_instance.copy_object.assert_called_once()++ # Verify new asset was created+ assert len(result) == 1+ new_asset_id = result[0]["new_asset_id"]+ new_asset = FileAsset.objects.get(id=new_asset_id)++ # Verify asset properties were copied correctly+ assert new_asset.workspace == workspace+ assert new_asset.project_id == project.id+ assert new_asset.entity_type == file_asset.entity_type+ assert new_asset.attributes == file_asset.attributes+ assert new_asset.size == file_asset.size+ assert new_asset.is_uploaded is True++ @pytest.mark.django_db+ @patch("plane.bgtasks.copy_s3_object.S3Storage")+ def test_copy_assets_empty_asset_ids(+ self, mock_s3_storage, workspace, project, issue+ ):+ """Test copying with empty asset_ids list"""+ # Arrange+ mock_storage_instance = MagicMock()+ mock_s3_storage.return_value = mock_storage_instance++ # Act+ result = copy_assets(+ entity=issue,+ entity_identifier=issue.id,+ project_id=project.id,+ asset_ids=[],+ user_id=issue.created_by_id,+ )++ # Assert+ assert result == []+ mock_storage_instance.copy_object.assert_not_called()++ @pytest.mark.django_db+ @patch("plane.bgtasks.copy_s3_object.S3Storage")+ def test_copy_assets_nonexistent_asset(+ self, mock_s3_storage, workspace, project, issue+ ):+ """Test copying with non-existent asset ID"""+ # Arrange+ mock_storage_instance = MagicMock()+ mock_s3_storage.return_value = mock_storage_instance+ non_existent_id = "00000000-0000-0000-0000-000000000000"++ # Act+ result = copy_assets(+ entity=issue,+ entity_identifier=issue.id,+ project_id=project.id,+ asset_ids=[non_existent_id],+ user_id=issue.created_by_id,+ )++ # Assert+ assert result == []+ mock_storage_instance.copy_object.assert_not_called()