:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(insights): remove file system entries when bulk deleting insights (#124042)
Matt Puaa39fc24d67c6
diff --git a/posthog/management/migration_analysis/atomic_false_acknowledged_migrations.txt b/posthog/management/migration_analysis/atomic_false_acknowledged_migrations.txtindex 6400428f91da..b2de685e7426 100644--- a/posthog/management/migration_analysis/atomic_false_acknowledged_migrations.txt+++ b/posthog/management/migration_analysis/atomic_false_acknowledged_migrations.txt@@ -57,3 +57,4 @@ ee.0065_migrate_dashboard_collaboratorsconversations.0076_tag_legacy_conversation_attachmentsposthog.1398_remove_userproductlist_statedata_modeling.0047_remove_datawarehousemodelpath_state+posthog.1414_remove_filesystem_rows_for_deleted_insightsdiff --git a/posthog/migrations/1414_remove_filesystem_rows_for_deleted_insights.py b/posthog/migrations/1414_remove_filesystem_rows_for_deleted_insights.pynew file mode 100644index 000000000000..330f1146bfc1--- /dev/null+++ b/posthog/migrations/1414_remove_filesystem_rows_for_deleted_insights.py@@ -0,0 +1,87 @@+import logging++from django.db import OperationalError, migrations, transaction++logger = logging.getLogger(__name__)++ID_CHUNK = 500_000+MIN_SPLIT = 1_000+TEAM_TIMEOUTS = ("15s", "60s", "300s")+++def _run(connection, timeout, fn):+ with transaction.atomic(using=connection.alias), connection.cursor() as cur:+ cur.execute(f"SET LOCAL statement_timeout = '{timeout}'")+ cur.execute("SET LOCAL lock_timeout = '2s'")+ return fn(cur)+++def _teams_in_range(connection, lo, hi, found):+ def query(cur):+ cur.execute(+ "SELECT DISTINCT team_id FROM posthog_dashboarditem WHERE deleted = true AND id > %s AND id <= %s",+ [lo, hi],+ )+ return [row[0] for row in cur.fetchall()]++ try:+ found.update(_run(connection, "60s", query))+ except OperationalError:+ if hi - lo <= MIN_SPLIT:+ logger.warning("1414: could not scan insight ids %s to %s", lo, hi)+ return+ mid = (lo + hi) // 2+ _teams_in_range(connection, lo, mid, found)+ _teams_in_range(connection, mid, hi, found)+++def _clean_team(connection, team_id, timeout):+ def work(cur):+ # The subquery runs inside the DELETE, so an insight restored after team discovery keeps its entry.+ for table in ("posthog_filesystem", "posthog_filesystemshortcut"):+ cur.execute(+ f"""+ DELETE FROM {table}+ WHERE team_id = %s AND type = 'insight'+ AND ref IN (SELECT short_id FROM posthog_dashboarditem WHERE team_id = %s AND deleted = true)+ """,+ [team_id, team_id],+ )++ _run(connection, timeout, work)+++def remove_rows(apps, schema_editor):+ connection = schema_editor.connection++ with connection.cursor() as cur:+ cur.execute("SELECT max(id) FROM posthog_dashboarditem")+ max_id = cur.fetchone()[0] or 0++ team_ids: set[int] = set()+ for lo in range(0, max_id, ID_CHUNK):+ _teams_in_range(connection, lo, min(lo + ID_CHUNK, max_id), team_ids)++ # A team that times out is retried with a longer timeout, never failed: the cleanup is optional.+ pending = sorted(team_ids)+ for timeout in TEAM_TIMEOUTS:+ still_pending = []+ for team_id in pending:+ try:+ _clean_team(connection, team_id, timeout)+ except OperationalError:+ still_pending.append(team_id)+ pending = still_pending+ if not pending:+ break++ for team_id in pending:+ logger.warning("1414: skipped team %s", team_id)+++class Migration(migrations.Migration):+ atomic = False++ dependencies = [("posthog", "1413_event_filter_pipeline_override")]++ operations = [migrations.RunPython(remove_rows, migrations.RunPython.noop)]diff --git a/posthog/migrations/max_migration.txt b/posthog/migrations/max_migration.txtindex 295058da0cb4..352f862057bc 100644--- a/posthog/migrations/max_migration.txt+++ b/posthog/migrations/max_migration.txt@@ -1 +1 @@-1413_event_filter_pipeline_override+1414_remove_filesystem_rows_for_deleted_insightsdiff --git a/posthog/models/file_system/file_system.py b/posthog/models/file_system/file_system.pyindex 7f501ce7071f..bbe8ef1008f4 100644--- a/posthog/models/file_system/file_system.py+++ b/posthog/models/file_system/file_system.py@@ -1,3 +1,4 @@+from collections.abc import Sequencefrom datetime import datetimefrom typing import Optional@@ -124,6 +125,12 @@ def delete_file(*, team: Team, file_type: str, ref: str, surface: str = DEFAULT_FileSystemShortcut.objects.filter(surface_q(surface), team=team, type=file_type, ref=ref).delete()+def delete_files(*, team: Team, file_type: str, refs: Sequence[str], surface: str = DEFAULT_SURFACE) -> None:+ count, _ = FileSystem.objects.filter(surface_q(surface), team=team, type=file_type, ref__in=refs).delete()+ if count > 0:+ FileSystemShortcut.objects.filter(surface_q(surface), team=team, type=file_type, ref__in=refs).delete()++def split_path(path: str) -> list[str]:segments = []current = ""diff --git a/posthog/models/file_system/file_system_mixin.py b/posthog/models/file_system/file_system_mixin.pyindex 095cd45f33cb..366482b6405e 100644--- a/posthog/models/file_system/file_system_mixin.py+++ b/posthog/models/file_system/file_system_mixin.py@@ -35,29 +35,7 @@ class FileSystemSyncMixin(Model):@receiver(post_save, sender=cls, weak=False)def _file_system_post_save(sender, instance: "FileSystemSyncMixin", created, **kwargs):- from posthog.models.file_system.file_system import create_or_update_file, delete_file-- fs_data = instance.get_file_system_representation()- try:- team = instance.team # type: ignore- if fs_data.should_delete:- delete_file(team=team, file_type=fs_data.type, ref=fs_data.ref, surface=fs_data.surface)- else:- create_or_update_file(- team=team,- base_folder=fs_data.base_folder,- name=fs_data.name,- file_type=fs_data.type,- ref=fs_data.ref,- href=fs_data.href,- meta=fs_data.meta,- created_at=fs_data.meta.get("created_at") or getattr(instance, "created_at", None),- created_by_id=fs_data.meta.get("created_by") or getattr(instance, "created_by_id", None),- surface=fs_data.surface,- )- except Exception as e:- # Don't raise exceptions in signals- capture_exception(e, additional_properties=dataclasses.asdict(fs_data))+ instance.sync_file_system()@receiver(post_delete, sender=cls, weak=False)def _file_system_post_delete(sender, instance: "FileSystemSyncMixin", **kwargs):@@ -75,6 +53,31 @@ class FileSystemSyncMixin(Model):# Don't raise exceptions in signalscapture_exception(e, additional_properties=dataclasses.asdict(fs_data))+ def sync_file_system(self) -> None:+ from posthog.models.file_system.file_system import create_or_update_file, delete_file++ fs_data = self.get_file_system_representation()+ try:+ team = self.team # type: ignore+ if fs_data.should_delete:+ delete_file(team=team, file_type=fs_data.type, ref=fs_data.ref, surface=fs_data.surface)+ else:+ create_or_update_file(+ team=team,+ base_folder=fs_data.base_folder,+ name=fs_data.name,+ file_type=fs_data.type,+ ref=fs_data.ref,+ href=fs_data.href,+ meta=fs_data.meta,+ created_at=fs_data.meta.get("created_at") or getattr(self, "created_at", None),+ created_by_id=fs_data.meta.get("created_by") or getattr(self, "created_by_id", None),+ surface=fs_data.surface,+ )+ except Exception as e:+ # Don't raise exceptions in signals+ capture_exception(e, additional_properties=dataclasses.asdict(fs_data))+@classmethoddef get_file_system_unfiled(cls, team: "Team", surface: str = DEFAULT_SURFACE) -> QuerySet[Any]:"""diff --git a/products/product_analytics/backend/presentation/insight.py b/products/product_analytics/backend/presentation/insight.pyindex 485c8d3d98f1..6ac90342edce 100644--- a/products/product_analytics/backend/presentation/insight.py+++ b/products/product_analytics/backend/presentation/insight.py@@ -1,5 +1,6 @@import jsonimport builtins+from collections import defaultdictfrom collections.abc import Sequencefrom datetime import datetime, timedeltafrom functools import lru_cache@@ -99,6 +100,7 @@ from posthog.models.activity_logging.activity_page import (activity_page_response,parse_activity_page_params,)+from posthog.models.file_system.file_system import delete_filesfrom posthog.models.organization import Organizationfrom posthog.models.tagged_item import TaggedItemfrom posthog.models.team.team import Team@@ -2750,7 +2752,7 @@ When set, the specified dashboard's filters and date range override will be applinsights = list(Insight.objects_including_soft_deleted.filter(id__in=ids, team__project_id=self.team.project_id, deleted=deleted- )+ ).select_related("team"))found_ids = {insight.id for insight in insights}skipped: list[dict[str, Any]] = [@@ -2809,6 +2811,13 @@ When set, the specified dashboard's filters and date range override will be applInsight.objects_including_soft_deleted.filter(id__in=insight_ids, team__project_id=self.team.project_id).update(deleted=True, last_modified_at=now(), last_modified_by=current_user)+ refs_by_team: dict[int, list[str]] = defaultdict(list)+ teams = {}+ for insight in insights:+ teams[insight.team_id] = insight.team+ refs_by_team[insight.team_id].append(insight.short_id)+ for team_id, refs in refs_by_team.items():+ delete_files(team=teams[team_id], file_type="insight", refs=refs)# Match InsightSerializer.update: hide the insights' tiles and remove linked alerts.hide_tiles_for_insights(insight_ids)delete_insight_alerts(insight_ids)@@ -2861,6 +2870,11 @@ When set, the specified dashboard's filters and date range override will be applInsight.objects_including_soft_deleted.filter(id__in=insight_ids, team__project_id=self.team.project_id).update(deleted=False, last_modified_at=now(), last_modified_by=current_user)+ restored_insights = Insight.objects.filter(+ id__in=insight_ids, team__project_id=self.team.project_id+ ).select_related("team")+ for insight in restored_insights:+ insight.sync_file_system()restore_tiles_for_insights(insight_ids, user_access_control=self.user_access_control)activity_log_entries: list[LogActivityEntry] = []diff --git a/products/product_analytics/backend/tests/api/test_insight.py b/products/product_analytics/backend/tests/api/test_insight.pyindex ec5428bedce7..86afceeefd05 100644--- a/products/product_analytics/backend/tests/api/test_insight.py+++ b/products/product_analytics/backend/tests/api/test_insight.py@@ -60,6 +60,7 @@ from posthog.exceptions import ClickHouseAtCapacity, ClickHouseQueryTimeOutfrom posthog.hogql_queries.query_runner import SHARED_FORCE_BLOCKING_STALENESS_WINDOW, ExecutionModefrom posthog.models import OrganizationMembership, SharingConfiguration, Team, Userfrom posthog.models.activity_logging.activity_log import ActivityLog+from posthog.models.file_system.file_system import FileSystemfrom posthog.models.project import Projectfrom posthog.query_scan.findings import build_warningfrom posthog.query_scan.flag import QueryScanFlag, QueryScanMode@@ -4474,6 +4475,9 @@ class TestInsightBulkDelete(ClickhouseTestMixin, APIBaseTest, QueryMatchingTest)def _bulk_delete(self, ids: list[int]) -> Any:return self.client.post(f"/api/environments/{self.team.id}/insights/bulk_delete/", {"ids": ids}, format="json")+ def _file_system_count(self, insight: Insight) -> int:+ return FileSystem.objects.filter(team=self.team, type="insight", ref=insight.short_id).count()+def _bulk_restore(self, ids: list[int]) -> Any:return self.client.post(f"/api/environments/{self.team.id}/insights/bulk_restore/", {"ids": ids}, format="json")@@ -4489,6 +4493,8 @@ class TestInsightBulkDelete(ClickhouseTestMixin, APIBaseTest, QueryMatchingTest)self.assertEqual(body["skipped"], [])self.assertTrue(Insight.objects_including_soft_deleted.get(id=one.id).deleted)self.assertTrue(Insight.objects_including_soft_deleted.get(id=two.id).deleted)+ self.assertEqual(self._file_system_count(one), 0)+ self.assertEqual(self._file_system_count(two), 0)def test_bulk_delete_hides_dashboard_tiles_and_removes_alerts(self) -> None:dashboard = Dashboard.objects.create(team=self.team, name="Dashboard")@@ -4554,6 +4560,7 @@ class TestInsightBulkDelete(ClickhouseTestMixin, APIBaseTest, QueryMatchingTest)self.assertEqual(self._bulk_delete([insight.id]).status_code, status.HTTP_200_OK)self.assertTrue(Insight.objects_including_soft_deleted.get(id=insight.id).deleted)self.assertTrue(DashboardTile.objects_including_soft_deleted.get(id=tile.id).deleted)+ self.assertEqual(self._file_system_count(insight), 0)response = self._bulk_restore([insight.id])@@ -4561,6 +4568,7 @@ class TestInsightBulkDelete(ClickhouseTestMixin, APIBaseTest, QueryMatchingTest)self.assertEqual([item["id"] for item in response.json()["restored"]], [insight.id])self.assertFalse(Insight.objects_including_soft_deleted.get(id=insight.id).deleted)self.assertFalse(DashboardTile.objects_including_soft_deleted.get(id=tile.id).deleted)+ self.assertEqual(self._file_system_count(insight), 1)def test_bulk_restore_skips_tiles_on_dashboards_the_user_cannot_edit(self) -> None:self.organization.available_product_features = [