Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(insights): remove file system entries when bulk deleting insights (#124042)

Matt Puaa39fc24d67c6
diff --git a/posthog/management/migration_analysis/atomic_false_acknowledged_migrations.txt b/posthog/management/migration_analysis/atomic_false_acknowledged_migrations.txt
index 6400428f91da..b2de685e7426 100644
--- a/posthog/management/migration_analysis/atomic_false_acknowledged_migrations.txt
+++ b/posthog/management/migration_analysis/atomic_false_acknowledged_migrations.txt
@@ -57,3 +57,4 @@ ee.0065_migrate_dashboard_collaborators
conversations.0076_tag_legacy_conversation_attachments
posthog.1398_remove_userproductlist_state
data_modeling.0047_remove_datawarehousemodelpath_state
+posthog.1414_remove_filesystem_rows_for_deleted_insights
diff --git a/posthog/migrations/1414_remove_filesystem_rows_for_deleted_insights.py b/posthog/migrations/1414_remove_filesystem_rows_for_deleted_insights.py
new file mode 100644
index 000000000000..330f1146bfc1
--- /dev/null
+++ b/posthog/migrations/1414_remove_filesystem_rows_for_deleted_insights.py
@@ -0,0 +1,87 @@
+import logging
+
+from django.db import OperationalError, migrations, transaction
+
+logger = logging.getLogger(__name__)
+
+ID_CHUNK = 500_000
+MIN_SPLIT = 1_000
+TEAM_TIMEOUTS = ("15s", "60s", "300s")
+
+
+def _run(connection, timeout, fn):
+ with transaction.atomic(using=connection.alias), connection.cursor() as cur:
+ cur.execute(f"SET LOCAL statement_timeout = '{timeout}'")
+ cur.execute("SET LOCAL lock_timeout = '2s'")
+ return fn(cur)
+
+
+def _teams_in_range(connection, lo, hi, found):
+ def query(cur):
+ cur.execute(
+ "SELECT DISTINCT team_id FROM posthog_dashboarditem WHERE deleted = true AND id > %s AND id <= %s",
+ [lo, hi],
+ )
+ return [row[0] for row in cur.fetchall()]
+
+ try:
+ found.update(_run(connection, "60s", query))
+ except OperationalError:
+ if hi - lo <= MIN_SPLIT:
+ logger.warning("1414: could not scan insight ids %s to %s", lo, hi)
+ return
+ mid = (lo + hi) // 2
+ _teams_in_range(connection, lo, mid, found)
+ _teams_in_range(connection, mid, hi, found)
+
+
+def _clean_team(connection, team_id, timeout):
+ def work(cur):
+ # The subquery runs inside the DELETE, so an insight restored after team discovery keeps its entry.
+ for table in ("posthog_filesystem", "posthog_filesystemshortcut"):
+ cur.execute(
+ f"""
+ DELETE FROM {table}
+ WHERE team_id = %s AND type = 'insight'
+ AND ref IN (SELECT short_id FROM posthog_dashboarditem WHERE team_id = %s AND deleted = true)
+ """,
+ [team_id, team_id],
+ )
+
+ _run(connection, timeout, work)
+
+
+def remove_rows(apps, schema_editor):
+ connection = schema_editor.connection
+
+ with connection.cursor() as cur:
+ cur.execute("SELECT max(id) FROM posthog_dashboarditem")
+ max_id = cur.fetchone()[0] or 0
+
+ team_ids: set[int] = set()
+ for lo in range(0, max_id, ID_CHUNK):
+ _teams_in_range(connection, lo, min(lo + ID_CHUNK, max_id), team_ids)
+
+ # A team that times out is retried with a longer timeout, never failed: the cleanup is optional.
+ pending = sorted(team_ids)
+ for timeout in TEAM_TIMEOUTS:
+ still_pending = []
+ for team_id in pending:
+ try:
+ _clean_team(connection, team_id, timeout)
+ except OperationalError:
+ still_pending.append(team_id)
+ pending = still_pending
+ if not pending:
+ break
+
+ for team_id in pending:
+ logger.warning("1414: skipped team %s", team_id)
+
+
+class Migration(migrations.Migration):
+ atomic = False
+
+ dependencies = [("posthog", "1413_event_filter_pipeline_override")]
+
+ operations = [migrations.RunPython(remove_rows, migrations.RunPython.noop)]
diff --git a/posthog/migrations/max_migration.txt b/posthog/migrations/max_migration.txt
index 295058da0cb4..352f862057bc 100644
--- a/posthog/migrations/max_migration.txt
+++ b/posthog/migrations/max_migration.txt
@@ -1 +1 @@
-1413_event_filter_pipeline_override
+1414_remove_filesystem_rows_for_deleted_insights
diff --git a/posthog/models/file_system/file_system.py b/posthog/models/file_system/file_system.py
index 7f501ce7071f..bbe8ef1008f4 100644
--- a/posthog/models/file_system/file_system.py
+++ b/posthog/models/file_system/file_system.py
@@ -1,3 +1,4 @@
+from collections.abc import Sequence
from datetime import datetime
from typing import Optional
@@ -124,6 +125,12 @@ def delete_file(*, team: Team, file_type: str, ref: str, surface: str = DEFAULT_
FileSystemShortcut.objects.filter(surface_q(surface), team=team, type=file_type, ref=ref).delete()
+def delete_files(*, team: Team, file_type: str, refs: Sequence[str], surface: str = DEFAULT_SURFACE) -> None:
+ count, _ = FileSystem.objects.filter(surface_q(surface), team=team, type=file_type, ref__in=refs).delete()
+ if count > 0:
+ FileSystemShortcut.objects.filter(surface_q(surface), team=team, type=file_type, ref__in=refs).delete()
+
+
def split_path(path: str) -> list[str]:
segments = []
current = ""
diff --git a/posthog/models/file_system/file_system_mixin.py b/posthog/models/file_system/file_system_mixin.py
index 095cd45f33cb..366482b6405e 100644
--- a/posthog/models/file_system/file_system_mixin.py
+++ b/posthog/models/file_system/file_system_mixin.py
@@ -35,29 +35,7 @@ class FileSystemSyncMixin(Model):
@receiver(post_save, sender=cls, weak=False)
def _file_system_post_save(sender, instance: "FileSystemSyncMixin", created, **kwargs):
- from posthog.models.file_system.file_system import create_or_update_file, delete_file
-
- fs_data = instance.get_file_system_representation()
- try:
- team = instance.team # type: ignore
- if fs_data.should_delete:
- delete_file(team=team, file_type=fs_data.type, ref=fs_data.ref, surface=fs_data.surface)
- else:
- create_or_update_file(
- team=team,
- base_folder=fs_data.base_folder,
- name=fs_data.name,
- file_type=fs_data.type,
- ref=fs_data.ref,
- href=fs_data.href,
- meta=fs_data.meta,
- created_at=fs_data.meta.get("created_at") or getattr(instance, "created_at", None),
- created_by_id=fs_data.meta.get("created_by") or getattr(instance, "created_by_id", None),
- surface=fs_data.surface,
- )
- except Exception as e:
- # Don't raise exceptions in signals
- capture_exception(e, additional_properties=dataclasses.asdict(fs_data))
+ instance.sync_file_system()
@receiver(post_delete, sender=cls, weak=False)
def _file_system_post_delete(sender, instance: "FileSystemSyncMixin", **kwargs):
@@ -75,6 +53,31 @@ class FileSystemSyncMixin(Model):
# Don't raise exceptions in signals
capture_exception(e, additional_properties=dataclasses.asdict(fs_data))
+ def sync_file_system(self) -> None:
+ from posthog.models.file_system.file_system import create_or_update_file, delete_file
+
+ fs_data = self.get_file_system_representation()
+ try:
+ team = self.team # type: ignore
+ if fs_data.should_delete:
+ delete_file(team=team, file_type=fs_data.type, ref=fs_data.ref, surface=fs_data.surface)
+ else:
+ create_or_update_file(
+ team=team,
+ base_folder=fs_data.base_folder,
+ name=fs_data.name,
+ file_type=fs_data.type,
+ ref=fs_data.ref,
+ href=fs_data.href,
+ meta=fs_data.meta,
+ created_at=fs_data.meta.get("created_at") or getattr(self, "created_at", None),
+ created_by_id=fs_data.meta.get("created_by") or getattr(self, "created_by_id", None),
+ surface=fs_data.surface,
+ )
+ except Exception as e:
+ # Don't raise exceptions in signals
+ capture_exception(e, additional_properties=dataclasses.asdict(fs_data))
+
@classmethod
def get_file_system_unfiled(cls, team: "Team", surface: str = DEFAULT_SURFACE) -> QuerySet[Any]:
"""
diff --git a/products/product_analytics/backend/presentation/insight.py b/products/product_analytics/backend/presentation/insight.py
index 485c8d3d98f1..6ac90342edce 100644
--- a/products/product_analytics/backend/presentation/insight.py
+++ b/products/product_analytics/backend/presentation/insight.py
@@ -1,5 +1,6 @@
import json
import builtins
+from collections import defaultdict
from collections.abc import Sequence
from datetime import datetime, timedelta
from functools import lru_cache
@@ -99,6 +100,7 @@ from posthog.models.activity_logging.activity_page import (
activity_page_response,
parse_activity_page_params,
)
+from posthog.models.file_system.file_system import delete_files
from posthog.models.organization import Organization
from posthog.models.tagged_item import TaggedItem
from posthog.models.team.team import Team
@@ -2750,7 +2752,7 @@ When set, the specified dashboard's filters and date range override will be appl
insights = list(
Insight.objects_including_soft_deleted.filter(
id__in=ids, team__project_id=self.team.project_id, deleted=deleted
- )
+ ).select_related("team")
)
found_ids = {insight.id for insight in insights}
skipped: list[dict[str, Any]] = [
@@ -2809,6 +2811,13 @@ When set, the specified dashboard's filters and date range override will be appl
Insight.objects_including_soft_deleted.filter(
id__in=insight_ids, team__project_id=self.team.project_id
).update(deleted=True, last_modified_at=now(), last_modified_by=current_user)
+ refs_by_team: dict[int, list[str]] = defaultdict(list)
+ teams = {}
+ for insight in insights:
+ teams[insight.team_id] = insight.team
+ refs_by_team[insight.team_id].append(insight.short_id)
+ for team_id, refs in refs_by_team.items():
+ delete_files(team=teams[team_id], file_type="insight", refs=refs)
# Match InsightSerializer.update: hide the insights' tiles and remove linked alerts.
hide_tiles_for_insights(insight_ids)
delete_insight_alerts(insight_ids)
@@ -2861,6 +2870,11 @@ When set, the specified dashboard's filters and date range override will be appl
Insight.objects_including_soft_deleted.filter(
id__in=insight_ids, team__project_id=self.team.project_id
).update(deleted=False, last_modified_at=now(), last_modified_by=current_user)
+ restored_insights = Insight.objects.filter(
+ id__in=insight_ids, team__project_id=self.team.project_id
+ ).select_related("team")
+ for insight in restored_insights:
+ insight.sync_file_system()
restore_tiles_for_insights(insight_ids, user_access_control=self.user_access_control)
activity_log_entries: list[LogActivityEntry] = []
diff --git a/products/product_analytics/backend/tests/api/test_insight.py b/products/product_analytics/backend/tests/api/test_insight.py
index ec5428bedce7..86afceeefd05 100644
--- a/products/product_analytics/backend/tests/api/test_insight.py
+++ b/products/product_analytics/backend/tests/api/test_insight.py
@@ -60,6 +60,7 @@ from posthog.exceptions import ClickHouseAtCapacity, ClickHouseQueryTimeOut
from posthog.hogql_queries.query_runner import SHARED_FORCE_BLOCKING_STALENESS_WINDOW, ExecutionMode
from posthog.models import OrganizationMembership, SharingConfiguration, Team, User
from posthog.models.activity_logging.activity_log import ActivityLog
+from posthog.models.file_system.file_system import FileSystem
from posthog.models.project import Project
from posthog.query_scan.findings import build_warning
from posthog.query_scan.flag import QueryScanFlag, QueryScanMode
@@ -4474,6 +4475,9 @@ class TestInsightBulkDelete(ClickhouseTestMixin, APIBaseTest, QueryMatchingTest)
def _bulk_delete(self, ids: list[int]) -> Any:
return self.client.post(f"/api/environments/{self.team.id}/insights/bulk_delete/", {"ids": ids}, format="json")
+ def _file_system_count(self, insight: Insight) -> int:
+ return FileSystem.objects.filter(team=self.team, type="insight", ref=insight.short_id).count()
+
def _bulk_restore(self, ids: list[int]) -> Any:
return self.client.post(f"/api/environments/{self.team.id}/insights/bulk_restore/", {"ids": ids}, format="json")
@@ -4489,6 +4493,8 @@ class TestInsightBulkDelete(ClickhouseTestMixin, APIBaseTest, QueryMatchingTest)
self.assertEqual(body["skipped"], [])
self.assertTrue(Insight.objects_including_soft_deleted.get(id=one.id).deleted)
self.assertTrue(Insight.objects_including_soft_deleted.get(id=two.id).deleted)
+ self.assertEqual(self._file_system_count(one), 0)
+ self.assertEqual(self._file_system_count(two), 0)
def test_bulk_delete_hides_dashboard_tiles_and_removes_alerts(self) -> None:
dashboard = Dashboard.objects.create(team=self.team, name="Dashboard")
@@ -4554,6 +4560,7 @@ class TestInsightBulkDelete(ClickhouseTestMixin, APIBaseTest, QueryMatchingTest)
self.assertEqual(self._bulk_delete([insight.id]).status_code, status.HTTP_200_OK)
self.assertTrue(Insight.objects_including_soft_deleted.get(id=insight.id).deleted)
self.assertTrue(DashboardTile.objects_including_soft_deleted.get(id=tile.id).deleted)
+ self.assertEqual(self._file_system_count(insight), 0)
response = self._bulk_restore([insight.id])
@@ -4561,6 +4568,7 @@ class TestInsightBulkDelete(ClickhouseTestMixin, APIBaseTest, QueryMatchingTest)
self.assertEqual([item["id"] for item in response.json()["restored"]], [insight.id])
self.assertFalse(Insight.objects_including_soft_deleted.get(id=insight.id).deleted)
self.assertFalse(DashboardTile.objects_including_soft_deleted.get(id=tile.id).deleted)
+ self.assertEqual(self._file_system_count(insight), 1)
def test_bulk_restore_skips_tiles_on_dashboards_the_user_cannot_edit(self) -> None:
self.organization.available_product_features = [