Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(dags): add a job to delete a team's embeddings for one product (#120521)

posthog[bot]ec1c09342f9e
diff --git a/docs/internal/clickhouse-deletion-coverage.md b/docs/internal/clickhouse-deletion-coverage.md
index 48182ab66ecd..82bc0e15d2ef 100644
--- a/docs/internal/clickhouse-deletion-coverage.md
+++ b/docs/internal/clickhouse-deletion-coverage.md
@@ -97,6 +97,8 @@ Leaving one out stays possible, and `PERSON_ID_REWRITE_EXEMPT` is where that dec
- `sharded_events_json` — all sweeps, on the events cluster, through patch parts instead of mutations (`uses_patch_parts`). Optional: only present after the native-JSON migration.
- `sharded_flag_evaluations` — person, team, queued-uuid, immediate and deferred event removal, and property removal of event and person properties (below). Immediate event removal with a HogQL predicate does not sweep it, and refuses while a matching row has no events copy. Deferred event removal with a predicate reaches the rows that have an events copy through their uuids, and the rest through the predicate compiled for this table (below). Property removal with a predicate does not sweep it. Optional.
- `sharded_posthog_document_embeddings_<model>` — event and team deletion, through `delete_event_documents`. An embedded document is keyed by the id of the thing it describes (`document_id`), and an Event deletion's key is that same id, so the pending dictionary is joined on `(team_id, Event, document_id)`. Every per-model table listed by the error tracking facade's `document_embedding_tables` is swept and counted.
+ A document keyed by another id is not reached by event deletion. The AI observability summaries use `<trace or generation id>::<job id>` under the `llm-analytics` product, so deleting the summary events leaves their embeddings until the table TTL.
+ To remove them, turn off AI data analysis for the organization first, so no new summaries arrive during the run. Then run `delete_team_product_documents_job` with the `team_id` and the `product`. It deletes only that product's documents for that team, from the model tables and the ingestion buffer. It fails when the team has no documents for that product (a misspelled name), when the deletes run past `timeout_seconds`, or when any document survives.
Native property-removal requests fail when the selected rows retain a requested permanent or temporary property, or a matching person `$set`/`$set_once` instruction.
They also fail when that property class has quarantine diagnostics, because malformed raw data cannot prove that the requested value is absent.
diff --git a/posthog/dags/deletes.py b/posthog/dags/deletes.py
index 616b3a132e6b..7d49223c18ca 100644
--- a/posthog/dags/deletes.py
+++ b/posthog/dags/deletes.py
@@ -1,4 +1,5 @@
import abc
+import time
import uuid
from collections import defaultdict
from collections.abc import Callable, Sequence
@@ -20,10 +21,12 @@ from posthog.clickhouse.adhoc_events_deletion import ADHOC_EVENTS_DELETION_TABLE
from posthog.clickhouse.cluster import (
ClickhouseCluster,
LightweightDeleteMutationRunner,
+ MutationNotFound,
MutationWaiter,
MutationWaiters,
NodeRole,
Query,
+ RetryPolicy,
Workload,
wait_for_mutations_on_shards,
)
@@ -61,7 +64,11 @@ from posthog.models.person.sql import (
PERSONS_TABLE,
)
-from products.error_tracking.backend.facade.api import DocumentEmbeddingTable, document_embedding_tables
+from products.error_tracking.backend.facade.api import (
+ DocumentEmbeddingTable,
+ document_embedding_tables,
+ document_embeddings_buffer_table,
+)
class DeleteConfig(dagster.Config):
@@ -123,6 +130,21 @@ class SweepTargetsConfig(dagster.Config):
)
+class TeamProductDocumentsDeleteConfig(dagster.Config):
+ team_id: int = pydantic.Field(description="Delete the embedding documents of this team.")
+ product: str = pydantic.Field(
+ min_length=1,
+ description="Delete only the documents this product wrote, e.g. llm-analytics. Other products "
+ "write to the same tables, so their documents stay.",
+ )
+ timeout_seconds: int = pydantic.Field(
+ default=6 * 60 * 60,
+ gt=0,
+ description="Fail the run if the deletes have not finished in this many seconds, including the wait "
+ "for other mutations on the tables to finish first.",
+ )
+
+
class MonthlyCleanupConfig(dagster.Config):
team_ids: list[int] = pydantic.Field(
default_factory=list,
@@ -1361,6 +1383,129 @@ def run_deletes_after_manual_trigger(context: dagster.RunStatusSensorContext) ->
return dagster.RunRequest(run_key=context.dagster_run.run_id, run_config=DELETES_RUN_CONFIG)
+_TEAM_PRODUCT_DOCUMENTS_PREDICATE = "team_id = %(team_id)s AND product = %(product)s"
+_MUTATION_POLL_SECONDS = 15.0
+_DOCUMENT_COUNT_MAX_EXECUTION_TIME = 300
+
+
+def _team_product_document_tables(cluster: ClickhouseCluster) -> list[str]:
+ """The sharded tables that can hold a document: every model table, and the buffer that feeds them."""
+ tables = [table.sharded_table for table in _present_document_embedding_tables(cluster)]
+ buffer = document_embeddings_buffer_table()
+ if _any_node_has(cluster, buffer):
+ tables.append(buffer)
+ return tables
+
+
+def _count_documents_per_table(
+ context: dagster.OpExecutionContext,
+ cluster: ClickhouseCluster,
+ tables: list[str],
+ parameters: dict[str, str | int],
+) -> dict[str, int]:
+ counts: dict[str, int] = {}
+ for table in tables:
+ count = _count_through(
+ context,
+ partial(_rows_per_shard, cluster),
+ f"{settings.CLICKHOUSE_DATABASE}.{table}",
+ parameters,
+ _DOCUMENT_COUNT_MAX_EXECUTION_TIME,
+ predicate=_TEAM_PRODUCT_DOCUMENTS_PREDICATE,
+ )
+ # A count that did not finish proves nothing either way, so it must not read as zero.
+ if count is None:
+ raise dagster.Failure(description=f"Could not count the documents in {table}")
+ counts[table] = count
+ return counts
+
+
+def _wait_until(waiters: MutationWaiters, deadline: float) -> Callable[[Client], None]:
+ def wait(client: Client) -> None:
+ while not waiters.is_done(client):
+ if time.monotonic() > deadline:
+ raise dagster.Failure(
+ description=f"Mutations {[w.mutation_ids for w in waiters.waiters]} did not finish in time"
+ )
+ time.sleep(_MUTATION_POLL_SECONDS)
+
+ return wait
+
+
+def delete_team_product_documents(
+ context: dagster.OpExecutionContext,
+ config: TeamProductDocumentsDeleteConfig,
+ cluster: dagster.ResourceParam[ClickhouseCluster],
+) -> None:
+ """Delete the embedding documents of one team for one product, and fail if any survive.
+
+ The event sweep matches a document to an event only when `document_id` is the event uuid. The AI
+ observability summaries use `<trace or generation id>::<job id>`, so an event deletion leaves
+ them in place until the table TTL drops them. This job removes them on request.
+ """
+ parameters: dict[str, str | int] = {"team_id": config.team_id, "product": config.product}
+ tables = _team_product_document_tables(cluster)
+
+ # The delete and the survivor count read the same product value, so a misspelled product
+ # deletes nothing and still reports zero survivors. Refuse it before deleting instead.
+ found = _count_documents_per_table(context, cluster, tables, parameters)
+ context.add_output_metadata({"documents_found": dagster.MetadataValue.json(found)})
+ if not any(found.values()):
+ products: set[str] = set()
+ for table in tables:
+ query = Query(
+ f"SELECT DISTINCT product FROM {settings.CLICKHOUSE_DATABASE}.{table} WHERE team_id = %(team_id)s",
+ parameters,
+ )
+ for rows in cluster.map_one_host_per_shard(query).result().values():
+ products.update(row[0] for row in rows)
+ raise dagster.Failure(
+ description=f"Team {config.team_id} has no documents for product {config.product!r}. "
+ f"Products with documents for this team: {sorted(products)}. Check the product name. "
+ "If an earlier run already deleted them, there is nothing left to do."
+ )
+
+ deadline = time.monotonic() + config.timeout_seconds
+ reuse_floor = _mutation_reuse_floor(cluster)
+ by_shard: dict[int, list[MutationWaiter]] = {}
+ for table in tables:
+ runner = LightweightDeleteMutationRunner(
+ table=table,
+ predicate=_TEAM_PRODUCT_DOCUMENTS_PREDICATE,
+ parameters=parameters,
+ reuse_since=reuse_floor,
+ # The runners start one after another, so each gets only what is left of the shared deadline.
+ # A zero timeout means no limit, so the floor keeps an expired deadline from lifting it.
+ capacity_timeout=max(1.0, deadline - time.monotonic()),
+ )
+ for host, mutation in cluster.map_one_host_per_shard(runner).result().items():
+ if host.shard_num is not None:
+ by_shard.setdefault(host.shard_num, []).append(mutation)
+
+ # Under replication lag a mutation can be briefly invisible on a replica, as in wait_for_mutations_on_shards.
+ retry_policy = RetryPolicy(max_attempts=3, delay=10.0, exceptions=(MutationNotFound,))
+ cluster.map_all_hosts_in_shards(
+ {
+ shard_num: retry_policy(_wait_until(MutationWaiters(waiters=waiters), deadline))
+ for shard_num, waiters in by_shard.items()
+ }
+ ).result()
+
+ surviving = _count_documents_per_table(context, cluster, tables, parameters)
+ context.add_output_metadata({"surviving_documents": dagster.MetadataValue.json(surviving)})
+ if any(surviving.values()):
+ raise dagster.Failure(
+ description=f"Documents of team {config.team_id} for product {config.product!r} survived the delete: {surviving}"
+ )
+
+
[email protected](tags={"owner": JobOwners.TEAM_CLICKHOUSE.value})
+def delete_team_product_documents_job():
+ """Delete the embedding documents of one team for one product, e.g. to complete a data deletion request."""
+ delete_team_product_documents()
+
+
@dagster.op(ins={"start_after": dagster.In(dagster.Nothing)})
def plan_old_events_cleanup(
context: dagster.OpExecutionContext,
diff --git a/posthog/dags/locations/clickhouse.py b/posthog/dags/locations/clickhouse.py
index 6b353f9364ab..2695a2f81861 100644
--- a/posthog/dags/locations/clickhouse.py
+++ b/posthog/dags/locations/clickhouse.py
@@ -39,6 +39,7 @@ defs = dagster.Definitions(
drop_materialized_column.drop_materialized_column,
deletes.deletes_job,
deletes.manual_deletes_job,
+ deletes.delete_team_product_documents_job,
deletes.monthly_retention_events_cleanup_job,
export_query_log_archive_to_s3.export_query_log_archive_to_s3,
backfill_materialized_column.backfill_materialized_column,
diff --git a/posthog/dags/tests/test_deletes.py b/posthog/dags/tests/test_deletes.py
index 8ccc2b097515..a4d99784af4f 100644
--- a/posthog/dags/tests/test_deletes.py
+++ b/posthog/dags/tests/test_deletes.py
@@ -35,6 +35,7 @@ from posthog.dags.deletes import (
_watermark_holders,
cleanup_old_events_by_partition,
delete_events,
+ delete_team_product_documents_job,
deletes_job,
ensure_no_concurrent_deletes_run,
manual_deletes_job,
@@ -599,6 +600,65 @@ def test_full_job_deletes_events_queued_in_postgres(cluster: ClickhouseCluster):
assert not AsyncDeletion.objects.filter(deletion_type=DeletionType.Event, delete_verified_at__isnull=True).exists()
+DOCUMENT_TABLES = (
+ "sharded_posthog_document_embeddings_buffer",
+ "distributed_posthog_document_embeddings_text_embedding_3_large_3072",
+)
+
+
+def test_delete_team_product_documents_job_keeps_other_products_and_teams(cluster: ClickhouseCluster):
+ timestamp = datetime.now().replace(microsecond=0)
+
+ # Ingestion writes to the buffer, and a materialized view copies each row into its model table,
+ # so every document below exists twice.
+ def insert_documents(client: Client) -> None:
+ client.execute(
+ """INSERT INTO sharded_posthog_document_embeddings_buffer
+ (team_id, product, document_type, model_name, rendering, document_id, timestamp, inserted_at, content, embedding) VALUES""",
+ [
+ (
+ team_id,
+ product,
+ "doc",
+ "text-embedding-3-large-3072",
+ "text",
+ f"trace-{team_id}-{product}::job",
+ timestamp,
+ timestamp,
+ "text",
+ [0.0] * 3072,
+ )
+ for team_id in (9100, 9101)
+ for product in ("llm-analytics", "error_tracking")
+ ],
+ )
+
+ def surviving_documents(table: str, client: Client) -> set[tuple[int, str]]:
+ result = client.execute(f"SELECT team_id, product FROM {table} WHERE team_id IN (9100, 9101)")
+ return {(row[0], row[1]) for row in result} if isinstance(result, list) else set()
+
+ cluster.any_host(insert_documents).result()
+
+ def run(product: str) -> bool:
+ return delete_team_product_documents_job.execute_in_process(
+ run_config={"ops": {"delete_team_product_documents": {"config": {"team_id": 9100, "product": product}}}},
+ resources={"cluster": cluster},
+ raise_on_error=False,
+ ).success
+
+ # A misspelled product matches nothing, so it must fail rather than report a clean delete.
+ assert not run("llm_analytics")
+ assert run("llm-analytics")
+
+ for table in DOCUMENT_TABLES:
+ assert cluster.any_host(partial(surviving_documents, table)).result() == {
+ (9100, "error_tracking"),
+ (9101, "llm-analytics"),
+ (9101, "error_tracking"),
+ }, table
+
+
def _mid_month(months_ago: int) -> datetime:
return (datetime.now() - relativedelta(months=months_ago)).replace(
day=15, hour=12, minute=0, second=0, microsecond=0
diff --git a/products/error_tracking/backend/facade/api.py b/products/error_tracking/backend/facade/api.py
index 05c6aa6c9446..96cbcb04ac8d 100644
--- a/products/error_tracking/backend/facade/api.py
+++ b/products/error_tracking/backend/facade/api.py
@@ -16,7 +16,7 @@ from posthog.event_usage import AnalyticsProps, groups
from products.access_control.backend.facade.api import valid_role_member_user_ids
from .. import logic, weekly_digest, weekly_digest_delivery
-from ..indexed_embedding import EMBEDDING_TABLES
+from ..indexed_embedding import DOCUMENT_EMBEDDINGS_BUFFER_SHARDED_TABLE, EMBEDDING_TABLES
from ..logic import external_references, github_external_references, rules
from ..models import (
ErrorTrackingIssue,
@@ -813,3 +813,8 @@ def document_embedding_tables() -> list[DocumentEmbeddingTable]:
)
for table in EMBEDDING_TABLES
]
+
+
+def document_embeddings_buffer_table() -> str:
+ """The sharded buffer that holds every new document, in each model, until its TTL drops it."""
+ return DOCUMENT_EMBEDDINGS_BUFFER_SHARDED_TABLE