:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
import json
import types
import typing
import datetime as dt
from dataclasses import Field, asdict, dataclass, field, fields
from typing import Literal
from uuid import UUID
from django.conf import settings
import structlog
import temporalio
import temporalio.common
from asgiref.sync import async_to_sync
from temporalio.client import (
Client,
Schedule,
ScheduleActionStartWorkflow,
ScheduleCalendarSpec,
ScheduleIntervalSpec,
ScheduleOverlapPolicy,
SchedulePolicy,
ScheduleRange,
ScheduleSpec,
ScheduleState,
WorkflowHandle,
)
from posthog.dataclasses import frozen
from posthog.temporal.common.client import sync_connect
from posthog.temporal.common.schedule import (
a_pause_schedule,
create_schedule,
delete_schedule,
pause_schedule,
unpause_schedule,
update_schedule,
)
from products.batch_exports.backend.facade.contracts import AWSCredentials
from products.batch_exports.backend.facade.enums import IncrementalMode
from products.batch_exports.backend.models.batch_export import (
BatchExport,
BatchExportBackfill,
BatchExportDestination,
BatchExportOnDemand,
BatchExportRun,
)
from products.batch_exports.backend.temporal.workflow_metadata import build_static_summary
logger = structlog.get_logger(__name__)
def align_timestamp_to_interval(timestamp: dt.datetime, batch_export: BatchExport) -> dt.datetime:
"""Align a timestamp to the batch export's interval boundary.
For batch exports, intervals can have an offset from the default start time,
specified in the batch export's timezone. For example, a daily export might
run at 5am US/Pacific instead of midnight UTC.
Args:
timestamp: The timestamp to align (must be timezone-aware, typically UTC).
batch_export: The batch export configuration with interval, offset, and timezone.
Returns:
The start of the interval containing the timestamp (in UTC).
Examples:
Daily interval at 5am UTC:
- 2021-01-15 10:30:00 UTC aligns to 2021-01-15 05:00:00 UTC
- 2021-01-15 04:30:00 UTC aligns to 2021-01-14 05:00:00 UTC
Daily interval at 1am US/Pacific (PST = UTC-8 in winter):
- 2021-01-15 10:00:00 UTC (= 02:00 PST) aligns to 2021-01-15 09:00:00 UTC (= 01:00 PST)
- 2021-01-15 08:30:00 UTC (= 00:30 PST) aligns to 2021-01-14 09:00:00 UTC (= 01:00 PST prev day)
"""
interval = batch_export.interval
interval_seconds = int(batch_export.interval_time_delta.total_seconds())
# For hourly or sub-hourly intervals, timezone doesn't matter
if interval == "hour" or interval.startswith("every"):
ts = timestamp.timestamp()
aligned = (ts // interval_seconds) * interval_seconds
return dt.datetime.fromtimestamp(aligned, tz=dt.UTC)
# Convert timestamp to the batch export's timezone for alignment
tz = batch_export.timezone_info
local_timestamp = timestamp.astimezone(tz)
offset_hour = batch_export.offset_hour or 0
if interval == "day":
# Find the start of the current "day" (which starts at offset_hour in local time)
day_start = local_timestamp.replace(hour=offset_hour, minute=0, second=0, microsecond=0)
if day_start > local_timestamp:
day_start -= dt.timedelta(days=1)
return day_start.astimezone(dt.UTC)
elif interval == "week":
offset_day = batch_export.offset_day or 0
# Get current day of week (Monday=0, Sunday=6 in Python)
# But batch exports use Sunday=0, so we need to convert
python_weekday = local_timestamp.weekday() # Monday=0
batch_export_weekday = (python_weekday + 1) % 7 # Sunday=0
# Calculate days since the start of the week (at offset_day)
days_since_week_start = (batch_export_weekday - offset_day) % 7
# Find the start of the current "week"
week_start_date = local_timestamp.date() - dt.timedelta(days=days_since_week_start)
week_start = dt.datetime.combine(
week_start_date,
dt.time(hour=offset_hour, minute=0, second=0),
tzinfo=tz,
)
if week_start > local_timestamp:
week_start -= dt.timedelta(weeks=1)
return week_start.astimezone(dt.UTC)
else:
raise ValueError(f"Unknown interval: {interval}")
class BatchExportField(typing.TypedDict):
"""A field to be queried from ClickHouse.
Attributes:
expression: A ClickHouse SQL expression that declares the field required.
alias: An alias to apply to the expression (after an 'AS' keyword).
"""
expression: str
alias: str
class BatchExportSchema(typing.TypedDict):
hogql_query: typing.NotRequired[str]
fields: list[BatchExportField]
# HogQL binds these at any type; narrowing breaks Temporal input decoding.
values: dict[str, typing.Any]
@dataclass(frozen=False)
class BatchExportModel:
name: str
schema: BatchExportSchema | None
filters: list[dict[str, str | bool | float | list[str | bool | float] | None]] | None = None
hogql_query: str | None = None
# The user who last modified the batch export. This is used for validating custom HogQL queries. This is stored alongside the query, not looked up at runtime, so that an edit during a run cannot pair the old query with a new user.
user_id: int | None = None
# A dict, not `HogQLQueryModifiers`, because Temporal's default converter only decodes dataclasses and JSON types.
hogql_modifiers: dict[str, typing.Any] | None = None
class _BatchExportModelProtocol(typing.Protocol):
"""Read-only protocol covering all batch export models."""
@property
def name(self) -> str: ...
@property
def incremental_mode(self) -> IncrementalMode: ...
@property
def primary_key(self) -> tuple[str, ...]: ...
@property
def version_key(self) -> tuple[str, ...]: ...
@frozen
class BatchExportEventsModel:
# NOTE: In this and classes that follow, 'name' should really be a class
# variable. However, fields() and asdict() exclude attributes with ClassVar
# annotations. In the prescence of a type union, this would make it
# possible for Temporal to accidentaly deserialize into the wrong class, if
# we are not strict with type hints. So, to have the additional check 'name'
# provides, I would use a field(init=False) for name to approximate a class
# variable that is included in serialization/deserialization. However, this
# is not supported in Temporal==1.34.0. A fix has landed on main but has not
# released yet.
# TODO: On temporal==1.35.0 (or later) add init=False.
name: Literal["events"] = field(default="events")
# NOTE: primary_key and version_key are hardcoded for pre-defined models.
# This maintains backwards compatibility, and rejects any user input for
# models maintained by PostHog.
primary_key: tuple[Literal["uuid"]] = field(default=("uuid",))
version_key: tuple[Literal["timestamp"]] = field(default=("timestamp",))
incremental_mode: IncrementalMode = IncrementalMode.APPEND
filters: list[dict[str, str | bool | float | list[str | bool | float] | None]] = field(default_factory=list)
@frozen
class BatchExportEventsCustomModel:
name: Literal["events"] = field(default="events")
schema: BatchExportSchema
# NOTE: With a custom schema, we can't know if any of the key fields are
# present or not, so we enforce an empty key just to keep up with the
# protocol.
primary_key: tuple[str, ...] = field(default=())
version_key: tuple[str, ...] = field(default=())
incremental_mode: IncrementalMode = field(default=IncrementalMode.APPEND)
filters: list[dict[str, str | bool | float | list[str | bool | float] | None]] = field(default_factory=list)
def __post_init__(self) -> None:
if self.incremental_mode != IncrementalMode.APPEND:
# NOTE: Ideally, we would express this restriction using different
# types: an append model and a merge model as required. But Temporal
# encoding/decoding cannot reconstruct the types correctly, as
# Literal types just pass their SERIALIZED value as-is. So we
# have to validate. At least the validation happens at construction
# (or right after) so if you have a model instance, it's valid!
raise ValueError("Model is APPEND-only")
def _validate_merge_only_model(model: _BatchExportModelProtocol) -> None:
if model.incremental_mode != IncrementalMode.MERGE:
raise ValueError("Model is MERGE-only")
_validate_keys(model)
def _validate_keys(model: _BatchExportModelProtocol) -> None:
if not model.primary_key:
raise ValueError("MERGE requires a primary key")
@frozen
class BatchExportPersonsModel:
name: Literal["persons"] = field(default="persons")
primary_key: tuple[Literal["team_id"], Literal["distinct_id"]] = field(default=("team_id", "distinct_id"))
version_key: tuple[Literal["person_version"], Literal["person_distinct_id_version"]] = field(
default=(
"person_version",
"person_distinct_id_version",
),
)
# NOTE: I am not sure if appending makes sense for persons or sessions, so
# this is hardcoded to merge. Happy to be corrected!
incremental_mode: IncrementalMode = field(default=IncrementalMode.MERGE)
def __post_init__(self) -> None:
_validate_merge_only_model(self)
@frozen
class BatchExportSessionsModel:
name: Literal["sessions"] = field(default="sessions")
primary_key: tuple[Literal["team_id"], Literal["session_id"]] = field(default=("team_id", "session_id"))
version_key: tuple[Literal["end_timestamp"]] = field(default=("end_timestamp",))
incremental_mode: IncrementalMode = field(default=IncrementalMode.MERGE)
def __post_init__(self):
_validate_merge_only_model(self)
@frozen
class BatchExportHogQLModel:
name: Literal["hogql"] = field(default="hogql")
query: str
user_id: int
modifiers: dict[str, typing.Any]
primary_key: tuple[str, ...] = ()
version_key: tuple[str, ...] = ()
incremental_mode: IncrementalMode = IncrementalMode.APPEND
def __post_init__(self) -> None:
if self.incremental_mode == IncrementalMode.MERGE:
_validate_keys(self)
AnyBatchExportModel = (
BatchExportEventsCustomModel
| BatchExportEventsModel
| BatchExportPersonsModel
| BatchExportSessionsModel
| BatchExportHogQLModel
)
BatchExportModelInput = AnyBatchExportModel | BatchExportModel
def migrate_batch_export_model(old: BatchExportModelInput | None, /) -> AnyBatchExportModel:
if old is None:
return BatchExportEventsModel()
if isinstance(old, AnyBatchExportModel):
return old
match old.name:
case "events":
if old.schema is not None:
return BatchExportEventsCustomModel(schema=old.schema, filters=old.filters or [])
return BatchExportEventsModel(filters=old.filters or [])
case "persons":
return BatchExportPersonsModel()
case "sessions":
return BatchExportSessionsModel()
case "hogql":
# These two should be set in the model, but we must be strict.
if old.user_id is None:
raise ValueError("HogQL model requires user_id")
if old.hogql_query is None:
raise ValueError("HogQL model requires a query")
return BatchExportHogQLModel(
query=old.hogql_query,
user_id=old.user_id,
modifiers=old.hogql_modifiers if old.hogql_modifiers is not None else {},
)
case s:
raise ValueError(f"Invalid model name: '{s}'")
def get_batch_export_model(batch_export: BatchExport | BatchExportOnDemand) -> AnyBatchExportModel:
match batch_export.model or "events":
case "events":
schema = batch_export.schema if isinstance(batch_export, BatchExport) else None
if schema is not None:
return BatchExportEventsCustomModel(schema=schema, filters=batch_export.filters or [])
return BatchExportEventsModel(
filters=batch_export.filters or [],
incremental_mode=IncrementalMode(batch_export.incremental_mode),
)
case "persons":
return BatchExportPersonsModel()
case "sessions":
return BatchExportSessionsModel()
case "hogql":
source = batch_export.source
if batch_export.last_modified_by_id is None:
raise ValueError("HogQL model requires user_id")
if source is None or source.hogql_query is None:
raise ValueError("HogQL model requires a query")
return BatchExportHogQLModel(
query=source.hogql_query,
user_id=batch_export.last_modified_by_id,
modifiers=source.hogql_modifiers or {},
incremental_mode=IncrementalMode(batch_export.incremental_mode),
primary_key=tuple(source.primary_key or ()),
version_key=tuple(source.version_key or ()),
)
case name:
raise ValueError(f"Invalid model name: '{name}'")
@dataclass
class BackfillDetails:
backfill_id: str | None
start_at: str | None
end_at: str | None
is_earliest_backfill: bool = False
def _field_is_type(f: Field, target: type) -> bool:
"""Check if a dataclass field's type is or contains the target type.
Handles plain types (int, bool), PEP 604 unions (int | None),
and typing.Optional / typing.Union.
"""
if f.type is target:
return True
if isinstance(f.type, types.UnionType):
return target in f.type.__args__
if typing.get_origin(f.type) is typing.Union:
return target in typing.get_args(f.type)
return False
@dataclass(frozen=False, kw_only=True)
class BaseBatchExportInputs:
"""Base class for all batch export inputs containing common fields.
Attributes:
batch_export_id: The ID of the parent BatchExport.
team_id: The ID of the team that contains the BatchExport whose data we are exporting.
interval: The range of data we are exporting.
data_interval_end: For manual runs, the end date of the batch. This should be set to `None` for regularly
scheduled runs and for backfills.
integration_id: The ID of the integration that contains the credentials for the destination.
"""
batch_export_id: str
team_id: int
interval: str = "hour"
timezone: str = "UTC"
data_interval_end: str | None = None
exclude_events: list[str] | None = None
include_events: list[str] | None = None
# TODO: Remove 'is_backfill' and 'is_earliest_backfill' after ensuring no existing backfills are running
is_backfill: bool = False
is_earliest_backfill: bool = False
backfill_details: BackfillDetails | None = None
batch_export_model: BatchExportModelInput | None = None
integration_id: int | None = None
def __post_init__(self) -> None:
"""Coerce string values to their declared types.
EncryptedJSONField may not preserve Python types, so fields can
arrive as strings (e.g. "true" instead of True, "5432" instead of 5432).
"""
for f in fields(self):
value = getattr(self, f.name)
if value is None:
continue
if _field_is_type(f, bool):
if isinstance(value, str):
setattr(self, f.name, value.lower() == "true")
elif _field_is_type(f, int):
if isinstance(value, str):
setattr(self, f.name, int(value))
def get_is_backfill(self) -> bool:
"""Needed for backwards compatibility with existing batch exports.
TODO: remove once all existing backfills are finished.
"""
# to check status of migration
if self.is_backfill and not self.backfill_details:
logger.info(
"Backfill inputs migration: BatchExport %s has is_backfill set to True but no backfill_details",
self.batch_export_id,
)
if self.backfill_details is not None:
return True
return self.is_backfill
def get_is_earliest_backfill(self) -> bool:
"""Needed for backwards compatibility with existing batch exports.
TODO: remove once all existing backfills are finished.
"""
if self.backfill_details is not None:
return self.backfill_details.is_earliest_backfill
return self.is_earliest_backfill
@dataclass(frozen=False, kw_only=True)
class S3BatchExportInputs(BaseBatchExportInputs):
"""Inputs for S3 export workflow.
This is the input dataclass the `s3-export` Temporal workflow declares, and it holds the
union of every S3-family destination's fields. No destination type dispatches with it;
each type dispatches with its own narrower dataclass. Temporal's data converter serializes
that dataclass to JSON, and the worker deserializes it into this one. A field the narrower
dataclass does not declare therefore takes the default declared here.
Credentials and the provider endpoint are never carried here: the activity resolves them from
the linked Integration at run time (see `integration_id`).
Attributes:
bucket_name: The S3 bucket we are exporting to.
region: The AWS region where the bucket is located.
prefix: A prefix for the file name to be created in S3.
compression: Compression algorithm to apply to exported files (e.g. "gzip", "brotli"), or None.
file_format: File format of exported objects (e.g. "JSONLines", "Parquet"). Defaults to JSONLines.
max_file_size_mb: The maximum file size in MB for each file to be uploaded.
encryption: Server-side encryption algorithm to apply (e.g. "AES256", "aws:kms"), or None. AWS-only.
kms_key_id: KMS key id to use when `encryption == "aws:kms"`, or None. AWS-only.
use_virtual_style_addressing: Whether to use virtual-hosted-style
addressing rather than path-style. None for AWS.
legacy_parquet_extension: Whether Parquet files keep the compression codec in their
extension, e.g. `.parquet.zst` rather than `.parquet`. Defaults to True because a
schedule created before this field existed has no value for it, so the missing field
decodes to this default and the export keeps the names it already writes.
"""
bucket_name: str
region: str
prefix: str
compression: str | None = None
file_format: str = "JSONLines"
max_file_size_mb: int | None = None
encryption: str | None = None
kms_key_id: str | None = None
use_virtual_style_addressing: bool = False
legacy_parquet_extension: bool = True
@dataclass(frozen=False, kw_only=True)
class S3FamilyBaseInputs(BaseBatchExportInputs):
"""Shared fields for every S3-family destination.
Per-destination dataclasses extend this with provider-specific fields. Credentials are never
carried here: the activity resolves them from the linked Integration at run time (see
`integration_id`).
"""
bucket_name: str
region: str
prefix: str
compression: str | None = None
file_format: str = "JSONLines"
max_file_size_mb: int | None = None
legacy_parquet_extension: bool = True
@dataclass(kw_only=True)
class AwsS3BatchExportInputs(S3FamilyBaseInputs):
"""Inputs for an AWS S3 batch export."""
encryption: str | None = None
kms_key_id: str | None = None
@dataclass(frozen=False, kw_only=True)
class S3CompatibleBatchExportInputs(S3FamilyBaseInputs):
"""Inputs for a non-AWS S3-compatible batch export.
Covers providers like DigitalOcean Spaces, Cloudflare R2, Hetzner, OVH, Backblaze, etc.
`endpoint_url` lives on the linked Integration alongside the credentials, so it is resolved at
run time rather than configured per export.
"""
use_virtual_style_addressing: bool = False
@dataclass(frozen=False, kw_only=True)
class FileDownloadBatchExportInputs(BaseBatchExportInputs):
"""Inputs for a file download batch export workflow.
Attributes:
data_interval_start: Lower bound for the batch export.
data_interval_end: Upper bound for the batch export.
file_format: File format to use when exporting files. Same as S3.
max_file_size_mb: The maximum file size in MB for each file to be uploaded. Same
as S3.
compression: Compression algorithm, if any. Same as S3.
expires_in: Number of seconds to expire the download URLs.
"""
data_interval_start: str | None = None
batch_export_run_id: UUID | None = None
file_format: str = "Parquet"
max_file_size_mb: int | None = None
compression: str | None = None
expires_in_seconds: int = 3600
main_activity_timeout_seconds: int | None = None
stage_activity_timeout_seconds: int | None = None
@dataclass(frozen=False, kw_only=True)
class SnowflakeBatchExportInputs(BaseBatchExportInputs):
"""Inputs for Snowflake export workflow.
Credentials are never carried here: the activity resolves them from the linked Integration at
run time (see `integration_id`).
"""
database: str
warehouse: str
schema: str
table_name: str = "events"
role: str | None = None
@dataclass(frozen=False, kw_only=True)
class PostgresBatchExportInputs(BaseBatchExportInputs):
"""Inputs for Postgres export workflow."""
database: str
schema: str = "public"
table_name: str = "events"
user: str | None = None
host: str | None = None
port: int | None = 5432
password: str | None = field(default=None, repr=False)
has_self_signed_cert: bool = False
IAMRole = str
IntegrationID = int
@frozen
class RedshiftCopyInputs:
s3_bucket: str
region_name: str
s3_key_prefix: str
# Authorization for Redshift to COPY data from the bucket: an IAM role ARN,
# inline credentials, or the id of a team-scoped aws-s3 integration.
authorization: IAMRole | AWSCredentials | IntegrationID
# Credentials used to stage files in the S3 bucket: inline credentials or
# the id of a team-scoped aws-s3 integration.
bucket_credentials: AWSCredentials | IntegrationID
@dataclass(frozen=False, kw_only=True)
class RedshiftBatchExportInputs(BaseBatchExportInputs):
"""Inputs for Redshift export workflow."""
database: str
user: str | None = None
password: str | None = field(default=None, repr=False)
host: str | None = None
schema: str = "public"
table_name: str = "events"
port: int = 5439
properties_data_type: str = "varchar"
mode: typing.Literal["COPY", "INSERT"] = "INSERT"
copy_inputs: RedshiftCopyInputs | None = None
def __post_init__(self) -> None:
super().__post_init__()
if (
self.mode == "COPY"
and self.copy_inputs is not None
and not isinstance(self.copy_inputs, RedshiftCopyInputs)
):
if isinstance(self.copy_inputs, str | bytes | bytearray): # type: ignore
raw_inputs = json.loads(self.copy_inputs)
elif isinstance(self.copy_inputs, dict):
raw_inputs = self.copy_inputs
else:
raise TypeError(f"Invalid type for copy inputs: '{type(self.copy_inputs)}'")
# `BatchExportDestination.config` is an `EncryptedJSONField`, which stringifies scalar
# leaves on the decrypt round trip: an integration id saved as an int reads back as a
# numeric string, so both forms must be accepted here.
raw_bucket_credentials = raw_inputs["bucket_credentials"]
bucket_credentials: AWSCredentials | IntegrationID
if isinstance(raw_bucket_credentials, IntegrationID):
bucket_credentials = raw_bucket_credentials
elif isinstance(raw_bucket_credentials, str) and raw_bucket_credentials.isdigit():
bucket_credentials = IntegrationID(raw_bucket_credentials)
else:
bucket_credentials = AWSCredentials(
aws_access_key_id=raw_bucket_credentials["aws_access_key_id"],
aws_secret_access_key=raw_bucket_credentials["aws_secret_access_key"],
)
raw_authorization = raw_inputs["authorization"]
authorization: IAMRole | AWSCredentials | IntegrationID
if isinstance(raw_authorization, IntegrationID):
authorization = raw_authorization
elif isinstance(raw_authorization, str):
authorization = IntegrationID(raw_authorization) if raw_authorization.isdigit() else raw_authorization
else:
authorization = AWSCredentials(
aws_access_key_id=raw_authorization["aws_access_key_id"],
aws_secret_access_key=raw_authorization["aws_secret_access_key"],
)
self.copy_inputs = RedshiftCopyInputs(
s3_bucket=raw_inputs["s3_bucket"],
s3_key_prefix=raw_inputs.get("s3_key_prefix", "/"),
region_name=raw_inputs["region_name"],
authorization=authorization,
bucket_credentials=bucket_credentials,
)
@dataclass(frozen=False, kw_only=True)
class BigQueryBatchExportInputs(BaseBatchExportInputs):
"""Inputs for BigQuery export workflow."""
dataset_id: str
table_id: str = "events"
project_id: str | None = None
private_key: str | None = field(default=None, repr=False)
private_key_id: str | None = None
token_uri: str | None = None
client_email: str | None = None
use_json_type: bool = False
@dataclass(kw_only=True)
class DatabricksBatchExportInputs(BaseBatchExportInputs):
"""Inputs for Databricks export workflow.
NOTE: we store config related to the Databricks instance in the integration model instead.
(including sensitive config such as client ID and client secret)
"""
http_path: str
catalog: str
schema: str
table_name: str
use_variant_type: bool = True
use_automatic_schema_evolution: bool = True
@dataclass(frozen=False, kw_only=True)
class AzureBlobBatchExportInputs(BaseBatchExportInputs):
"""Inputs for Azure Blob Storage export workflow.
NOTE: Connection credentials are stored in the Integration model.
The `integration_id` field from `BaseBatchExportInputs` is used to fetch them.
"""
container_name: str
prefix: str = ""
compression: str | None = None
file_format: str = "JSONLines"
max_file_size_mb: int | None = None
legacy_parquet_extension: bool = True
@dataclass(kw_only=True)
class WorkflowsBatchExportInputs(BaseBatchExportInputs):
"""Inputs for Workflows export workflow.
NOTE: "Workflows" in this context refers to PostHog Workflows. PostHog Workflows
are not related to Temporal Workflows.
"""
hog_function_id: str
@dataclass(kw_only=True)
class HttpBatchExportInputs(BaseBatchExportInputs):
"""Inputs for Http export workflow."""
url: str
token: str
@dataclass(kw_only=True)
class NoOpInputs(BaseBatchExportInputs):
"""NoOp Workflow is used for testing, it takes a single argument to echo back."""
arg: str = ""
DESTINATION_WORKFLOWS = {
"AwsS3": ("s3-export", AwsS3BatchExportInputs),
"AzureBlob": ("azure-blob-export", AzureBlobBatchExportInputs),
"BigQuery": ("bigquery-export", BigQueryBatchExportInputs),
"Databricks": ("databricks-export", DatabricksBatchExportInputs),
"FileDownload": ("file-download-export", FileDownloadBatchExportInputs),
"HTTP": ("http-export", HttpBatchExportInputs),
"NoOp": ("no-op", NoOpInputs),
"Postgres": ("postgres-export", PostgresBatchExportInputs),
"Redshift": ("redshift-export", RedshiftBatchExportInputs),
"S3Compatible": ("s3-export", S3CompatibleBatchExportInputs),
"Snowflake": ("snowflake-export", SnowflakeBatchExportInputs),
"Workflows": ("workflows-export", WorkflowsBatchExportInputs),
}
def coerce_config_to_declared_types(destination_type: str, config: dict[str, typing.Any]) -> dict[str, typing.Any]:
"""Restore stringified config values to the types declared on the destination's workflow inputs.
EncryptedJSONField stringifies values on write, so bool/int config fields read back as strings.
This function mirrors the coercion the worker applies in BaseBatchExportInputs.__post_init__, so
API responses return correct JSON types.
"""
if destination_type not in DESTINATION_WORKFLOWS:
return config
_, workflow_inputs = DESTINATION_WORKFLOWS[destination_type]
field_by_name = {f.name: f for f in fields(workflow_inputs)}
coerced: dict[str, typing.Any] = {}
for key, value in config.items():
field = field_by_name.get(key)
if field is not None and isinstance(value, str):
if _field_is_type(field, bool):
value = value.lower() == "true"
elif _field_is_type(field, int):
try:
value = int(value)
except ValueError:
pass
coerced[key] = value
if destination_type == "Redshift" and isinstance(coerced.get("copy_inputs"), dict):
copy_inputs = {**coerced["copy_inputs"]}
# Integration ids nested in copy_inputs also read back as digit strings.
for key in ("authorization", "bucket_credentials"):
value = copy_inputs.get(key)
if isinstance(value, str) and value.isdigit():
copy_inputs[key] = int(value)
coerced["copy_inputs"] = copy_inputs
return coerced
class BatchExportServiceError(Exception):
"""Base class for BatchExport service exceptions."""
class BatchExportIdError(BatchExportServiceError):
"""Exception raised when an id for a BatchExport is not found."""
def __init__(self, batch_export_id: str):
super().__init__(f"No BatchExport found with ID: '{batch_export_id}'")
class BatchExportServiceRPCError(BatchExportServiceError):
"""Exception raised when the underlying Temporal RPC fails."""
class BatchExportWithNoEndNotAllowedError(BatchExportServiceError):
"""Exception raised when a BatchExport without an end_at is not allowed for a given destination."""
class BatchExportServiceScheduleNotFound(BatchExportServiceRPCError):
"""Exception raised when the underlying Temporal RPC fails because a schedule was not found."""
def __init__(self, schedule_id: str):
self.schedule_id = schedule_id
super().__init__(f"The Temporal Schedule {schedule_id} was not found (maybe it was deleted?)")
def _get_batch_export_for_schedule_change(batch_export_id: str, team_id: int | None) -> BatchExport:
"""Load the export that a pause or unpause changes.
With ``team_id``, only a live export of that team matches, so a request cannot reach another
team's export. Internal callers (Temporal, deletes) pass no team and match any export.
"""
# nosemgrep: idor-lookup-without-team (scoped below when the caller passes team_id)
batch_exports = BatchExport.objects.filter(id=batch_export_id)
if team_id is not None:
batch_exports = batch_exports.filter(team_id=team_id, deleted=False)
try:
return batch_exports.get()
except BatchExport.DoesNotExist:
raise BatchExportIdError(batch_export_id)
def pause_batch_export(
temporal: Client, batch_export_id: str, note: str | None = None, *, team_id: int | None = None
) -> bool:
"""Pause this BatchExport.
We pass the call to the underlying Temporal Schedule.
Returns:
`True` if the batch export was paused, `False` if it was already paused.
Raises:
BatchExportIdError: If no export matches ``batch_export_id`` (and ``team_id``, when given).
"""
batch_export = _get_batch_export_for_schedule_change(batch_export_id, team_id)
if batch_export.paused is True:
return False
try:
pause_schedule(temporal, schedule_id=batch_export_id, note=note)
except Exception as exc:
raise BatchExportServiceRPCError(f"BatchExport {batch_export_id} could not be paused") from exc
batch_export.paused = True
batch_export.last_paused_at = dt.datetime.now(dt.UTC)
batch_export.save()
return True
async def apause_batch_export(temporal: Client, batch_export_id: str, note: str | None = None) -> bool:
"""Pause this BatchExport.
We pass the call to the underlying Temporal Schedule.
Returns:
`True` if the batch export was paused, `False` if it was already paused.
"""
try:
# nosemgrep: idor-lookup-without-team (internal service called from Temporal, not user-facing)
batch_export = await BatchExport.objects.aget(id=batch_export_id)
except BatchExport.DoesNotExist:
raise BatchExportIdError(batch_export_id)
if batch_export.paused is True:
return False
try:
await a_pause_schedule(temporal, schedule_id=batch_export_id, note=note)
except Exception as exc:
raise BatchExportServiceRPCError(f"BatchExport {batch_export_id} could not be paused") from exc
batch_export.paused = True
batch_export.last_paused_at = dt.datetime.now(dt.UTC)
await batch_export.asave()
return True
def unpause_batch_export(
temporal: Client,
batch_export_id: str,
note: str | None = None,
backfill: bool = False,
*,
team_id: int | None = None,
) -> None:
"""Unpause this BatchExport.
We pass the call to the underlying Temporal Schedule. Additionally, we can trigger a backfill
to backfill runs missed while paused.
Args:
temporal: The Temporal client to execute calls.
batch_export_id: The ID of the BatchExport to unpause.
note: An optional note to include in the Schedule when unpausing.
backfill: If True, a backfill will be triggered since the BatchExport's last_paused_at.
team_id: If set, only a live export of this team matches.
Raises:
BatchExportIdError: If no export matches ``batch_export_id`` (and ``team_id``, when given).
"""
batch_export = _get_batch_export_for_schedule_change(batch_export_id, team_id)
if batch_export.paused is False:
return
try:
unpause_schedule(temporal, schedule_id=batch_export_id, note=note)
except Exception as exc:
raise BatchExportServiceRPCError(f"BatchExport {batch_export_id} could not be unpaused") from exc
batch_export.paused = False
batch_export.save()
if backfill is False:
return
batch_export.refresh_from_db()
start_at = batch_export.last_paused_at
end_at = batch_export.last_updated_at
backfill_export(temporal, batch_export_id, batch_export.team_id, start_at, end_at)
def delete_batch_export(instance: BatchExport):
"""Delete a batch export.
This process involves:
* First, pausing the batch export so no new runs are scheduled.
* Second, canceling any running backfills.
* Third, canceling any running runs.
* Fourth and finally, deleting the Temporal Schedule.
The first step is necessary to avoid new runs being scheduled while we execute all
other steps.
"""
temporal = sync_connect()
instance.deleted = True
try:
pause_batch_export(temporal, instance.id, note=f"Pausing due to delete request by team {instance.team_id}")
except BatchExportServiceRPCError:
logger.exception(
"Failed to pause batch export before deletion",
batch_export_id=instance.id,
)
for backfill in running_backfills_for_batch_export(instance.id):
try:
async_to_sync(cancel_running_batch_export_backfill)(temporal, backfill)
except Exception:
logger.exception(
"Failed to cancel backfill",
backfill_id=backfill.id,
batch_export_id=instance.id,
)
for run in running_runs_for_batch_export(instance.id):
try:
cancel_running_batch_export_run(temporal, run)
except Exception:
logger.exception(
"Failed to cancel run",
run_id=run.id,
back_export_id=instance.id,
)
try:
batch_export_delete_schedule(temporal, str(instance.pk))
except BatchExportServiceScheduleNotFound as e:
logger.warning(
"Schedule not found during delete",
schedule_id=e.schedule_id,
)
instance.save()
def batch_export_delete_schedule(temporal: Client, schedule_id: str) -> None:
"""Delete a Temporal Schedule."""
try:
delete_schedule(temporal, schedule_id)
except temporalio.service.RPCError as e:
if e.status == temporalio.service.RPCStatusCode.NOT_FOUND:
raise BatchExportServiceScheduleNotFound(schedule_id)
else:
raise BatchExportServiceRPCError() from e
def running_backfills_for_batch_export(batch_export_id: UUID):
"""Return an iterator over running batch export backfills."""
return BatchExportBackfill.objects.filter(
batch_export_id=batch_export_id, status=BatchExportBackfill.Status.RUNNING
).select_related("batch_export")
def running_runs_for_batch_export(batch_export_id: UUID):
"""Return an iterator over running batch export runs."""
return BatchExportRun.objects.filter(
batch_export_id=batch_export_id, status=BatchExportRun.Status.RUNNING
).select_related("batch_export")
async def cancel_running_batch_export_backfill(temporal: Client, batch_export_backfill: BatchExportBackfill) -> None:
"""Cancel a running BatchExportBackfill and any associated runs (should only be one max).
We cancel the backfill workflow first to prevent any new runs from starting.
"""
handle = temporal.get_workflow_handle(workflow_id=batch_export_backfill.workflow_id)
await handle.cancel()
batch_export_backfill.status = BatchExportBackfill.Status.CANCELLED
await batch_export_backfill.asave()
cancellable_statuses = [BatchExportRun.Status.RUNNING, BatchExportRun.Status.STARTING]
async for run in batch_export_backfill.runs.filter(status__in=cancellable_statuses):
try:
run_handle = temporal.get_workflow_handle(workflow_id=run.workflow_id)
await run_handle.cancel()
run.status = BatchExportRun.Status.CANCELLED
await run.asave()
except Exception:
logger.exception("Failed to cancel batch export run %s", run.id)
def sync_cancel_running_batch_export_backfill(temporal: Client, batch_export_backfill: BatchExportBackfill) -> None:
"""Cancel a running BatchExportBackfill and any associated runs (should only be one max).
We cancel the backfill workflow first to prevent any new runs from starting.
"""
handle = temporal.get_workflow_handle(workflow_id=batch_export_backfill.workflow_id)
async_to_sync(handle.cancel)()
batch_export_backfill.status = BatchExportBackfill.Status.CANCELLED
batch_export_backfill.save()
cancellable_statuses = [BatchExportRun.Status.RUNNING, BatchExportRun.Status.STARTING]
for run in batch_export_backfill.runs.filter(status__in=cancellable_statuses):
try:
cancel_running_batch_export_run(temporal, run)
except Exception:
logger.exception("Failed to cancel batch export run %s", run.id)
def cancel_running_batch_export_run(temporal: Client, batch_export_run: BatchExportRun) -> None:
"""Cancel a running BatchExportRun."""
handle = temporal.get_workflow_handle(workflow_id=batch_export_run.workflow_id)
async_to_sync(handle.cancel)()
batch_export_run.status = BatchExportRun.Status.CANCELLED
batch_export_run.save()
@dataclass
class BackfillBatchExportInputs:
"""Inputs for the BackfillBatchExport Workflow."""
team_id: int
batch_export_id: str
start_at: str | None
end_at: str | None
buffer_limit: int = 1
start_delay: float = 1.0
backfill_id: str | None = None
def backfill_export(
temporal: Client,
batch_export_id: str,
team_id: int,
start_at: dt.datetime | None,
end_at: dt.datetime | None,
backfill_id: str | None = None,
) -> str:
"""Starts a backfill for given team and batch export covering given date range.
Arguments:
temporal: A Temporal Client to trigger the workflow.
batch_export_id: The id of the BatchExport to backfill.
team_id: The id of the Team the BatchExport belongs to.
start_at: From when to backfill.
end_at: Up to when to backfill, if None it will backfill until it has caught up with realtime
and then unpause the underlying BatchExport.
backfill_id: Pre-generated UUID for the backfill model. If provided, the Temporal workflow
will use this ID when creating the BatchExportBackfill record.
"""
try:
batch_export = BatchExport.objects.select_related("destination").get(id=batch_export_id, team_id=team_id)
except BatchExport.DoesNotExist:
raise BatchExportIdError(batch_export_id)
# Ensure we don't allow users access to this feature until we are ready.
if not end_at and batch_export.destination.type not in (
BatchExportDestination.Destination.HTTP,
BatchExportDestination.Destination.NOOP, # For tests.
):
raise BatchExportWithNoEndNotAllowedError(f"BatchExport {batch_export_id} has no end_at and is not HTTP")
inputs = BackfillBatchExportInputs(
batch_export_id=batch_export_id,
team_id=team_id,
start_at=start_at.isoformat() if start_at else None,
end_at=end_at.isoformat() if end_at else None,
backfill_id=backfill_id,
)
start_at_utc_str = start_at.astimezone(tz=dt.UTC).isoformat() if start_at else "START"
# TODO: Should we use another signal besides "None"? i.e. "Inf" or "END".
# Keeping it like this for now for backwards compatibility.
end_at_utc_str = end_at.astimezone(tz=dt.UTC).isoformat() if end_at else "END"
workflow_id = f"{inputs.batch_export_id}-Backfill-{start_at_utc_str}-{end_at_utc_str}"
workflow_id = start_backfill_batch_export_workflow(
temporal,
workflow_id,
inputs=inputs,
destination_type=batch_export.destination.type,
model=batch_export.model or "events",
interval=batch_export.interval,
)
return workflow_id
@async_to_sync
async def start_backfill_batch_export_workflow(
temporal: Client,
workflow_id: str,
inputs: BackfillBatchExportInputs,
destination_type: str,
model: str,
interval: str,
) -> str:
"""Async call to start a BackfillBatchExportWorkflow."""
await temporal.start_workflow(
"backfill-batch-export",
inputs,
id=workflow_id,
task_queue=settings.BATCH_EXPORTS_TASK_QUEUE,
static_summary=build_static_summary(destination_type, model, interval, is_backfill=True),
)
return workflow_id
@async_to_sync
async def start_batch_export_workflow(
temporal: Client, name: str, workflow_id: str, inputs: BaseBatchExportInputs
) -> WorkflowHandle:
"""Async call to start a batch export workflow."""
handle = await temporal.start_workflow(
name,
inputs,
id=workflow_id,
task_queue=settings.BATCH_EXPORTS_TASK_QUEUE,
)
return handle
def start_file_download_batch_export(
batch_export: BatchExportOnDemand,
workflow_id: str,
data_interval_start: dt.datetime | None,
data_interval_end: dt.datetime | None,
batch_export_model: BatchExportModelInput,
batch_export_run_id: UUID | None = None,
compression: str | None = None,
format: str = "Parquet",
max_size_mb: int = 0,
include_events: list[str] | None = None,
exclude_events: list[str] | None = None,
) -> None:
incomplete_interval = data_interval_start is None or data_interval_end is None
if incomplete_interval and (batch_export_model.name != "hogql" or batch_export_run_id is None):
raise ValueError("Only on-demand HogQL exports can omit interval bounds")
inputs = FileDownloadBatchExportInputs(
batch_export_id=batch_export.id,
batch_export_model=batch_export_model,
batch_export_run_id=batch_export_run_id,
team_id=batch_export.team_id,
data_interval_start=data_interval_start.isoformat() if data_interval_start is not None else None,
data_interval_end=data_interval_end.isoformat() if data_interval_end is not None else None,
compression=compression,
file_format=format,
max_file_size_mb=max_size_mb,
include_events=include_events,
exclude_events=exclude_events,
# Persist timeout choices in the workflow input so configuration changes do not affect replay.
main_activity_timeout_seconds=settings.BATCH_EXPORT_HOGQL_ON_DEMAND_MAIN_TIMEOUT_SECONDS
if incomplete_interval
else None,
stage_activity_timeout_seconds=settings.BATCH_EXPORT_HOGQL_MAX_EXECUTION_TIME + 300
if incomplete_interval
else None,
)
temporal = sync_connect()
start_batch_export_workflow(temporal, "file-download-export", workflow_id, inputs)
def create_batch_export_run(
batch_export_id: UUID,
data_interval_start: str | None,
data_interval_end: str,
status: str = BatchExportRun.Status.STARTING,
backfill_id: UUID | None = None,
) -> BatchExportRun:
"""Create a BatchExportRun after a Temporal Workflow execution.
In a first approach, this method is intended to be called only by Temporal Workflows,
as only the Workflows themselves can know when they start.
Args:
batch_export_id: The UUID of the BatchExport the BatchExportRun to create belongs to.
data_interval_start: The start of the period of data exported in this BatchExportRun.
data_interval_end: The end of the period of data exported in this BatchExportRun.
status: The initial status for the created BatchExportRun.
backfill_id: The UUID of the BatchExportBackfill the BatchExportRun belongs to (if any).
"""
run = BatchExportRun(
batch_export_id=batch_export_id,
status=status,
data_interval_start=dt.datetime.fromisoformat(data_interval_start) if data_interval_start else None,
data_interval_end=dt.datetime.fromisoformat(data_interval_end),
backfill_id=backfill_id,
)
run.save()
return run
def update_batch_export_run(
run_id: UUID,
**kwargs,
) -> BatchExportRun:
"""Update the BatchExportRun with given run_id and provided **kwargs.
Arguments:
run_id: The id of the BatchExportRun to update.
"""
# nosemgrep: idor-lookup-without-team (internal service, team_id passed as parameter)
model = BatchExportRun.objects.select_related(
"batch_export",
"batch_export__destination",
"batch_export_on_demand",
"batch_export_on_demand__destination",
).filter(id=run_id)
update_at = dt.datetime.now(dt.UTC)
updated = model.update(
**kwargs,
last_updated_at=update_at,
)
if not updated:
raise ValueError(f"BatchExportRun with id {run_id} not found.")
return model.get()
async def aupdate_batch_export_run(
run_id: UUID,
**kwargs,
) -> BatchExportRun:
"""Update the BatchExportRun with given run_id and provided **kwargs.
Arguments:
run_id: The id of the BatchExportRun to update.
"""
# nosemgrep: idor-lookup-without-team (internal service, team_id passed as parameter)
model = BatchExportRun.objects.filter(id=run_id)
update_at = dt.datetime.now(dt.UTC)
updated = await model.aupdate(
**kwargs,
last_updated_at=update_at,
)
if not updated:
raise ValueError(f"BatchExportRun with id {run_id} not found.")
return await model.aget()
def count_failed_batch_export_runs(batch_export_id: UUID, last_n: int) -> int:
"""Count failed batch export runs in the 'last_n' runs."""
count_of_failures = (
# nosemgrep: idor-lookup-without-team (internal service, team_id passed as parameter)
BatchExportRun.objects.filter(
id__in=BatchExportRun.objects.filter(batch_export_id=batch_export_id)
.order_by("-last_updated_at")
.values("id")[:last_n]
)
.filter(status=BatchExportRun.Status.FAILED)
.count()
)
return count_of_failures
async def acount_failed_batch_export_runs(batch_export_id: UUID, last_n: int) -> int:
"""Count failed batch export runs in the 'last_n' runs."""
count_of_failures = (
# nosemgrep: idor-lookup-without-team (internal service, team_id passed as parameter)
await BatchExportRun.objects.filter(
id__in=BatchExportRun.objects.filter(batch_export_id=batch_export_id)
.order_by("-last_updated_at")
.values("id")[:last_n]
)
.filter(status=BatchExportRun.Status.FAILED)
.acount()
)
return count_of_failures
def _get_schedule_spec(batch_export: BatchExport) -> ScheduleSpec:
timezone = str(batch_export.timezone_info)
# if daily or weekly interval, use ScheduleCalendarSpec so we can set the time of day to run (and ensure timezones
# are respected)
if batch_export.interval == "day":
offset_hour = batch_export.offset_hour
# should never be the case so assert is safe
assert offset_hour is not None
return ScheduleSpec(
start_at=batch_export.start_at,
end_at=batch_export.end_at,
calendars=[
ScheduleCalendarSpec(
comment=f"Daily at {offset_hour} hours after midnight local time",
hour=[ScheduleRange(start=offset_hour, end=offset_hour)],
)
],
jitter=batch_export.jitter,
time_zone_name=timezone,
)
elif batch_export.interval == "week":
offset_day = batch_export.offset_day
assert offset_day is not None
day_name = batch_export.offset_day_name
assert day_name is not None
offset_hour = batch_export.offset_hour
assert offset_hour is not None
return ScheduleSpec(
start_at=batch_export.start_at,
end_at=batch_export.end_at,
calendars=[
ScheduleCalendarSpec(
comment=f"Weekly on {day_name} at {offset_hour} hours after midnight local time",
day_of_week=[ScheduleRange(start=offset_day, end=offset_day)],
hour=[ScheduleRange(start=offset_hour, end=offset_hour)],
)
],
jitter=batch_export.jitter,
time_zone_name=timezone,
)
# for other intervals, use ScheduleIntervalSpec
else:
return ScheduleSpec(
start_at=batch_export.start_at,
end_at=batch_export.end_at,
# nosemgrep: schedule-must-avoid-minute-zero -- each run exports the interval that just closed, and batch_export.jitter already spreads the start
intervals=[ScheduleIntervalSpec(every=batch_export.interval_time_delta)],
jitter=batch_export.jitter,
time_zone_name=timezone,
)
def sync_batch_export(batch_export: BatchExport, created: bool):
workflow, workflow_inputs = DESTINATION_WORKFLOWS[batch_export.destination.type]
state = ScheduleState(
note=f"Schedule updated for BatchExport {batch_export.id} to Destination {batch_export.destination.id} in Team {batch_export.team.id}.",
paused=batch_export.paused,
)
destination_config_fields = {field.name for field in fields(workflow_inputs)}
destination_config = {k: v for k, v in batch_export.destination.config.items() if k in destination_config_fields}
task_queue = (
settings.SYNC_BATCH_EXPORTS_TASK_QUEUE
if batch_export.destination.type == "HTTP"
else settings.BATCH_EXPORTS_TASK_QUEUE
)
temporal = sync_connect()
schedule = Schedule(
action=ScheduleActionStartWorkflow(
workflow,
asdict(
workflow_inputs(
team_id=batch_export.team.id,
batch_export_id=str(batch_export.id),
interval=str(batch_export.interval),
timezone=str(batch_export.timezone_info),
batch_export_model=get_batch_export_model(batch_export),
integration_id=batch_export.destination.integration_id,
**destination_config,
)
),
id=str(batch_export.id),
task_queue=task_queue,
retry_policy=temporalio.common.RetryPolicy(
initial_interval=dt.timedelta(seconds=10),
maximum_interval=dt.timedelta(seconds=60),
maximum_attempts=2,
non_retryable_error_types=["ActivityError", "ApplicationError", "CancelledError"],
),
static_summary=build_static_summary(
batch_export.destination.type, batch_export.model or "events", batch_export.interval
),
),
spec=_get_schedule_spec(batch_export),
state=state,
policy=SchedulePolicy(overlap=ScheduleOverlapPolicy.ALLOW_ALL),
)
if created:
create_schedule(temporal, id=str(batch_export.id), schedule=schedule)
else:
update_schedule(temporal, id=str(batch_export.id), schedule=schedule)
return batch_export
async def aget_or_create_batch_export_backfill(
batch_export_id: UUID,
team_id: int,
start_at: str | None,
end_at: str | None,
status: str = BatchExportRun.Status.RUNNING,
backfill_id: str | None = None,
) -> BatchExportBackfill:
"""Create a BatchExportBackfill, or return an existing one if a backfill_id is provided and already exists.
We use `aget_or_create` to handle the rare case (which we have seen in production) where the Temporal activity
retries due to a timeout, but the previous attempt has committed the row to Postgres.
Args:
batch_export_id: The UUID of the BatchExport the BatchExportBackfill to create belongs to.
team_id: The id of the Team the BatchExportBackfill to create belongs to.
start_at: The start of the period to backfill in this BatchExportBackfill.
end_at: The end of the period to backfill in this BatchExportBackfill.
status: The initial status for the created BatchExportBackfill.
backfill_id: Pre-generated UUID for the backfill. If provided, will be used as the model's primary key.
"""
defaults: dict = {
"status": status,
"start_at": dt.datetime.fromisoformat(start_at) if start_at else None,
"end_at": dt.datetime.fromisoformat(end_at) if end_at else None,
}
if backfill_id is not None:
backfill, _created = await BatchExportBackfill.objects.aget_or_create(
id=backfill_id,
team_id=team_id,
batch_export_id=batch_export_id,
defaults=defaults,
)
else:
backfill = BatchExportBackfill(**defaults, team_id=team_id, batch_export_id=batch_export_id)
await backfill.asave()
return backfill
def update_batch_export_backfill(
backfill_id: UUID,
*,
adjusted_start_at: str | None = None,
total_records_count: int | None = None,
status: str | None = None,
finished_at: dt.datetime | None = None,
) -> BatchExportBackfill:
"""Update a BatchExportBackfill with the given fields.
Args:
backfill_id: The id of the BatchExportBackfill to update.
adjusted_start_at: The adjusted start time after validation.
total_records_count: The total number of records to export.
status: The new status for the BatchExportBackfill.
finished_at: The time the BatchExportBackfill finished.
"""
update_fields: dict[str, dt.datetime | int | str] = {}
if adjusted_start_at is not None:
update_fields["adjusted_start_at"] = dt.datetime.fromisoformat(adjusted_start_at)
if total_records_count is not None:
update_fields["total_records_count"] = total_records_count
if status is not None:
update_fields["status"] = status
if finished_at is not None:
update_fields["finished_at"] = finished_at
if not update_fields:
raise ValueError("At least one field must be provided to update")
# nosemgrep: idor-lookup-without-team (internal service, team_id passed as parameter)
model = BatchExportBackfill.objects.filter(id=backfill_id)
updated = model.update(**update_fields)
if not updated:
raise ValueError(f"BatchExportBackfill with id {backfill_id} not found.")
return model.get()
async def aupdate_records_total_count(
batch_export_id: UUID, interval_start: dt.datetime, interval_end: dt.datetime, count: int
) -> int:
"""Update the expected records count for a set of batch export runs.
Typically, there is one batch export run per batch export interval, however
there could be multiple if data has been backfilled.
"""
rows_updated = await BatchExportRun.objects.filter(
batch_export_id=batch_export_id,
data_interval_start=interval_start,
data_interval_end=interval_end,
).aupdate(records_total_count=count)
return rows_updated
async def afetch_last_run_records_completed(
batch_export_id: UUID,
*,
matching_interval_duration: dt.timedelta | None,
before_or_at_interval_end: dt.datetime | None = None,
not_older_than: dt.datetime | None = None,
) -> int | None:
"""Async fetch the `records_completed` of the most recent completed run for a batch export.
Used as a rough estimate to pick how many staging files to write. Only scheduled `BatchExport`
runs are matched. A `BatchExportOnDemand` is created for each request and runs once, so it has no
earlier run to estimate from, and `compute_num_partitions` does not call this for one.
The `before_or_at_interval_end` and `not_older_than` filters are relative to the interval being
processed (which improves accuracy for backfills):
- `before_or_at_interval_end`: only runs whose `data_interval_end` is at or before it.
- `not_older_than`: only runs whose `data_interval_end` is at or after it, to ignore stale runs from
a long-paused export.
`matching_interval_duration` is the current interval's length. We look only at the single most
recent run and return its count only if that run's own interval length matches; if the export's
frequency changed, the latest run isn't comparable, so we return None and let the next run (which
will have a same-frequency predecessor) re-adjust.
Returns None when no usable run exists (e.g. the first ever run, or a frequency change).
"""
queryset = BatchExportRun.objects.filter(
batch_export_id=batch_export_id,
status=BatchExportRun.Status.COMPLETED,
records_completed__isnull=False,
)
if before_or_at_interval_end is not None:
queryset = queryset.filter(data_interval_end__lte=before_or_at_interval_end)
if not_older_than is not None:
queryset = queryset.filter(data_interval_end__gte=not_older_than)
run = (
await queryset.order_by("-data_interval_end")
.values("records_completed", "data_interval_start", "data_interval_end")
.afirst()
)
if run is None:
return None
if matching_interval_duration is not None:
start = run["data_interval_start"]
end = run["data_interval_end"]
last_interval_duration = end - start if start is not None and end is not None else None
if last_interval_duration != matching_interval_duration:
return None
return run["records_completed"]
async def afetch_batch_export_runs_in_range(
batch_export_id: UUID,
interval_start: dt.datetime,
interval_end: dt.datetime,
) -> list[BatchExportRun]:
"""Async fetch all BatchExportRuns for a given batch export within a time interval.
Arguments:
batch_export_id: The UUID of the BatchExport to fetch runs for.
interval_start: The start of the time interval to fetch runs from.
interval_end: The end of the time interval to fetch runs until.
Returns:
A list of BatchExportRun objects within the given interval, ordered by data_interval_start.
"""
queryset = BatchExportRun.objects.filter(
batch_export_id=batch_export_id,
data_interval_start__gte=interval_start,
data_interval_end__lte=interval_end,
).order_by("data_interval_start")
return [run async for run in queryset]
@dataclass(frozen=False, kw_only=True)
class BatchExportInsertInputs:
"""Base dataclass for batch export insert inputs containing common fields."""
team_id: int
data_interval_start: str | None
data_interval_end: str | None
exclude_events: list[str] | None = None
include_events: list[str] | None = None
run_id: str | None = None
backfill_details: BackfillDetails | None = None
batch_export_model: BatchExportModelInput | None = None
# TODO: Remove after updating existing batch exports to use backfill_details
is_backfill: bool = False
# TODO - pass these in to all inherited classes
batch_export_id: str | None = None
destination_default_fields: list[BatchExportField] | None = None
stage_folder: str | None = None
# Total rows staged for this run (from ClickHouse), used to report export progress
records_total: int | None = None
on_demand: bool = False
def get_is_backfill(self) -> bool:
"""Needed for backwards compatibility with existing batch exports.
TODO: remove once all existing backfills are finished.
"""
# to check status of migration
if self.is_backfill and not self.backfill_details:
logger.info(
"Backfill inputs migration: BatchExport for team %s has is_backfill set to True but no backfill_details",
self.team_id,
)
if self.backfill_details is not None:
return True
return self.is_backfill
@property
def properties_to_log(self) -> dict[str, typing.Any]:
"""Return a dictionary of properties that we want to log if an error is raised.
We list these explicitly rather than setting all fields as safe to log, so that the default is opt-out (just in
case new fields get added which are sensitive).
"""
return {
"team_id": self.team_id,
"data_interval_start": self.data_interval_start,
"data_interval_end": self.data_interval_end,
"exclude_events": self.exclude_events,
"include_events": self.include_events,
"run_id": self.run_id,
"backfill_details": self.backfill_details,
"batch_export_model": self.batch_export_model,
"batch_export_id": self.batch_export_id,
"stage_folder": self.stage_folder,
}