Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(api): Expose incremental fields in API and MCP (#114537)

Tomás Farías Santana82fc23358e15
diff --git a/products/batch_exports/backend/facade/enums.py b/products/batch_exports/backend/facade/enums.py
index 01c6d22a1ef7..276cf17388f4 100644
--- a/products/batch_exports/backend/facade/enums.py
+++ b/products/batch_exports/backend/facade/enums.py
@@ -70,3 +70,11 @@ class BatchExportInterval(LabeledStrEnum):
WEEK = "week", "week"
EVERY_5_MINUTES = "every 5 minutes", "every 5 minutes"
EVERY_15_MINUTES = "every 15 minutes", "every 15 minutes"
+
+
+class IncrementalMode(LabeledStrEnum):
+ """Different modes to handle incremental updates."""
+
+ APPEND = "APPEND"
+ MERGE = "MERGE"
+ # TODO: Add 'FULL_REFRESH': data is re-created on every run.
diff --git a/products/batch_exports/backend/hogql_source.py b/products/batch_exports/backend/hogql_source.py
index 45bdf60c2e09..1cbe0b968d29 100644
--- a/products/batch_exports/backend/hogql_source.py
+++ b/products/batch_exports/backend/hogql_source.py
@@ -205,7 +205,7 @@ def _validate_select_columns_are_named(parsed: ast.SelectQuery | ast.SelectSetQu
def validate_hogql_query_for_batch_export(
hogql_query: str, team: "Team", *, user: "User", modifiers: HogQLQueryModifiers | None = None
-) -> None:
+) -> ast.SelectQuery | ast.SelectSetQuery:
"""Validate a HogQL query can power a batch export for the given team.
Parses the query, checks output columns are named, and compiles it with the user's
@@ -218,6 +218,9 @@ def validate_hogql_query_for_batch_export(
Raises:
UnsupportedHogQLQueryError: If the query cannot power a batch export.
InternalHogQLError: Left to propagate, as in `parse_hogql_select_for_batch_export`.
+
+ Returns:
+ The valid, parsed AST for a select query.
"""
validate_hogql_batch_export_user(team, user)
parsed = parse_hogql_select_for_batch_export(hogql_query)
@@ -235,6 +238,8 @@ def validate_hogql_query_for_batch_export(
except ExposedHogQLError as e:
raise UnsupportedHogQLQueryError(f"Invalid HogQL query: {e}") from e
+ return prepared
+
def drop_unknown_hogql_modifiers(stored: dict[str, typing.Any]) -> dict[str, typing.Any]:
"""Return the stored HogQL modifiers without keys that `HogQLQueryModifiers` no longer has.
diff --git a/products/batch_exports/backend/migrations/0011_batchexport_incremental_mode_and_more.py b/products/batch_exports/backend/migrations/0011_batchexport_incremental_mode_and_more.py
new file mode 100644
index 000000000000..9ebca0b6c7a4
--- /dev/null
+++ b/products/batch_exports/backend/migrations/0011_batchexport_incremental_mode_and_more.py
@@ -0,0 +1,59 @@
+# Generated by Django 5.2.17 on 2026-10-09 09:50
+
+import django.contrib.postgres.fields
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+ dependencies = [
+ ("batch_exports", "0010_alter_batchexportdestination_type"),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name="batchexport",
+ name="incremental_mode",
+ field=models.CharField(
+ choices=[("APPEND", "Append"), ("MERGE", "Merge")],
+ db_default="APPEND",
+ default="APPEND",
+ help_text="How this batch export handles incremental updates.",
+ max_length=64,
+ ),
+ ),
+ migrations.AddField(
+ model_name="batchexportondemand",
+ name="incremental_mode",
+ field=models.CharField(
+ choices=[("APPEND", "Append"), ("MERGE", "Merge")],
+ db_default="APPEND",
+ default="APPEND",
+ help_text="How this batch export handles incremental updates.",
+ max_length=64,
+ ),
+ ),
+ migrations.AddField(
+ model_name="batchexportsource",
+ name="primary_key",
+ field=django.contrib.postgres.fields.ArrayField(
+ base_field=models.TextField(
+ help_text="A column name to use as part of a primary key. If set, cannot be empty or null"
+ ),
+ help_text="The column or columns making up the primary key for this source.",
+ null=True,
+ size=None,
+ ),
+ ),
+ migrations.AddField(
+ model_name="batchexportsource",
+ name="version_key",
+ field=django.contrib.postgres.fields.ArrayField(
+ base_field=models.TextField(
+ help_text="A column name to use as part of a version key. If set, cannot be empty or null"
+ ),
+ help_text="The column or columns making up the version key for this source.",
+ null=True,
+ size=None,
+ ),
+ ),
+ ]
diff --git a/products/batch_exports/backend/migrations/max_migration.txt b/products/batch_exports/backend/migrations/max_migration.txt
index 6c8cf523aabc..08d71eb17e4c 100644
--- a/products/batch_exports/backend/migrations/max_migration.txt
+++ b/products/batch_exports/backend/migrations/max_migration.txt
@@ -1 +1 @@
-0010_alter_batchexportdestination_type
+0011_batchexport_incremental_mode_and_more
diff --git a/products/batch_exports/backend/models/batch_export.py b/products/batch_exports/backend/models/batch_export.py
index 539bcffb9a39..7076d53ee353 100644
--- a/products/batch_exports/backend/models/batch_export.py
+++ b/products/batch_exports/backend/models/batch_export.py
@@ -5,6 +5,7 @@ from enum import IntEnum
from math import ceil
from zoneinfo import ZoneInfo
+from django.contrib.postgres.fields import ArrayField
from django.db import models
from django.db.models import Q
@@ -127,7 +128,7 @@ class BatchExportSource(TeamScopedRootMixin, UUIDTModel):
This model answers the question: what data are we exporting? For now it holds a
HogQL query whose results are exported, but it's designed to grow into the single
place that captures how a batch export selects its data (data warehouse view
- references, export mode, data interval field, primary and version keys) in future.
+ references, data interval field, primary and version keys) in future.
"""
class Meta:
@@ -154,6 +155,24 @@ class BatchExportSource(TeamScopedRootMixin, UUIDTModel):
blank=True,
help_text="HogQL modifiers used to run the query. They override the team modifiers key by key.",
)
+ primary_key = ArrayField(
+ base_field=models.TextField(
+ blank=False,
+ null=False,
+ help_text="A column name to use as part of a primary key. If set, cannot be empty or null",
+ ),
+ null=True,
+ help_text="The column or columns making up the primary key for this source.",
+ )
+ version_key = ArrayField(
+ base_field=models.TextField(
+ blank=False,
+ null=False,
+ help_text="A column name to use as part of a version key. If set, cannot be empty or null",
+ ),
+ null=True,
+ help_text="The column or columns making up the version key for this source.",
+ )
created_at = models.DateTimeField(
auto_now_add=True,
help_text="The timestamp at which this BatchExportSource was created.",
@@ -297,6 +316,7 @@ class BatchExport(ModelActivityMixin, UUIDTModel):
db_table = "posthog_batchexport"
Model = enums.BatchExportModel
+ IncrementalMode = enums.IncrementalMode
team = models.ForeignKey(
"posthog.Team", on_delete=models.CASCADE, help_text="The team this belongs to.", related_name="+"
@@ -371,6 +391,13 @@ class BatchExport(ModelActivityMixin, UUIDTModel):
default=Model.EVENTS.value,
help_text="Which model this BatchExport is exporting.",
)
+ incremental_mode = models.CharField(
+ max_length=64,
+ default=IncrementalMode.APPEND.value,
+ db_default=IncrementalMode.APPEND.value,
+ choices=IncrementalMode.choices,
+ help_text="How this batch export handles incremental updates.",
+ )
filters = models.JSONField(null=True, blank=True)
# determines the timezone used for daily or weekly exports
timezone = models.CharField(max_length=240, choices=TIMEZONES, default="UTC")
@@ -487,6 +514,16 @@ class BatchExport(ModelActivityMixin, UUIDTModel):
"""Return the HogQL modifiers of this batch export's source, if it has any."""
return self.source.hogql_modifiers if self.source is not None else None
+ @property
+ def primary_key(self) -> list[str] | None:
+ """Return the primary key of this batch export's source, if it has any."""
+ return self.source.primary_key if self.source is not None else None
+
+ @property
+ def version_key(self) -> list[str] | None:
+ """Return the version key of this batch export's source, if it has any."""
+ return self.source.version_key if self.source is not None else None
+
def get_batch_exports_using_integration(team_id: int, integration_id: int) -> list[BatchExport]:
"""Return a list of batch exports using integration_id.
@@ -668,6 +705,7 @@ class BatchExportOnDemand(TeamScopedRootMixin, ModelActivityMixin, UUIDTModel):
db_table = "posthog_batchexportondemand"
Model = enums.BatchExportModel
+ IncrementalMode = enums.IncrementalMode
team = models.ForeignKey(
"posthog.Team", on_delete=models.CASCADE, help_text="The team this belongs to.", related_name="+"
@@ -711,4 +749,11 @@ class BatchExportOnDemand(TeamScopedRootMixin, ModelActivityMixin, UUIDTModel):
default=Model.EVENTS.value,
help_text="Which model this batch export is exporting.",
)
+ incremental_mode = models.CharField(
+ max_length=64,
+ default=IncrementalMode.APPEND.value,
+ db_default=IncrementalMode.APPEND.value,
+ choices=IncrementalMode.choices,
+ help_text="How this batch export handles incremental updates.",
+ )
filters = models.JSONField(null=True, blank=True)
diff --git a/products/batch_exports/backend/presentation/views/batch_export/exports.py b/products/batch_exports/backend/presentation/views/batch_export/exports.py
index f508337d902a..41a9a2f392e6 100644
--- a/products/batch_exports/backend/presentation/views/batch_export/exports.py
+++ b/products/batch_exports/backend/presentation/views/batch_export/exports.py
@@ -148,6 +148,25 @@ class BatchExportRequestSerializer(serializers.Serializer):
allow_null=True,
help_text=HOGQL_MODIFIERS_HELP_TEXT,
)
+ primary_key = serializers.ListField(
+ child=serializers.CharField(allow_null=False, allow_blank=False),
+ required=False,
+ allow_null=True,
+ allow_empty=False,
+ help_text="The column or columns in the provided HogQL query that make up the primary key for a HogQL-backed batch export. Required if incremental mode is 'MERGE'",
+ )
+ version_key = serializers.ListField(
+ child=serializers.CharField(allow_null=False, allow_blank=False),
+ required=False,
+ allow_null=True,
+ allow_empty=False,
+ help_text="The column or columns in the provided HogQL query that make up the version key for a HogQL-backed batch export.",
+ )
+ incremental_mode = serializers.ChoiceField(
+ choices=BatchExport.IncrementalMode.choices,
+ default=serializers.CreateOnlyDefault(BatchExport.IncrementalMode.APPEND), # type: ignore[arg-type]
+ help_text="How this batch export handles incremental updates.",
+ )
filters = serializers.JSONField(
required=False,
allow_null=True,
@@ -280,6 +299,25 @@ class BatchExportSerializer(serializers.ModelSerializer):
allow_null=True,
help_text=HOGQL_MODIFIERS_HELP_TEXT,
)
+ primary_key = serializers.ListField(
+ child=serializers.CharField(allow_null=False, allow_blank=False),
+ required=False,
+ allow_null=True,
+ allow_empty=False,
+ help_text="The column or columns in the provided HogQL query that make up the primary key for a HogQL-backed batch export. Required if incremental mode is 'MERGE'.",
+ )
+ version_key = serializers.ListField(
+ child=serializers.CharField(allow_null=False, allow_blank=False),
+ required=False,
+ allow_null=True,
+ allow_empty=False,
+ help_text="The column or columns in the provided HogQL query that make up the version key for a HogQL-backed batch export.",
+ )
+ incremental_mode = serializers.ChoiceField(
+ choices=BatchExport.IncrementalMode.choices,
+ default=serializers.CreateOnlyDefault(BatchExport.IncrementalMode.APPEND), # type: ignore[arg-type]
+ help_text="How this batch export handles incremental updates.",
+ )
timezone = serializers.ChoiceField(
choices=TIMEZONES,
required=False,
@@ -319,6 +357,9 @@ class BatchExportSerializer(serializers.ModelSerializer):
"latest_runs",
"hogql_query",
"hogql_modifiers",
+ "primary_key",
+ "version_key",
+ "incremental_mode",
"schema",
"filters",
"timezone",
@@ -344,6 +385,10 @@ class BatchExportSerializer(serializers.ModelSerializer):
self._validate_attribute_only_supported_for_model(attrs, name="filters", model=BatchExport.Model.EVENTS)
self._validate_attribute_only_supported_for_model(attrs, name="hogql_modifiers", model=BatchExport.Model.HOGQL)
+ # Pre-defined models have these values hard-coded, only HogQL allows user input
+ self._validate_attribute_only_supported_for_model(attrs, name="primary_key", model=BatchExport.Model.HOGQL)
+ self._validate_attribute_only_supported_for_model(attrs, name="version_key", model=BatchExport.Model.HOGQL)
+
# TODO: Once these teams are migrated, remove this.
if model == BatchExport.Model.EVENTS and (hogql_query := attrs.get("hogql_query")) is not None:
is_grandfathered_team_id = BatchExport.objects.filter(
@@ -464,10 +509,45 @@ class BatchExportSerializer(serializers.ModelSerializer):
team = self.context["get_team"]()
hogql_modifiers = self._get_hogql_modifiers(attrs)
try:
- validate_hogql_query_for_batch_export(hogql_query, team, user=user, modifiers=hogql_modifiers)
+ parsed = validate_hogql_query_for_batch_export(hogql_query, team, user=user, modifiers=hogql_modifiers)
except UnsupportedHogQLQueryError as e:
raise serializers.ValidationError({"hogql_query": str(e)}) from e
+ # NOTE: Should be populated after a successful preparation
+ assert isinstance(parsed.type, (ast.SelectQueryType, ast.SelectSetQueryType))
+ output_columns = set(parsed.type.columns)
+
+ saved_incremental_mode = self.instance.incremental_mode if self.instance is not None else None
+ incremental_mode = attrs.get("incremental_mode", saved_incremental_mode)
+
+ saved_primary_key = source.primary_key if source is not None else None
+ primary_key = set(attrs.get("primary_key", saved_primary_key) or ())
+
+ if not primary_key and incremental_mode == BatchExport.IncrementalMode.MERGE:
+ raise serializers.ValidationError(
+ {"primary_key": "'primary_key' is required when 'incremental_mode' is 'MERGE'"}
+ )
+
+ missing_primary_key = primary_key - output_columns
+
+ if missing_primary_key:
+ raise serializers.ValidationError(
+ {
+ "primary_key": f"'primary_key' contains columns that are not present in the query: {', '.join(missing_primary_key)}"
+ }
+ )
+
+ saved_version_key = source.version_key if source is not None else None
+ version_key = set(attrs.get("version_key", saved_version_key) or ())
+ missing_version_key = version_key - output_columns
+
+ if missing_version_key:
+ raise serializers.ValidationError(
+ {
+ "version_key": f"'version_key' contains columns that are not present in the query: {','.join(missing_version_key)}"
+ }
+ )
+
def _get_hogql_modifiers(self, attrs: dict[str, typing.Any]) -> HogQLQueryModifiers | None:
if "hogql_modifiers" in attrs:
if attrs["hogql_modifiers"] is None:
@@ -560,6 +640,24 @@ class BatchExportSerializer(serializers.ModelSerializer):
return offset_hour
+ def validate_primary_key(self, primary_key: list[str] | None) -> list[str] | None:
+ """Validate primary key contains no duplicates."""
+ if not primary_key:
+ return primary_key
+
+ if not len(set(primary_key)) == len(primary_key):
+ raise serializers.ValidationError("'primary_key' contains duplicate column names.")
+ return primary_key
+
+ def validate_version_key(self, version_key: list[str] | None) -> list[str] | None:
+ """Validate version key contains no duplicates."""
+ if not version_key:
+ return version_key
+
+ if not len(set(version_key)) == len(version_key):
+ raise serializers.ValidationError("'version_key' contains duplicate column names.")
+ return version_key
+
def validate_filters(self, filters):
try:
validate_batch_export_filters(filters)
@@ -875,10 +973,18 @@ class BatchExportSerializer(serializers.ModelSerializer):
model = validated_data["model"]
hogql_query = validated_data.pop("hogql_query", None)
hogql_modifiers = validated_data.pop("hogql_modifiers", None)
+ primary_key = validated_data.pop("primary_key", None)
+ version_key = validated_data.pop("version_key", None)
source = None
if model == BatchExport.Model.HOGQL:
- source = BatchExportSource(team_id=team_id, hogql_query=hogql_query, hogql_modifiers=hogql_modifiers)
+ source = BatchExportSource(
+ team_id=team_id,
+ hogql_query=hogql_query,
+ hogql_modifiers=hogql_modifiers,
+ primary_key=primary_key,
+ version_key=version_key,
+ )
elif hogql_query is not None:
# TODO: Migrate batch exports using a HogQL query to HogQL model.
validated_data["schema"] = self.serialize_hogql_query_to_batch_export_schema(hogql_query)
@@ -997,10 +1103,12 @@ class BatchExportSerializer(serializers.ModelSerializer):
def update(self, batch_export: BatchExport, validated_data: dict) -> BatchExport:
"""Update a BatchExport."""
destination_data = validated_data.pop("destination", None)
- hogql_query_provided = "hogql_query" in validated_data
- hogql_query = validated_data.pop("hogql_query", None)
- hogql_modifiers_provided = "hogql_modifiers" in validated_data
- hogql_modifiers = validated_data.pop("hogql_modifiers", None)
+
+ missing = object()
+ hogql_query = validated_data.pop("hogql_query", missing)
+ hogql_modifiers = validated_data.pop("hogql_modifiers", missing)
+ primary_key = validated_data.pop("primary_key", missing)
+ version_key = validated_data.pop("version_key", missing)
user = self.context["request"].user
if not isinstance(user, User):
@@ -1023,18 +1131,26 @@ class BatchExportSerializer(serializers.ModelSerializer):
batch_export.destination.integration = integration
if batch_export.model == BatchExport.Model.HOGQL:
- if hogql_query is not None or hogql_modifiers_provided:
+ hogql_attributes = (hogql_query, hogql_modifiers, primary_key, version_key)
+ if any(attr is not missing for attr in hogql_attributes):
source = batch_export.source or BatchExportSource(team_id=batch_export.team_id)
- if hogql_query is not None:
+
+ if hogql_query is not missing:
source.hogql_query = hogql_query
- if hogql_modifiers_provided:
+ if hogql_modifiers is not missing:
source.hogql_modifiers = hogql_modifiers
+ if primary_key is not missing:
+ source.primary_key = primary_key
+ if version_key is not missing:
+ source.version_key = version_key
+
source.save()
batch_export.source = source
- elif hogql_query is not None:
- validated_data["schema"] = self.serialize_hogql_query_to_batch_export_schema(hogql_query)
- elif hogql_query_provided:
- validated_data["schema"] = None
+
+ elif hogql_query is not missing:
+ validated_data["schema"] = (
+ self.serialize_hogql_query_to_batch_export_schema(hogql_query) if hogql_query is not None else None
+ )
batch_export.destination.save()
batch_export = super().update(batch_export, validated_data)
diff --git a/products/batch_exports/backend/tests/api/test_serializers.py b/products/batch_exports/backend/tests/api/test_serializers.py
index c1fd88dc686e..df2f4769e20b 100644
--- a/products/batch_exports/backend/tests/api/test_serializers.py
+++ b/products/batch_exports/backend/tests/api/test_serializers.py
@@ -15,8 +15,9 @@ from posthog.hogql.property_access_types import RestrictedProperty
from posthog.api.scoped_related_fields import TeamScopedPrimaryKeyRelatedField
from posthog.models import Organization, PropertyDefinition, Team
from posthog.models.integration import Integration
+from posthog.models.scoping import team_scope
-from products.batch_exports.backend.models.batch_export import BatchExport
+from products.batch_exports.backend.models.batch_export import BatchExport, BatchExportSource
from products.batch_exports.backend.presentation.views.batch_export.destinations import BatchExportDestinationSerializer
from products.batch_exports.backend.presentation.views.batch_export.exports import (
BatchExportSerializer,
@@ -297,6 +298,75 @@ class TestSerializeHogQLQueryToBatchExportSchema(BaseTest):
assert "JSONExtractRaw(" in field["expression"], field
+class TestBatchExportIncrementalMode(BaseTest):
+ @parameterized.expand([("events",), ("hogql",)])
+ def test_create_and_patch_incremental_mode(self, model: str) -> None:
+ context = {
+ "team_id": self.team.pk,
+ "get_team": lambda: self.team,
+ "request": SimpleNamespace(user=self.user, method="POST"),
+ }
+ data = {
+ "name": "Incremental export",
+ "model": model,
+ "interval": "hour",
+ "destination": {"type": "NoOp", "config": {}},
+ "incremental_mode": "MERGE",
+ }
+ if model == "hogql":
+ data.update(hogql_query="SELECT uuid FROM events", primary_key=["uuid"])
+
+ with (
+ team_scope(self.team.pk),
+ patch("products.batch_exports.backend.presentation.views.batch_export.exports.sync_batch_export"),
+ patch(
+ "products.batch_exports.backend.presentation.views.utils.posthoganalytics.feature_enabled",
+ return_value=True,
+ ),
+ ):
+ serializer = BatchExportSerializer(data=data, context=context)
+ assert serializer.is_valid(), serializer.errors
+ batch_export = serializer.save()
+ batch_export.refresh_from_db()
+ assert batch_export.incremental_mode == BatchExport.IncrementalMode.MERGE
+ assert BatchExportSerializer(batch_export).data["incremental_mode"] == "MERGE"
+
+ serializer = BatchExportSerializer(
+ batch_export, data={"name": "Renamed export"}, partial=True, context=context
+ )
+ assert serializer.is_valid(), serializer.errors
+ serializer.save()
+ batch_export.refresh_from_db()
+ assert batch_export.incremental_mode == BatchExport.IncrementalMode.MERGE
+
+ serializer = BatchExportSerializer(
+ batch_export, data={"incremental_mode": "APPEND"}, partial=True, context=context
+ )
+ assert serializer.is_valid(), serializer.errors
+ serializer.save()
+ batch_export.refresh_from_db()
+ assert batch_export.incremental_mode == BatchExport.IncrementalMode.APPEND
+ assert BatchExportSerializer(batch_export).data["incremental_mode"] == "APPEND"
+
+ @parameterized.expand([("MERGE", False), ("APPEND", True)])
+ def test_patch_validates_keys_against_saved_export_mode(self, mode: str, valid: bool) -> None:
+ batch_export = BatchExport(
+ model=BatchExport.Model.HOGQL,
+ incremental_mode=mode,
+ source=BatchExportSource(team=self.team, hogql_query="SELECT uuid FROM events", primary_key=["uuid"]),
+ )
+ serializer = BatchExportSerializer(
+ batch_export,
+ data={"primary_key": None},
+ partial=True,
+ context={"get_team": lambda: self.team, "request": SimpleNamespace(user=self.user)},
+ )
+
+ assert serializer.is_valid() is valid, serializer.errors
+ if not valid:
+ assert "primary_key" in serializer.errors
+
+
class TestBatchExportDestinationSerializerTeamScoping(BaseTest):
def _make_integration(self, team: Team) -> Integration:
return Integration.objects.create(team=team, kind="databricks", integration_id="server")
diff --git a/products/batch_exports/frontend/generated/api.schemas.ts b/products/batch_exports/frontend/generated/api.schemas.ts
index 79e4bb218361..05836f264e9c 100644
--- a/products/batch_exports/frontend/generated/api.schemas.ts
+++ b/products/batch_exports/frontend/generated/api.schemas.ts
@@ -873,6 +873,17 @@ export interface HogQLQueryModifiersApi {
webAnalyticsFirstPageviewFilters?: boolean | null
}
+/**
+ * * `APPEND` - Append
+ * * `MERGE` - Merge
+ */
+export type IncrementalModeEnumApi = (typeof IncrementalModeEnumApi)[keyof typeof IncrementalModeEnumApi]
+
+export const IncrementalModeEnumApi = {
+ Append: 'APPEND',
+ Merge: 'MERGE',
+} as const
+
/**
* Serializer for a BatchExport model.
*/
@@ -929,6 +940,21 @@ export interface BatchExportApi {
hogql_query?: string | null
/** HogQL modifiers to use when the query runs, as an object keyed by modifier name, for example {"convertToProjectTimezone": true}. Only supported when 'model' is 'hogql'. Each modifier set here overrides the project modifier with the same name, and the project modifiers apply to all others. If neither sets convertToProjectTimezone, it defaults to false, so timestamps are exported in UTC. Set it to true to export timestamps in the project timezone instead. Unknown modifier names are rejected. On update, the object replaces the modifiers stored on the export, so include every modifier to keep, or send null to remove them all. */
hogql_modifiers?: HogQLQueryModifiersApi | null
+ /**
+ * The column or columns in the provided HogQL query that make up the primary key for a HogQL-backed batch export. Required if incremental mode is 'MERGE'.
+ * @nullable
+ */
+ primary_key?: string[] | null
+ /**
+ * The column or columns in the provided HogQL query that make up the version key for a HogQL-backed batch export.
+ * @nullable
+ */
+ version_key?: string[] | null
+ /** How this batch export handles incremental updates.
+ *
+ * * `APPEND` - Append
+ * * `MERGE` - Merge */
+ incremental_mode?: IncrementalModeEnumApi
/** A schema of custom fields to select when exporting data. */
readonly schema: unknown
filters?: unknown
@@ -1738,6 +1764,21 @@ export interface BatchExportRequestApi {
hogql_query?: string | null
/** HogQL modifiers to use when the query runs, as an object keyed by modifier name, for example {"convertToProjectTimezone": true}. Only supported when 'model' is 'hogql'. Each modifier set here overrides the project modifier with the same name, and the project modifiers apply to all others. If neither sets convertToProjectTimezone, it defaults to false, so timestamps are exported in UTC. Set it to true to export timestamps in the project timezone instead. Unknown modifier names are rejected. On update, the object replaces the modifiers stored on the export, so include every modifier to keep, or send null to remove them all. */
hogql_modifiers?: HogQLQueryModifiersApi | null
+ /**
+ * The column or columns in the provided HogQL query that make up the primary key for a HogQL-backed batch export. Required if incremental mode is 'MERGE'
+ * @nullable
+ */
+ primary_key?: string[] | null
+ /**
+ * The column or columns in the provided HogQL query that make up the version key for a HogQL-backed batch export.
+ * @nullable
+ */
+ version_key?: string[] | null
+ /** How this batch export handles incremental updates.
+ *
+ * * `APPEND` - Append
+ * * `MERGE` - Merge */
+ incremental_mode?: IncrementalModeEnumApi
/** Optional list of property filters to restrict which events are exported. Each filter is a serialized HogQL property filter object with a 'type' of one of: 'event', 'hogql', 'person' (e.g. {"key": "$browser", "operator": "exact", "type": "event", "value": ["Firefox"]}). */
filters?: unknown
/**
@@ -1904,6 +1945,21 @@ export interface PatchedBatchExportRequestApi {
hogql_query?: string | null
/** HogQL modifiers to use when the query runs, as an object keyed by modifier name, for example {"convertToProjectTimezone": true}. Only supported when 'model' is 'hogql'. Each modifier set here overrides the project modifier with the same name, and the project modifiers apply to all others. If neither sets convertToProjectTimezone, it defaults to false, so timestamps are exported in UTC. Set it to true to export timestamps in the project timezone instead. Unknown modifier names are rejected. On update, the object replaces the modifiers stored on the export, so include every modifier to keep, or send null to remove them all. */
hogql_modifiers?: HogQLQueryModifiersApi | null
+ /**
+ * The column or columns in the provided HogQL query that make up the primary key for a HogQL-backed batch export. Required if incremental mode is 'MERGE'
+ * @nullable
+ */
+ primary_key?: string[] | null
+ /**
+ * The column or columns in the provided HogQL query that make up the version key for a HogQL-backed batch export.
+ * @nullable
+ */
+ version_key?: string[] | null
+ /** How this batch export handles incremental updates.
+ *
+ * * `APPEND` - Append
+ * * `MERGE` - Merge */
+ incremental_mode?: IncrementalModeEnumApi
/** Optional list of property filters to restrict which events are exported. Each filter is a serialized HogQL property filter object with a 'type' of one of: 'event', 'hogql', 'person' (e.g. {"key": "$browser", "operator": "exact", "type": "event", "value": ["Firefox"]}). */
filters?: unknown
/**
diff --git a/products/batch_exports/frontend/generated/api.zod.ts b/products/batch_exports/frontend/generated/api.zod.ts
index 1274aaf05c6d..24bfb150c0db 100644
--- a/products/batch_exports/frontend/generated/api.zod.ts
+++ b/products/batch_exports/frontend/generated/api.zod.ts
@@ -702,6 +702,23 @@ export const BatchExportsCreateBody = /* @__PURE__ */ zod
.describe(
"HogQL modifiers to use when the query runs, as an object keyed by modifier name, for example {\"convertToProjectTimezone\": true}. Only supported when 'model' is 'hogql'. Each modifier set here overrides the project modifier with the same name, and the project modifiers apply to all others. If neither sets convertToProjectTimezone, it defaults to false, so timestamps are exported in UTC. Set it to true to export timestamps in the project timezone instead. Unknown modifier names are rejected. On update, the object replaces the modifiers stored on the export, so include every modifier to keep, or send null to remove them all."
),
+ primary_key: zod
+ .array(zod.string())
+ .nullish()
+ .describe(
+ "The column or columns in the provided HogQL query that make up the primary key for a HogQL-backed batch export. Required if incremental mode is 'MERGE'"
+ ),
+ version_key: zod
+ .array(zod.string())
+ .nullish()
+ .describe(
+ 'The column or columns in the provided HogQL query that make up the version key for a HogQL-backed batch export.'
+ ),
+ incremental_mode: zod
+ .enum(['APPEND', 'MERGE'])
+ .describe('\* `APPEND` - Append\n\* `MERGE` - Merge')
+ .optional()
+ .describe('How this batch export handles incremental updates.\n\n\* `APPEND` - Append\n\* `MERGE` - Merge'),
filters: zod
.unknown()
.optional()
@@ -1671,6 +1688,23 @@ export const BatchExportsUpdateBody = /* @__PURE__ */ zod
.describe(
"HogQL modifiers to use when the query runs, as an object keyed by modifier name, for example {\"convertToProjectTimezone\": true}. Only supported when 'model' is 'hogql'. Each modifier set here overrides the project modifier with the same name, and the project modifiers apply to all others. If neither sets convertToProjectTimezone, it defaults to false, so timestamps are exported in UTC. Set it to true to export timestamps in the project timezone instead. Unknown modifier names are rejected. On update, the object replaces the modifiers stored on the export, so include every modifier to keep, or send null to remove them all."
),
+ primary_key: zod
+ .array(zod.string())
+ .nullish()
+ .describe(
+ "The column or columns in the provided HogQL query that make up the primary key for a HogQL-backed batch export. Required if incremental mode is 'MERGE'"
+ ),
+ version_key: zod
+ .array(zod.string())
+ .nullish()
+ .describe(
+ 'The column or columns in the provided HogQL query that make up the version key for a HogQL-backed batch export.'
+ ),
+ incremental_mode: zod
+ .enum(['APPEND', 'MERGE'])
+ .describe('\* `APPEND` - Append\n\* `MERGE` - Merge')
+ .optional()
+ .describe('How this batch export handles incremental updates.\n\n\* `APPEND` - Append\n\* `MERGE` - Merge'),
filters: zod
.unknown()
.optional()
@@ -2399,6 +2433,23 @@ export const BatchExportsPartialUpdateBody = /* @__PURE__ */ zod
.describe(
"HogQL modifiers to use when the query runs, as an object keyed by modifier name, for example {\"convertToProjectTimezone\": true}. Only supported when 'model' is 'hogql'. Each modifier set here overrides the project modifier with the same name, and the project modifiers apply to all others. If neither sets convertToProjectTimezone, it defaults to false, so timestamps are exported in UTC. Set it to true to export timestamps in the project timezone instead. Unknown modifier names are rejected. On update, the object replaces the modifiers stored on the export, so include every modifier to keep, or send null to remove them all."
),
+ primary_key: zod
+ .array(zod.string())
+ .nullish()
+ .describe(
+ "The column or columns in the provided HogQL query that make up the primary key for a HogQL-backed batch export. Required if incremental mode is 'MERGE'"
+ ),
+ version_key: zod
+ .array(zod.string())
+ .nullish()
+ .describe(
+ 'The column or columns in the provided HogQL query that make up the version key for a HogQL-backed batch export.'
+ ),
+ incremental_mode: zod
+ .enum(['APPEND', 'MERGE'])
+ .describe('\* `APPEND` - Append\n\* `MERGE` - Merge')
+ .optional()
+ .describe('How this batch export handles incremental updates.\n\n\* `APPEND` - Append\n\* `MERGE` - Merge'),
filters: zod
.unknown()
.optional()
@@ -3112,6 +3163,23 @@ export const BatchExportsRunTestStepCreateBody = /* @__PURE__ */ zod
.describe(
"HogQL modifiers to use when the query runs, as an object keyed by modifier name, for example {\"convertToProjectTimezone\": true}. Only supported when 'model' is 'hogql'. Each modifier set here overrides the project modifier with the same name, and the project modifiers apply to all others. If neither sets convertToProjectTimezone, it defaults to false, so timestamps are exported in UTC. Set it to true to export timestamps in the project timezone instead. Unknown modifier names are rejected. On update, the object replaces the modifiers stored on the export, so include every modifier to keep, or send null to remove them all."
),
+ primary_key: zod
+ .array(zod.string())
+ .nullish()
+ .describe(
+ "The column or columns in the provided HogQL query that make up the primary key for a HogQL-backed batch export. Required if incremental mode is 'MERGE'."
+ ),
+ version_key: zod
+ .array(zod.string())
+ .nullish()
+ .describe(
+ 'The column or columns in the provided HogQL query that make up the version key for a HogQL-backed batch export.'
+ ),
+ incremental_mode: zod
+ .enum(['APPEND', 'MERGE'])
+ .describe('\* `APPEND` - Append\n\* `MERGE` - Merge')
+ .optional()
+ .describe('How this batch export handles incremental updates.\n\n\* `APPEND` - Append\n\* `MERGE` - Merge'),
filters: zod.unknown().optional(),
timezone: zod
.union([zod.string(), zod.null()])
@@ -3844,6 +3912,23 @@ export const BatchExportsRunTestStepNewCreateBody = /* @__PURE__ */ zod
.describe(
"HogQL modifiers to use when the query runs, as an object keyed by modifier name, for example {\"convertToProjectTimezone\": true}. Only supported when 'model' is 'hogql'. Each modifier set here overrides the project modifier with the same name, and the project modifiers apply to all others. If neither sets convertToProjectTimezone, it defaults to false, so timestamps are exported in UTC. Set it to true to export timestamps in the project timezone instead. Unknown modifier names are rejected. On update, the object replaces the modifiers stored on the export, so include every modifier to keep, or send null to remove them all."
),
+ primary_key: zod
+ .array(zod.string())
+ .nullish()
+ .describe(
+ "The column or columns in the provided HogQL query that make up the primary key for a HogQL-backed batch export. Required if incremental mode is 'MERGE'."
+ ),
+ version_key: zod
+ .array(zod.string())
+ .nullish()
+ .describe(
+ 'The column or columns in the provided HogQL query that make up the version key for a HogQL-backed batch export.'
+ ),
+ incremental_mode: zod
+ .enum(['APPEND', 'MERGE'])
+ .describe('\* `APPEND` - Append\n\* `MERGE` - Merge')
+ .optional()
+ .describe('How this batch export handles incremental updates.\n\n\* `APPEND` - Append\n\* `MERGE` - Merge'),
filters: zod.unknown().optional(),
timezone: zod
.union([zod.string(), zod.null()])
diff --git a/products/batch_exports/mcp/tools.yaml b/products/batch_exports/mcp/tools.yaml
index 194c04ee7dbc..07d931cb93d1 100644
--- a/products/batch_exports/mcp/tools.yaml
+++ b/products/batch_exports/mcp/tools.yaml
@@ -25,7 +25,9 @@ tools:
provide hogql_query. Optionally provide hogql_modifiers, which override the project HogQL modifiers with the
same names for this export only. Timestamps are exported in UTC unless the export or the project sets
convertToProjectTimezone, for example {"convertToProjectTimezone": true} to export them in the project
- timezone.
+ timezone. incremental_mode accepts APPEND (the default) or MERGE for any model. For hogql, MERGE requires a
+ primary_key. Use nonempty lists of query output column names or aliases for primary_key and the optional
+ version_key. These keys are only configurable for the hogql model.
exclude_params:
- id
- team_id
@@ -62,7 +64,7 @@ tools:
title: Get batch export
description: >
Get a batch export by ID. Returns full non-sensitive configuration including destination type and config,
- linked integration ID when present, schedule, model, and the 10 most recent runs.
+ linked integration ID when present, schedule, model, incremental_mode, and the 10 most recent runs.
batch-export-update:
operation: batch_exports_partial_update
enabled: true
@@ -75,9 +77,11 @@ tools:
title: Update batch export
description: >
Partially update a batch export. Top-level fields (name, paused, interval, timezone, offset_day,
- offset_hour) work for any destination type. For the hogql model, hogql_query and hogql_modifiers can also
- change. hogql_modifiers replaces the stored object rather than merging into it, so read the current value
- with batch-export-get and send every modifier to keep, or send null to remove them all. Timestamps are
+ offset_hour, incremental_mode) work for any destination type. incremental_mode accepts APPEND or MERGE; omit
+ it to preserve the saved mode. For the hogql model, hogql_query, hogql_modifiers, primary_key, and
+ version_key can also change. Keys use query output column names or aliases, and MERGE requires a
+ primary_key. hogql_modifiers replaces the stored object rather than merging into it, so read the current
+ value with batch-export-get and send every modifier to keep, or send null to remove them all. Timestamps are
exported in UTC unless the export or the project sets convertToProjectTimezone to true. Typed destination
config updates are available for Databricks, AzureBlob, BigQuery, Postgres, AwsS3, S3Compatible, Snowflake,
and Redshift. The destination type cannot be changed. Do not change the model field on an existing batch
@@ -103,15 +107,16 @@ tools:
list: true
title: List batch exports
description: >
- List batch exports in the project. Batch exports send event, person, or session data on a schedule to
+ List batch exports in the project. Batch exports send event, person, session data, or HogQL query results to
destinations like S3, Snowflake, BigQuery, Databricks, Azure Blob, Postgres, Redshift, or HTTP. Returns
- destination type, interval, paused state, and non-sensitive config; credentials are never exposed. Use
- batch-export-get with the ID for full details including latest runs, query, and filters.
+ destination type, incremental_mode, interval, paused state, and non-sensitive config; credentials are never
+ exposed. Use batch-export-get with the ID for full details including latest runs, query, and filters.
response:
include:
- id
- name
- model
+ - incremental_mode
- destination
- interval
- paused
diff --git a/services/mcp/schema/generated-tool-definitions.json b/services/mcp/schema/generated-tool-definitions.json
index 020afcef0f0c..cf22125eb549 100644
--- a/services/mcp/schema/generated-tool-definitions.json
+++ b/services/mcp/schema/generated-tool-definitions.json
@@ -1450,7 +1450,7 @@
"feature_flag": "autoresearch"
},
"batch-export-create": {
- "description": "Create a new batch export. This tool supports typed destination configs for Databricks, AzureBlob, BigQuery, Postgres, AwsS3, S3Compatible, Snowflake, and Redshift. All require a team-scoped integration for credentials. Use integrations-list to find one and pass its ID as destination.integration_id. Always specify the model field (events, persons, sessions, or hogql). Omitting it defaults to events. For the hogql model, provide hogql_query. Optionally provide hogql_modifiers, which override the project HogQL modifiers with the same names for this export only. Timestamps are exported in UTC unless the export or the project sets convertToProjectTimezone, for example {\"convertToProjectTimezone\": true} to export them in the project timezone.",
+ "description": "Create a new batch export. This tool supports typed destination configs for Databricks, AzureBlob, BigQuery, Postgres, AwsS3, S3Compatible, Snowflake, and Redshift. All require a team-scoped integration for credentials. Use integrations-list to find one and pass its ID as destination.integration_id. Always specify the model field (events, persons, sessions, or hogql). Omitting it defaults to events. For the hogql model, provide hogql_query. Optionally provide hogql_modifiers, which override the project HogQL modifiers with the same names for this export only. Timestamps are exported in UTC unless the export or the project sets convertToProjectTimezone, for example {\"convertToProjectTimezone\": true} to export them in the project timezone. incremental_mode accepts APPEND (the default) or MERGE for any model. For hogql, MERGE requires a primary_key. Use nonempty lists of query output column names or aliases for primary_key and the optional version_key. These keys are only configurable for the hogql model.",
"category": "Data pipelines",
"feature": "batch_exports",
"summary": "Create batch export",
@@ -1478,7 +1478,7 @@
}
},
"batch-export-get": {
- "description": "Get a batch export by ID. Returns full non-sensitive configuration including destination type and config, linked integration ID when present, schedule, model, and the 10 most recent runs.",
+ "description": "Get a batch export by ID. Returns full non-sensitive configuration including destination type and config, linked integration ID when present, schedule, model, incremental_mode, and the 10 most recent runs.",
"category": "Data pipelines",
"feature": "batch_exports",
"summary": "Get batch export",
@@ -1492,7 +1492,7 @@
}
},
"batch-export-update": {
- "description": "Partially update a batch export. Top-level fields (name, paused, interval, timezone, offset_day, offset_hour) work for any destination type. For the hogql model, hogql_query and hogql_modifiers can also change. hogql_modifiers replaces the stored object rather than merging into it, so read the current value with batch-export-get and send every modifier to keep, or send null to remove them all. Timestamps are exported in UTC unless the export or the project sets convertToProjectTimezone to true. Typed destination config updates are available for Databricks, AzureBlob, BigQuery, Postgres, AwsS3, S3Compatible, Snowflake, and Redshift. The destination type cannot be changed. Do not change the model field on an existing batch export.",
+ "description": "Partially update a batch export. Top-level fields (name, paused, interval, timezone, offset_day, offset_hour, incremental_mode) work for any destination type. incremental_mode accepts APPEND or MERGE; omit it to preserve the saved mode. For the hogql model, hogql_query, hogql_modifiers, primary_key, and version_key can also change. Keys use query output column names or aliases, and MERGE requires a primary_key. hogql_modifiers replaces the stored object rather than merging into it, so read the current value with batch-export-get and send every modifier to keep, or send null to remove them all. Timestamps are exported in UTC unless the export or the project sets convertToProjectTimezone to true. Typed destination config updates are available for Databricks, AzureBlob, BigQuery, Postgres, AwsS3, S3Compatible, Snowflake, and Redshift. The destination type cannot be changed. Do not change the model field on an existing batch export.",
"category": "Data pipelines",
"feature": "batch_exports",
"summary": "Update batch export",
@@ -1506,7 +1506,7 @@
}
},
"batch-exports-list": {
- "description": "List batch exports in the project. Batch exports send event, person, or session data on a schedule to destinations like S3, Snowflake, BigQuery, Databricks, Azure Blob, Postgres, Redshift, or HTTP. Returns destination type, interval, paused state, and non-sensitive config; credentials are never exposed. Use batch-export-get with the ID for full details including latest runs, query, and filters.",
+ "description": "List batch exports in the project. Batch exports send event, person, session data, or HogQL query results to destinations like S3, Snowflake, BigQuery, Databricks, Azure Blob, Postgres, Redshift, or HTTP. Returns destination type, incremental_mode, interval, paused state, and non-sensitive config; credentials are never exposed. Use batch-export-get with the ID for full details including latest runs, query, and filters.",
"category": "Data pipelines",
"feature": "batch_exports",
"summary": "List batch exports",
diff --git a/services/mcp/schema/tool-definitions-all.json b/services/mcp/schema/tool-definitions-all.json
index f2bdaa9bc9bd..75beb7bbdf0a 100644
--- a/services/mcp/schema/tool-definitions-all.json
+++ b/services/mcp/schema/tool-definitions-all.json
@@ -1465,7 +1465,7 @@
"feature_flag": "autoresearch"
},
"batch-export-create": {
- "description": "Create a new batch export. This tool supports typed destination configs for Databricks, AzureBlob, BigQuery, Postgres, AwsS3, S3Compatible, Snowflake, and Redshift. All require a team-scoped integration for credentials. Use integrations-list to find one and pass its ID as destination.integration_id. Always specify the model field (events, persons, sessions, or hogql). Omitting it defaults to events. For the hogql model, provide hogql_query. Optionally provide hogql_modifiers, which override the project HogQL modifiers with the same names for this export only. Timestamps are exported in UTC unless the export or the project sets convertToProjectTimezone, for example {\"convertToProjectTimezone\": true} to export them in the project timezone.",
+ "description": "Create a new batch export. This tool supports typed destination configs for Databricks, AzureBlob, BigQuery, Postgres, AwsS3, S3Compatible, Snowflake, and Redshift. All require a team-scoped integration for credentials. Use integrations-list to find one and pass its ID as destination.integration_id. Always specify the model field (events, persons, sessions, or hogql). Omitting it defaults to events. For the hogql model, provide hogql_query. Optionally provide hogql_modifiers, which override the project HogQL modifiers with the same names for this export only. Timestamps are exported in UTC unless the export or the project sets convertToProjectTimezone, for example {\"convertToProjectTimezone\": true} to export them in the project timezone. incremental_mode accepts APPEND (the default) or MERGE for any model. For hogql, MERGE requires a primary_key. Use nonempty lists of query output column names or aliases for primary_key and the optional version_key. These keys are only configurable for the hogql model.",
"category": "Data pipelines",
"feature": "batch_exports",
"summary": "Create batch export",
@@ -1493,7 +1493,7 @@
}
},
"batch-export-get": {
- "description": "Get a batch export by ID. Returns full non-sensitive configuration including destination type and config, linked integration ID when present, schedule, model, and the 10 most recent runs.",
+ "description": "Get a batch export by ID. Returns full non-sensitive configuration including destination type and config, linked integration ID when present, schedule, model, incremental_mode, and the 10 most recent runs.",
"category": "Data pipelines",
"feature": "batch_exports",
"summary": "Get batch export",
@@ -1507,7 +1507,7 @@
}
},
"batch-export-update": {
- "description": "Partially update a batch export. Top-level fields (name, paused, interval, timezone, offset_day, offset_hour) work for any destination type. For the hogql model, hogql_query and hogql_modifiers can also change. hogql_modifiers replaces the stored object rather than merging into it, so read the current value with batch-export-get and send every modifier to keep, or send null to remove them all. Timestamps are exported in UTC unless the export or the project sets convertToProjectTimezone to true. Typed destination config updates are available for Databricks, AzureBlob, BigQuery, Postgres, AwsS3, S3Compatible, Snowflake, and Redshift. The destination type cannot be changed. Do not change the model field on an existing batch export.",
+ "description": "Partially update a batch export. Top-level fields (name, paused, interval, timezone, offset_day, offset_hour, incremental_mode) work for any destination type. incremental_mode accepts APPEND or MERGE; omit it to preserve the saved mode. For the hogql model, hogql_query, hogql_modifiers, primary_key, and version_key can also change. Keys use query output column names or aliases, and MERGE requires a primary_key. hogql_modifiers replaces the stored object rather than merging into it, so read the current value with batch-export-get and send every modifier to keep, or send null to remove them all. Timestamps are exported in UTC unless the export or the project sets convertToProjectTimezone to true. Typed destination config updates are available for Databricks, AzureBlob, BigQuery, Postgres, AwsS3, S3Compatible, Snowflake, and Redshift. The destination type cannot be changed. Do not change the model field on an existing batch export.",
"category": "Data pipelines",
"feature": "batch_exports",
"summary": "Update batch export",
@@ -1521,7 +1521,7 @@
}
},
"batch-exports-list": {
- "description": "List batch exports in the project. Batch exports send event, person, or session data on a schedule to destinations like S3, Snowflake, BigQuery, Databricks, Azure Blob, Postgres, Redshift, or HTTP. Returns destination type, interval, paused state, and non-sensitive config; credentials are never exposed. Use batch-export-get with the ID for full details including latest runs, query, and filters.",
+ "description": "List batch exports in the project. Batch exports send event, person, session data, or HogQL query results to destinations like S3, Snowflake, BigQuery, Databricks, Azure Blob, Postgres, Redshift, or HTTP. Returns destination type, incremental_mode, interval, paused state, and non-sensitive config; credentials are never exposed. Use batch-export-get with the ID for full details including latest runs, query, and filters.",
"category": "Data pipelines",
"feature": "batch_exports",
"summary": "List batch exports",
diff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.ts
index 010c7dc20fd4..d6a79528badd 100644
--- a/services/mcp/src/api/generated.ts
+++ b/services/mcp/src/api/generated.ts
@@ -16266,6 +16266,18 @@ export namespace Schemas {
backfill?: string | null;
}
+ /**
+ * * `APPEND` - Append
+ * * `MERGE` - Merge
+ */
+ export type IncrementalModeEnum = typeof IncrementalModeEnum[keyof typeof IncrementalModeEnum];
+
+
+ export const IncrementalModeEnum = {
+ Append: 'APPEND',
+ Merge: 'MERGE',
+ } as const;
+
/**
* Serializer for a BatchExport model.
*/
@@ -16322,6 +16334,21 @@ export namespace Schemas {
hogql_query?: string | null;
/** HogQL modifiers to use when the query runs, as an object keyed by modifier name, for example {"convertToProjectTimezone": true}. Only supported when 'model' is 'hogql'. Each modifier set here overrides the project modifier with the same name, and the project modifiers apply to all others. If neither sets convertToProjectTimezone, it defaults to false, so timestamps are exported in UTC. Set it to true to export timestamps in the project timezone instead. Unknown modifier names are rejected. On update, the object replaces the modifiers stored on the export, so include every modifier to keep, or send null to remove them all. */
hogql_modifiers?: HogQLQueryModifiers | null;
+ /**
+ * The column or columns in the provided HogQL query that make up the primary key for a HogQL-backed batch export. Required if incremental mode is 'MERGE'.
+ * @nullable
+ */
+ primary_key?: string[] | null;
+ /**
+ * The column or columns in the provided HogQL query that make up the version key for a HogQL-backed batch export.
+ * @nullable
+ */
+ version_key?: string[] | null;
+ /** How this batch export handles incremental updates.
+ *
+ * * `APPEND` - Append
+ * * `MERGE` - Merge */
+ incremental_mode?: IncrementalModeEnum;
/** A schema of custom fields to select when exporting data. */
readonly schema: unknown;
filters?: unknown;
@@ -17171,6 +17198,21 @@ export namespace Schemas {
hogql_query?: string | null;
/** HogQL modifiers to use when the query runs, as an object keyed by modifier name, for example {"convertToProjectTimezone": true}. Only supported when 'model' is 'hogql'. Each modifier set here overrides the project modifier with the same name, and the project modifiers apply to all others. If neither sets convertToProjectTimezone, it defaults to false, so timestamps are exported in UTC. Set it to true to export timestamps in the project timezone instead. Unknown modifier names are rejected. On update, the object replaces the modifiers stored on the export, so include every modifier to keep, or send null to remove them all. */
hogql_modifiers?: HogQLQueryModifiers | null;
+ /**
+ * The column or columns in the provided HogQL query that make up the primary key for a HogQL-backed batch export. Required if incremental mode is 'MERGE'
+ * @nullable
+ */
+ primary_key?: string[] | null;
+ /**
+ * The column or columns in the provided HogQL query that make up the version key for a HogQL-backed batch export.
+ * @nullable
+ */
+ version_key?: string[] | null;
+ /** How this batch export handles incremental updates.
+ *
+ * * `APPEND` - Append
+ * * `MERGE` - Merge */
+ incremental_mode?: IncrementalModeEnum;
/** Optional list of property filters to restrict which events are exported. Each filter is a serialized HogQL property filter object with a 'type' of one of: 'event', 'hogql', 'person' (e.g. {"key": "$browser", "operator": "exact", "type": "event", "value": ["Firefox"]}). */
filters?: unknown;
/**
@@ -79224,6 +79266,21 @@ export namespace Schemas {
hogql_query?: string | null;
/** HogQL modifiers to use when the query runs, as an object keyed by modifier name, for example {"convertToProjectTimezone": true}. Only supported when 'model' is 'hogql'. Each modifier set here overrides the project modifier with the same name, and the project modifiers apply to all others. If neither sets convertToProjectTimezone, it defaults to false, so timestamps are exported in UTC. Set it to true to export timestamps in the project timezone instead. Unknown modifier names are rejected. On update, the object replaces the modifiers stored on the export, so include every modifier to keep, or send null to remove them all. */
hogql_modifiers?: HogQLQueryModifiers | null;
+ /**
+ * The column or columns in the provided HogQL query that make up the primary key for a HogQL-backed batch export. Required if incremental mode is 'MERGE'
+ * @nullable
+ */
+ primary_key?: string[] | null;
+ /**
+ * The column or columns in the provided HogQL query that make up the version key for a HogQL-backed batch export.
+ * @nullable
+ */
+ version_key?: string[] | null;
+ /** How this batch export handles incremental updates.
+ *
+ * * `APPEND` - Append
+ * * `MERGE` - Merge */
+ incremental_mode?: IncrementalModeEnum;
/** Optional list of property filters to restrict which events are exported. Each filter is a serialized HogQL property filter object with a 'type' of one of: 'event', 'hogql', 'person' (e.g. {"key": "$browser", "operator": "exact", "type": "event", "value": ["Firefox"]}). */
filters?: unknown;
/**
diff --git a/services/mcp/src/generated/batch_exports/api.ts b/services/mcp/src/generated/batch_exports/api.ts
index 5993dfee36db..6afad9096d0c 100644
--- a/services/mcp/src/generated/batch_exports/api.ts
+++ b/services/mcp/src/generated/batch_exports/api.ts
@@ -714,6 +714,23 @@ export const BatchExportsCreateBody = () => zod
.describe(
"HogQL modifiers to use when the query runs, as an object keyed by modifier name, for example {\"convertToProjectTimezone\": true}. Only supported when 'model' is 'hogql'. Each modifier set here overrides the project modifier with the same name, and the project modifiers apply to all others. If neither sets convertToProjectTimezone, it defaults to false, so timestamps are exported in UTC. Set it to true to export timestamps in the project timezone instead. Unknown modifier names are rejected. On update, the object replaces the modifiers stored on the export, so include every modifier to keep, or send null to remove them all."
),
+ primary_key: zod
+ .array(zod.string())
+ .nullish()
+ .describe(
+ "The column or columns in the provided HogQL query that make up the primary key for a HogQL-backed batch export. Required if incremental mode is 'MERGE'"
+ ),
+ version_key: zod
+ .array(zod.string())
+ .nullish()
+ .describe(
+ 'The column or columns in the provided HogQL query that make up the version key for a HogQL-backed batch export.'
+ ),
+ incremental_mode: zod
+ .enum(['APPEND', 'MERGE'])
+ .describe('\* `APPEND` - Append\n\* `MERGE` - Merge')
+ .optional()
+ .describe('How this batch export handles incremental updates.\n\n\* `APPEND` - Append\n\* `MERGE` - Merge'),
timezone: zod
.string()
.nullish()
@@ -1446,6 +1463,23 @@ export const BatchExportsPartialUpdateBody = () => zod
.describe(
"HogQL modifiers to use when the query runs, as an object keyed by modifier name, for example {\"convertToProjectTimezone\": true}. Only supported when 'model' is 'hogql'. Each modifier set here overrides the project modifier with the same name, and the project modifiers apply to all others. If neither sets convertToProjectTimezone, it defaults to false, so timestamps are exported in UTC. Set it to true to export timestamps in the project timezone instead. Unknown modifier names are rejected. On update, the object replaces the modifiers stored on the export, so include every modifier to keep, or send null to remove them all."
),
+ primary_key: zod
+ .array(zod.string())
+ .nullish()
+ .describe(
+ "The column or columns in the provided HogQL query that make up the primary key for a HogQL-backed batch export. Required if incremental mode is 'MERGE'"
+ ),
+ version_key: zod
+ .array(zod.string())
+ .nullish()
+ .describe(
+ 'The column or columns in the provided HogQL query that make up the version key for a HogQL-backed batch export.'
+ ),
+ incremental_mode: zod
+ .enum(['APPEND', 'MERGE'])
+ .describe('\* `APPEND` - Append\n\* `MERGE` - Merge')
+ .optional()
+ .describe('How this batch export handles incremental updates.\n\n\* `APPEND` - Append\n\* `MERGE` - Merge'),
timezone: zod
.string()
.nullish()
diff --git a/services/mcp/src/tools/generated/batch_exports.ts b/services/mcp/src/tools/generated/batch_exports.ts
index 6166efa3527c..9632fa38191b 100644
--- a/services/mcp/src/tools/generated/batch_exports.ts
+++ b/services/mcp/src/tools/generated/batch_exports.ts
@@ -38,6 +38,15 @@ const batchExportCreate = (): ToolBase<ReturnType<typeof BatchExportCreateSchema
if (params.hogql_modifiers !== undefined) {
body['hogql_modifiers'] = params.hogql_modifiers
}
+ if (params.primary_key !== undefined) {
+ body['primary_key'] = params.primary_key
+ }
+ if (params.version_key !== undefined) {
+ body['version_key'] = params.version_key
+ }
+ if (params.incremental_mode !== undefined) {
+ body['incremental_mode'] = params.incremental_mode
+ }
if (params.timezone !== undefined) {
body['timezone'] = params.timezone
}
@@ -125,6 +134,15 @@ const batchExportUpdate = (): ToolBase<ReturnType<typeof BatchExportUpdateSchema
if (params.hogql_modifiers !== undefined) {
body['hogql_modifiers'] = params.hogql_modifiers
}
+ if (params.primary_key !== undefined) {
+ body['primary_key'] = params.primary_key
+ }
+ if (params.version_key !== undefined) {
+ body['version_key'] = params.version_key
+ }
+ if (params.incremental_mode !== undefined) {
+ body['incremental_mode'] = params.incremental_mode
+ }
if (params.timezone !== undefined) {
body['timezone'] = params.timezone
}
@@ -171,6 +189,7 @@ const batchExportsList = (): ToolBase<
'id',
'name',
'model',
+ 'incremental_mode',
'destination',
'interval',
'paused',
diff --git a/services/mcp/tests/unit/__snapshots__/tool-schemas/batch-export-create.json b/services/mcp/tests/unit/__snapshots__/tool-schemas/batch-export-create.json
index 700b3f33f961..5d27745681e1 100644
--- a/services/mcp/tests/unit/__snapshots__/tool-schemas/batch-export-create.json
+++ b/services/mcp/tests/unit/__snapshots__/tool-schemas/batch-export-create.json
@@ -1160,6 +1160,11 @@
],
"description": "HogQL SELECT query. With model 'hogql', its results are the data exported by every run. The query may reference the {data_interval_start} and {data_interval_end} placeholders, replaced with each run's data interval bounds, for example: WHERE timestamp >= {data_interval_start} AND timestamp < {data_interval_end}. Without them every run exports all rows the query returns. Required when model is 'hogql'."
},
+ "incremental_mode": {
+ "description": "How this batch export handles incremental updates.\n\n* `APPEND` - Append\n* `MERGE` - Merge",
+ "enum": ["APPEND", "MERGE"],
+ "type": "string"
+ },
"interval": {
"description": "How often the batch export should run.\n\n* `hour` - hour\n* `day` - day\n* `week` - week\n* `every 5 minutes` - every 5 minutes\n* `every 15 minutes` - every 15 minutes",
"enum": ["hour", "day", "week", "every 5 minutes", "every 15 minutes"],
@@ -1204,6 +1209,20 @@
"description": "Whether the batch export is paused.",
"type": "boolean"
},
+ "primary_key": {
+ "anyOf": [
+ {
+ "items": {
+ "type": "string"
+ },
+ "type": "array"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "description": "The column or columns in the provided HogQL query that make up the primary key for a HogQL-backed batch export. Required if incremental mode is 'MERGE'"
+ },
"timezone": {
"anyOf": [
{
@@ -1214,6 +1233,20 @@
}
],
"description": "IANA timezone name (e.g. 'America/New_York', 'Europe/London', 'UTC') controlling daily and weekly interval boundaries."
+ },
+ "version_key": {
+ "anyOf": [
+ {
+ "items": {
+ "type": "string"
+ },
+ "type": "array"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "description": "The column or columns in the provided HogQL query that make up the version key for a HogQL-backed batch export."
}
},
"required": ["name", "destination", "interval"],
diff --git a/services/mcp/tests/unit/__snapshots__/tool-schemas/batch-export-update.json b/services/mcp/tests/unit/__snapshots__/tool-schemas/batch-export-update.json
index 58e7429d417c..e8014b818963 100644
--- a/services/mcp/tests/unit/__snapshots__/tool-schemas/batch-export-update.json
+++ b/services/mcp/tests/unit/__snapshots__/tool-schemas/batch-export-update.json
@@ -1163,6 +1163,11 @@
"description": "A UUID string identifying this batch export.",
"type": "string"
},
+ "incremental_mode": {
+ "description": "How this batch export handles incremental updates.\n\n* `APPEND` - Append\n* `MERGE` - Merge",
+ "enum": ["APPEND", "MERGE"],
+ "type": "string"
+ },
"interval": {
"description": "How often the batch export should run.\n\n* `hour` - hour\n* `day` - day\n* `week` - week\n* `every 5 minutes` - every 5 minutes\n* `every 15 minutes` - every 15 minutes",
"enum": ["hour", "day", "week", "every 5 minutes", "every 15 minutes"],
@@ -1207,6 +1212,20 @@
"description": "Whether the batch export is paused.",
"type": "boolean"
},
+ "primary_key": {
+ "anyOf": [
+ {
+ "items": {
+ "type": "string"
+ },
+ "type": "array"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "description": "The column or columns in the provided HogQL query that make up the primary key for a HogQL-backed batch export. Required if incremental mode is 'MERGE'"
+ },
"timezone": {
"anyOf": [
{
@@ -1217,6 +1236,20 @@
}
],
"description": "IANA timezone name (e.g. 'America/New_York', 'Europe/London', 'UTC') controlling daily and weekly interval boundaries."
+ },
+ "version_key": {
+ "anyOf": [
+ {
+ "items": {
+ "type": "string"
+ },
+ "type": "array"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "description": "The column or columns in the provided HogQL query that make up the version key for a HogQL-backed batch export."
}
},
"required": ["id"],
diff --git a/services/mcp/tests/unit/batch-exports-generated.test.ts b/services/mcp/tests/unit/batch-exports-generated.test.ts
index 603b3b6f1115..33f3485f1beb 100644
--- a/services/mcp/tests/unit/batch-exports-generated.test.ts
+++ b/services/mcp/tests/unit/batch-exports-generated.test.ts
@@ -7,13 +7,14 @@ describe('Generated batch export tools', () => {
it.each([
['batch-export-create', 'POST'],
['batch-export-update', 'PATCH'],
- ])('%s forwards the HogQL query', async (name, method) => {
+ ])('%s forwards the HogQL query and incremental settings', async (name, method) => {
const request = vi.fn().mockResolvedValue({})
const context = {
api: { request },
stateManager: { getProjectId: async () => '42' },
} as unknown as Context
- const hogqlQuery = 'SELECT event FROM events WHERE timestamp < {data_interval_end}'
+ const hogqlQuery =
+ 'SELECT uuid AS id, timestamp AS updated_at FROM events WHERE timestamp < {data_interval_end}'
const tool = GENERATED_TOOLS[name]!()
const params = tool.schema.parse({
id: '00000000-0000-4000-8000-000000000001',
@@ -22,12 +23,23 @@ describe('Generated batch export tools', () => {
interval: 'hour',
destination: { type: 'BigQuery', integration_id: 1, config: { dataset_id: 'exports' } },
hogql_query: hogqlQuery,
+ primary_key: ['id'],
+ version_key: ['updated_at'],
+ incremental_mode: 'MERGE',
})
await tool.handler(context, params)
expect(request).toHaveBeenCalledWith(
- expect.objectContaining({ method, body: expect.objectContaining({ hogql_query: hogqlQuery }) })
+ expect.objectContaining({
+ method,
+ body: expect.objectContaining({
+ hogql_query: hogqlQuery,
+ primary_key: ['id'],
+ version_key: ['updated_at'],
+ incremental_mode: 'MERGE',
+ }),
+ })
)
})
})