Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(warehouse): add depot job metrics, artifacts and failure diagnosis tables (#119245)

Tom Owersf38aa935a9e2
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index 61b96721af18..900437b9bef8 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -3683,14 +3683,14 @@ Note: The v2 Deploy API is tiny — 15 GET operations. Everything queryable as a
## Depot — gaps
-Today (1): `job_attempts`
+Today (4): `artifacts`, `failure_diagnoses`, `job_attempts`, `job_metrics`
Diffed against: <https://depot.dev/openapi/depot-ci/v1.json>
-- [ ] `Job metrics (POST /depot.ci.v1.CIService/GetJobMetrics)` — measure CPU and memory use across attempts to identify inefficient jobs and improve runner sizing (high)
+- [x] `Job metrics (POST /depot.ci.v1.CIService/GetJobMetrics)` — measure CPU and memory use across attempts to identify inefficient jobs and improve runner sizing (high)
- [ ] `Job attempt logs (POST /depot.ci.v1.CIService/GetJobAttemptLogs)` — analyze failure messages and workflow step activity across imported attempts (medium)
-- [ ] `Failure diagnosis (POST /depot.ci.v1.CIService/GetFailureDiagnosis)` — group recurring failures by fingerprint and compare causes across runs, workflows, and attempts (medium)
-- [ ] `Artifacts (POST /depot.ci.v1.CIService/ListArtifacts)` — track artifact counts, sizes, and production times by workflow and job (medium)
+- [x] `Failure diagnosis (POST /depot.ci.v1.CIService/GetFailureDiagnosis)` — group recurring failures by fingerprint and compare causes across runs, workflows, and attempts (medium)
+- [x] `Artifacts (POST /depot.ci.v1.CIService/ListArtifacts)` — track artifact counts, sizes, and production times by workflow and job (medium)
Note: Parsed the official CI OpenAPI document and read the operation descriptions. Depot uses POST for read RPCs; job_attempts already includes run, workflow, job, and attempt metadata. Missing metrics, logs, diagnoses, and artifact collections need parent-ID fan-out; GetJobAttemptMetrics offers an alternative for individual attempts. Secrets, variables, downloads, and execution commands were excluded; the audit covers Depot CI.
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/depot/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/depot/canonical_descriptions.py
index ec5b79b8ec93..9c5f4d20f5fb 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/depot/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/depot/canonical_descriptions.py
@@ -42,4 +42,80 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"sandbox_id": "Identifier of the sandbox that ran the attempt.",
},
},
+ "job_metrics": {
+ "description": (
+ "One row per attempt of a Depot CI job, with CPU and memory statistics from the sandbox that ran it. "
+ "Jobs that never ran have no rows."
+ ),
+ "docs_url": "https://depot.dev/docs/api/ci/reference",
+ "columns": {
+ "run_id": "Unique identifier of the Depot CI run.",
+ "run_created_at": "Time the run was created, as an RFC 3339 timestamp.",
+ "workflow_id": "Unique identifier of the workflow within the run.",
+ "workflow_name": "Name of the workflow.",
+ "workflow_path": "Path of the workflow file in the repository.",
+ "job_id": "Unique identifier of the job.",
+ "job_key": "Key of the job from the workflow.",
+ "job_status": "Status of the job.",
+ "job_conclusion": "Conclusion of the job: success, failure, cancelled, skipped or timed_out.",
+ "snapshot_at": "Time Depot computed the metrics, as an RFC 3339 timestamp.",
+ "attempt_id": "Unique identifier of the job attempt.",
+ "attempt": "Attempt number of the job.",
+ "attempt_status": "Status of the attempt.",
+ "attempt_conclusion": "Conclusion of the attempt: success, failure, cancelled, skipped or timed_out.",
+ "attempt_started_at": "Time the attempt started, as an RFC 3339 timestamp.",
+ "attempt_finished_at": "Time the attempt finished, as an RFC 3339 timestamp.",
+ "sandbox_id": "Identifier of the sandbox that ran the attempt.",
+ "availability": "Why metrics are present or not, for example available, no_sandbox, no_time_range or no_samples.",
+ "sample_count": "Number of CPU and memory samples Depot observed for the attempt.",
+ "peak_cpu_utilization": "Peak CPU utilization, as a fraction from 0 to 1.",
+ "average_cpu_utilization": "Average CPU utilization, as a fraction from 0 to 1.",
+ "peak_memory_utilization": "Peak memory utilization, as a fraction from 0 to 1.",
+ "average_memory_utilization": "Average memory utilization, as a fraction from 0 to 1.",
+ "peak_memory_usage_bytes": "Peak memory usage in bytes.",
+ "average_memory_usage_bytes": "Average memory usage in bytes.",
+ "observed_started_at": "Time of the first sample, as an RFC 3339 timestamp.",
+ "observed_finished_at": "Time of the last sample, as an RFC 3339 timestamp.",
+ },
+ },
+ "artifacts": {
+ "description": "One row per artifact that a Depot CI job attempt uploaded. Download URLs are not synced.",
+ "docs_url": "https://depot.dev/docs/api/ci/reference",
+ "columns": {
+ "artifact_id": "Unique identifier of the artifact.",
+ "run_id": "Unique identifier of the Depot CI run.",
+ "run_created_at": "Time the run was created, as an RFC 3339 timestamp.",
+ "workflow_id": "Unique identifier of the workflow that produced the artifact.",
+ "workflow_path": "Path of the workflow file in the repository.",
+ "job_id": "Unique identifier of the job that produced the artifact.",
+ "job_key": "Key of the job from the workflow.",
+ "attempt_id": "Unique identifier of the job attempt that produced the artifact.",
+ "attempt": "Attempt number that produced the artifact.",
+ "name": "Artifact name from the workflow.",
+ "size_bytes": "Size of the artifact in bytes.",
+ "created_at": "Time the artifact was created, as an RFC 3339 timestamp.",
+ },
+ },
+ "failure_diagnoses": {
+ "description": (
+ "One row per failure group in a failed or cancelled Depot CI run. Depot groups failures whose errors "
+ "match after it removes variable details such as timestamps and IDs. A run with too many failed jobs "
+ "to diagnose one by one has no rows."
+ ),
+ "docs_url": "https://depot.dev/docs/api/ci/reference",
+ "columns": {
+ "run_id": "Unique identifier of the Depot CI run.",
+ "run_created_at": "Time the run was created, as an RFC 3339 timestamp.",
+ "run_status": "Terminal status of the run: failed or cancelled.",
+ "fingerprint": "Identifier of the failure group, computed from the normalized error text.",
+ "source": "Origin of the error text: attempt_error, job_error, workflow_error or status.",
+ "count": "Number of attempts in the failure group.",
+ "error_message": "Representative error message of the group, possibly truncated.",
+ "error_message_truncated": "True when Depot truncated the error message.",
+ "diagnosis": "AI-generated summary of the failure. Empty when not available.",
+ "possible_fix": "AI-generated suggested fix. Empty when not available.",
+ "representatives": "Sample attempts of the failure, with their workflow, job, error, and relevant log lines.",
+ "omitted_representative_count": "Number of attempts in the group that are not in representatives.",
+ },
+ },
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/depot/depot.py b/products/warehouse_sources/backend/temporal/data_imports/sources/depot/depot.py
index f5a9576d524e..623eb895fc46 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/depot/depot.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/depot/depot.py
@@ -12,8 +12,11 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.htt
)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.depot.settings import (
+ ARTIFACTS,
+ FAILURE_DIAGNOSES,
JOB_ATTEMPTS,
- PRIMARY_KEY,
+ JOB_METRICS,
+ PRIMARY_KEYS,
RUN_CREATED_AT,
)
@@ -26,6 +29,9 @@ REQUEST_TIMEOUT_SECONDS = 60
# until Depot fixes its cursor. Depot caps pages at 100.
LIST_RUNS_PAGE_SIZES = (100, 57)
TERMINAL_STATUSES = ["finished", "failed", "cancelled"]
+# A finished run has no failed or cancelled job left to diagnose.
+DIAGNOSED_RUN_STATUSES = frozenset({"failed", "cancelled"})
+LIST_ARTIFACTS_PAGE_SIZE = 500
# Connect sends every RPC as a POST. Every RPC this source calls is a read, so a retry is as safe as
# a retried GET.
_DEPOT_RETRY = DEFAULT_RETRY.new(allowed_methods=frozenset(DEFAULT_RETRY.allowed_methods or ()) | {"POST"})
@@ -148,13 +154,145 @@ def _run_attempt_rows(session: Session, run: JSONObject) -> list[JSONObject]:
]
+def _int_or_none(value: int | str | None) -> int | None:
+ # Connect encodes int64 fields as JSON strings.
+ return int(value) if value is not None and value != "" else None
+
+
+def _run_job_metrics_batches(session: Session, run: JSONObject) -> Iterator[list[JSONObject]]:
+ # GetRunMetrics fails with ResourceExhausted for a run with many attempts, so metrics are read per job.
+ workflows = _call(session, "GetRunStatus", {"runId": run["runId"]}).get("workflows", [])
+ for workflow in workflows:
+ for job in workflow.get("jobs", []):
+ if not job.get("attempts"):
+ continue
+ metrics = _call(session, "GetJobMetrics", {"jobId": job["jobId"]})
+ workflow_context = metrics.get("workflow") or {}
+ job_context = metrics.get("job") or {}
+ shared_columns = {
+ "run_id": run["runId"],
+ "run_created_at": run["createdAt"],
+ "workflow_id": workflow["workflowId"],
+ "workflow_name": workflow_context.get("name"),
+ "workflow_path": workflow_context.get("workflowPath"),
+ "job_id": job["jobId"],
+ "job_key": job_context.get("jobKey"),
+ "job_status": job_context.get("status"),
+ "job_conclusion": job_context.get("conclusion"),
+ "snapshot_at": metrics.get("snapshotAt"),
+ }
+ rows: list[JSONObject] = []
+ for summary in metrics.get("attempts", []):
+ attempt = summary.get("attempt") or {}
+ availability = summary.get("availability") or {}
+ stats = summary.get("stats") or {}
+ rows.append(
+ {
+ **shared_columns,
+ "attempt_id": attempt["attemptId"],
+ "attempt": attempt.get("attempt"),
+ "attempt_status": attempt.get("status"),
+ "attempt_conclusion": attempt.get("conclusion"),
+ "attempt_started_at": attempt.get("startedAt"),
+ "attempt_finished_at": attempt.get("finishedAt"),
+ "sandbox_id": attempt.get("sandboxId"),
+ "availability": availability.get("reason"),
+ "sample_count": stats.get("sampleCount"),
+ "peak_cpu_utilization": stats.get("peakCpuUtilization"),
+ "average_cpu_utilization": stats.get("averageCpuUtilization"),
+ "peak_memory_utilization": stats.get("peakMemoryUtilization"),
+ "average_memory_utilization": stats.get("averageMemoryUtilization"),
+ "peak_memory_usage_bytes": _int_or_none(stats.get("peakMemoryUsageBytes")),
+ "average_memory_usage_bytes": stats.get("averageMemoryUsageBytes"),
+ "observed_started_at": stats.get("observedStartedAt"),
+ "observed_finished_at": stats.get("observedFinishedAt"),
+ }
+ )
+ if rows:
+ yield rows
+
+
+def _run_artifact_batches(session: Session, run: JSONObject) -> Iterator[list[JSONObject]]:
+ body: JSONObject = {"runId": run["runId"], "pageSize": LIST_ARTIFACTS_PAGE_SIZE}
+ while True:
+ page = _call(session, "ListArtifacts", body)
+ rows = [
+ {
+ "artifact_id": artifact["artifactId"],
+ "run_id": run["runId"],
+ "run_created_at": run["createdAt"],
+ "workflow_id": artifact.get("workflowId"),
+ "workflow_path": artifact.get("workflowPath"),
+ "job_id": artifact.get("jobId"),
+ "job_key": artifact.get("jobKey"),
+ "attempt_id": artifact.get("attemptId"),
+ "attempt": artifact.get("attempt"),
+ "name": artifact.get("name"),
+ "size_bytes": _int_or_none(artifact.get("sizeBytes")),
+ "created_at": artifact.get("createdAt"),
+ }
+ for artifact in page.get("artifacts", [])
+ ]
+ if rows:
+ yield rows
+ next_page_token = page.get("nextPageToken")
+ if not next_page_token:
+ return
+ body = {**body, "pageToken": next_page_token}
+
+
+def _run_failure_diagnosis_batches(session: Session, run: JSONObject) -> Iterator[list[JSONObject]]:
+ if run.get("status") not in DIAGNOSED_RUN_STATUSES:
+ return
+ diagnosis = _call(
+ session,
+ "GetFailureDiagnosis",
+ {"targetId": run["runId"], "targetType": "FAILURE_DIAGNOSIS_TARGET_TYPE_RUN"},
+ )
+ rows = [
+ {
+ "run_id": run["runId"],
+ "run_created_at": run["createdAt"],
+ "run_status": run.get("status"),
+ "fingerprint": group["fingerprint"],
+ "source": group.get("source"),
+ "count": group.get("count"),
+ "error_message": group.get("errorMessage"),
+ "error_message_truncated": group.get("errorMessageTruncated"),
+ "diagnosis": group.get("diagnosis"),
+ "possible_fix": group.get("possibleFix"),
+ "representatives": group.get("representatives", []),
+ "omitted_representative_count": group.get("omittedRepresentativeCount"),
+ }
+ for group in diagnosis.get("failureGroups", [])
+ ]
+ if rows:
+ yield rows
+
+
+def _run_job_attempt_batches(session: Session, run: JSONObject) -> Iterator[list[JSONObject]]:
+ rows = _run_attempt_rows(session, run)
+ if rows:
+ yield rows
+
+
+_RUN_BATCHES: dict[str, Callable[[Session, JSONObject], Iterator[list[JSONObject]]]] = {
+ JOB_ATTEMPTS: _run_job_attempt_batches,
+ JOB_METRICS: _run_job_metrics_batches,
+ ARTIFACTS: _run_artifact_batches,
+ FAILURE_DIAGNOSES: _run_failure_diagnosis_batches,
+}
+
+
def depot_source(
api_token: str,
repository: str,
created_after: dt.datetime | str | None,
logger: FilteringBoundLogger,
- on_complete: Callable[[], None] | None = None,
+ on_complete: Callable[[str | None], None] | None = None,
+ endpoint: str = JOB_ATTEMPTS,
) -> SourceResponse:
+ run_batches = _RUN_BATCHES[endpoint]
lower_bound = _parse_timestamp(created_after) if created_after is not None else None
def items() -> Iterator[list[JSONObject]]:
@@ -163,20 +301,19 @@ def depot_source(
runs = [run for _, run in reversed(_list_runs(session, repository, TERMINAL_STATUSES, lower_bound))]
logger.info(
"depot_ci.runs_to_sync",
+ endpoint=endpoint,
run_count=len(runs),
created_after=lower_bound.isoformat() if lower_bound else None,
)
for run in runs:
- rows = _run_attempt_rows(session, run)
- if rows:
- yield rows
+ yield from run_batches(session, run)
if on_complete is not None:
- on_complete()
+ on_complete(runs[0]["createdAt"] if runs else None)
return SourceResponse(
- name=JOB_ATTEMPTS,
+ name=endpoint,
items=items,
- primary_keys=[PRIMARY_KEY],
+ primary_keys=PRIMARY_KEYS[endpoint],
partition_mode="datetime",
partition_format="week",
partition_keys=[RUN_CREATED_AT],
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/depot/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/depot/settings.py
index c40fcba8f953..81a47727a6f7 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/depot/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/depot/settings.py
@@ -1,21 +1,33 @@
from products.warehouse_sources.backend.types import IncrementalField, IncrementalFieldType
JOB_ATTEMPTS = "job_attempts"
+JOB_METRICS = "job_metrics"
+ARTIFACTS = "artifacts"
+FAILURE_DIAGNOSES = "failure_diagnoses"
-ENDPOINTS = (JOB_ATTEMPTS,)
+ENDPOINTS = (JOB_ATTEMPTS, JOB_METRICS, ARTIFACTS, FAILURE_DIAGNOSES)
-PRIMARY_KEY = "attempt_id"
+PRIMARY_KEYS: dict[str, list[str]] = {
+ JOB_ATTEMPTS: ["attempt_id"],
+ JOB_METRICS: ["attempt_id"],
+ ARTIFACTS: ["artifact_id"],
+ # Depot computes the fingerprint per run, so the same failure in two runs has two rows.
+ FAILURE_DIAGNOSES: ["run_id", "fingerprint"],
+}
# Creation time is stable for partitioning. The source replays runs to pick up late retries.
RUN_CREATED_AT = "run_created_at"
+# Failure diagnoses sync as a full refresh: a retry that passes removes a run's failure groups, and a
+# merge never deletes the rows it no longer receives.
INCREMENTAL_FIELDS: dict[str, list[IncrementalField]] = {
- JOB_ATTEMPTS: [
+ endpoint: [
{
"label": RUN_CREATED_AT,
"type": IncrementalFieldType.DateTime,
"field": RUN_CREATED_AT,
"field_type": IncrementalFieldType.DateTime,
},
- ],
+ ]
+ for endpoint in (JOB_ATTEMPTS, JOB_METRICS, ARTIFACTS)
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/depot/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/depot/source.py
index bfa9dc8fbae5..d76d749ec674 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/depot/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/depot/source.py
@@ -49,6 +49,8 @@ _APPEND_UNSUPPORTED = (
class DepotReconciliationCursor:
cursor_kind: ClassVar[str] = "depot_reconciliation"
reconciled_at: str
+ # The newest run a completed sync scanned. It stands in for the watermark of a table whose runs yield no rows.
+ scanned_through: str | None = None
@SourceRegistry.register
@@ -72,7 +74,7 @@ class DepotSource(SimpleSource[DepotSourceConfig], CursorSource[DepotReconciliat
category=DataWarehouseSourceCategory.ENGINEERING___MONITORING,
label="Depot",
keywords=["depot.dev", "depot ci"],
- caption="Sync Depot CI job attempts for one GitHub repository. Create an organization API token in your Depot organization settings, then enter the repository as `owner/name`.",
+ caption="Sync Depot CI job attempts, job metrics, artifacts, and failure diagnoses for one GitHub repository. Create an organization API token in your Depot organization settings, then enter the repository as `owner/name`.",
releaseStatus=ReleaseStatus.ALPHA,
docsUrl="https://posthog.com/docs/cdp/sources/depot",
iconPath="/static/services/depot.png",
@@ -146,7 +148,9 @@ class DepotSource(SimpleSource[DepotSourceConfig], CursorSource[DepotReconciliat
now = datetime.datetime.now(datetime.UTC)
reconciled_at = parse_datetime_value(cursor.reconciled_at) if cursor is not None else None
watermark = (
- parse_datetime_value(inputs.db_incremental_field_last_value)
+ parse_datetime_value(
+ inputs.db_incremental_field_last_value or (cursor.scanned_through if cursor is not None else None)
+ )
if inputs.should_use_incremental_field
else None
)
@@ -159,12 +163,21 @@ class DepotSource(SimpleSource[DepotSourceConfig], CursorSource[DepotReconciliat
if inputs.history_start is not None:
lower_bound = max(lower_bound, inputs.history_start)
+ previous = cursor or DepotReconciliationCursor(reconciled_at=now.isoformat())
+
+ def on_complete(scanned_through: str | None) -> None:
+ manager.stage(
+ DepotReconciliationCursor(
+ reconciled_at=now.isoformat() if reconcile else previous.reconciled_at,
+ scanned_through=scanned_through or previous.scanned_through,
+ )
+ )
+
return depot_source(
api_token=config.api_token,
repository=config.repository,
created_after=lower_bound,
logger=inputs.logger,
- on_complete=(lambda: manager.stage(DepotReconciliationCursor(reconciled_at=now.isoformat())))
- if reconcile
- else None,
+ endpoint=inputs.schema_name,
+ on_complete=on_complete,
)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/depot/tests/test_depot.py b/products/warehouse_sources/backend/temporal/data_imports/sources/depot/tests/test_depot.py
index 80acde19233a..727c916ddc56 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/depot/tests/test_depot.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/depot/tests/test_depot.py
@@ -21,6 +21,12 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.depot.depo
depot_source,
validate_credentials,
)
+from products.warehouse_sources.backend.temporal.data_imports.sources.depot.settings import (
+ ARTIFACTS,
+ FAILURE_DIAGNOSES,
+ JOB_ATTEMPTS,
+ JOB_METRICS,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.depot.source import (
DepotReconciliationCursor,
DepotSource,
@@ -83,8 +89,31 @@ def _fake_session(
next_page_token = str(end) if end < len(terminal_runs) else ""
return _response(200, {"runs": page, "nextPageToken": next_page_token}, method)
if method == "GetRunStatus":
- run_workflows = workflows.get(json["runId"], [])
- return _response(200, {"workflows": [{"workflowId": w["workflowId"]} for w in run_workflows]}, method)
+ return _response(200, {"workflows": workflows.get(json["runId"], [])}, method)
+ if method == "GetJobMetrics":
+ attempts = next(
+ job["attempts"]
+ for workflow in workflows_by_id.values()
+ for job in workflow["jobs"]
+ if job["jobId"] == json["jobId"]
+ )
+ summaries = [{"attempt": attempt, "stats": {"peakMemoryUsageBytes": "1024"}} for attempt in attempts]
+ return _response(200, {"job": {"jobKey": "build"}, "attempts": summaries}, method)
+ if method == "ListArtifacts":
+ artifacts_page: dict[str, Any]
+ if "pageToken" not in json:
+ artifacts_page = {
+ "artifacts": [{"artifactId": f"{json['runId']}-a1", "sizeBytes": "2048"}],
+ "nextPageToken": "p2",
+ }
+ else:
+ artifacts_page = {"artifacts": [{"artifactId": f"{json['runId']}-a2", "sizeBytes": "4096"}]}
+ return _response(200, artifacts_page, method)
+ if method == "GetFailureDiagnosis":
+ groups = [{"fingerprint": "fp-1", "count": 2, "errorMessage": "exit status 1"}]
+ return _response(
+ 200, {"state": "FAILURE_DIAGNOSIS_STATE_GROUPED_FAILURES", "failureGroups": groups}, method
+ )
return _response(200, workflows_by_id[json["workflowId"]], method)
session = mock.MagicMock()
@@ -96,9 +125,11 @@ def _batches(response: SourceResponse) -> list[list[dict[str, Any]]]:
return list(cast(Iterable[list[dict[str, Any]]], response.items()))
-def _synced_rows(session: mock.MagicMock, created_after: dt.datetime | str | None) -> list[dict[str, Any]]:
+def _synced_rows(
+ session: mock.MagicMock, created_after: dt.datetime | str | None, endpoint: str = JOB_ATTEMPTS
+) -> list[dict[str, Any]]:
with mock.patch(f"{MODULE}.make_tracked_session", return_value=session):
- batches = _batches(depot_source(API_TOKEN, REPOSITORY, created_after, mock.MagicMock()))
+ batches = _batches(depot_source(API_TOKEN, REPOSITORY, created_after, mock.MagicMock(), endpoint=endpoint))
return [row for batch in batches for row in batch]
@@ -147,17 +178,75 @@ class TestDepotSource:
assert [row["run_id"] for row in rows] == expected_run_ids
+ @pytest.mark.parametrize(
+ "endpoint, columns, expected_rows, expected_calls",
+ [
+ (
+ JOB_METRICS,
+ ("run_id", "attempt_id", "peak_memory_usage_bytes"),
+ [("failed", "failed-attempt", 1024), ("ok", "ok-attempt", 1024)],
+ [("GetJobMetrics", {"jobId": "failed-job"}), ("GetJobMetrics", {"jobId": "ok-job"})],
+ ),
+ (
+ ARTIFACTS,
+ ("run_id", "artifact_id", "size_bytes"),
+ [
+ ("failed", "failed-a1", 2048),
+ ("failed", "failed-a2", 4096),
+ ("ok", "ok-a1", 2048),
+ ("ok", "ok-a2", 4096),
+ ],
+ [
+ ("ListArtifacts", {"runId": "failed", "pageSize": 500}),
+ ("ListArtifacts", {"runId": "failed", "pageSize": 500, "pageToken": "p2"}),
+ ("ListArtifacts", {"runId": "ok", "pageSize": 500}),
+ ("ListArtifacts", {"runId": "ok", "pageSize": 500, "pageToken": "p2"}),
+ ],
+ ),
+ (
+ FAILURE_DIAGNOSES,
+ ("run_id", "fingerprint", "count"),
+ [("failed", "fp-1", 2)],
+ [("GetFailureDiagnosis", {"targetId": "failed", "targetType": "FAILURE_DIAGNOSIS_TARGET_TYPE_RUN"})],
+ ),
+ ],
+ )
+ def test_run_child_tables(
+ self,
+ endpoint: str,
+ columns: tuple[str, ...],
+ expected_rows: list[tuple[Any, ...]],
+ expected_calls: list[tuple[str, dict[str, Any]]],
+ ) -> None:
+ runs = [{**_run("failed", dt.timedelta(hours=1)), "status": "failed"}, _run("ok", dt.timedelta(hours=2))]
+ workflows = {run["runId"]: [_single_attempt_workflow(run)] for run in runs}
+ workflows["ok"][0]["jobs"].append({"jobId": "ok-skipped-job", "attempts": []})
+ session = _fake_session(runs, workflows_by_run=workflows)
+
+ rows = _synced_rows(session, None, endpoint)
+
+ assert [tuple(row[column] for column in columns) for row in rows] == expected_rows
+ assert all(row["run_created_at"] for row in rows)
+ called_method = expected_calls[0][0]
+ calls = [
+ (call.args[0].rsplit("/", 1)[-1], call.kwargs["json"])
+ for call in session.post.call_args_list
+ if call.args[0].endswith(f"/{called_method}")
+ ]
+ assert calls == expected_calls
+
class TestDepotReconciliation:
@pytest.mark.parametrize(
- "reconciled_days_ago, history_days, has_watermark, expected_runs, expected_reconciled",
+ "reconciled_days_ago, history_days, has_watermark, scanned_without_rows, expected_runs, expected_reconciled",
[
- (None, 30, True, ["recent", "old"], True),
- (1, 30, True, ["recent"], False),
- (7, 30, True, ["recent", "old"], True),
- (1, 30, False, ["recent", "old"], True),
- (None, None, True, ["recent", "old", "outside"], True),
- (1, 1, True, [], False),
+ (None, 30, True, False, ["recent", "old"], True),
+ (1, 30, True, False, ["recent"], False),
+ (7, 30, True, False, ["recent", "old"], True),
+ (1, 30, False, False, ["recent", "old"], True),
+ (1, 30, False, True, ["recent"], False),
+ (None, None, True, False, ["recent", "old", "outside"], True),
+ (1, 1, True, False, [], False),
],
)
def test_replays_retries_and_reconciles_retained_history(
@@ -165,12 +254,16 @@ class TestDepotReconciliation:
reconciled_days_ago: int | None,
history_days: int | None,
has_watermark: bool,
+ scanned_without_rows: bool,
expected_runs: list[str],
expected_reconciled: bool,
) -> None:
source = DepotSource()
cursor = (
- DepotReconciliationCursor(reconciled_at=_iso(NOW - dt.timedelta(days=reconciled_days_ago)))
+ DepotReconciliationCursor(
+ reconciled_at=_iso(NOW - dt.timedelta(days=reconciled_days_ago)),
+ scanned_through=_iso(NOW - dt.timedelta(minutes=10)) if scanned_without_rows else None,
+ )
if reconciled_days_ago is not None
else None
)
@@ -180,6 +273,7 @@ class TestDepotReconciliation:
db_incremental_field_last_value=_iso(NOW - dt.timedelta(minutes=10)) if has_watermark else None,
history_start=NOW - dt.timedelta(days=history_days) if history_days is not None else None,
source_cursor=manager,
+ schema_name=JOB_ATTEMPTS,
)
runs = [
_run("recent", dt.timedelta(days=2)),
@@ -198,12 +292,19 @@ class TestDepotReconciliation:
rows = [row for batch in _batches(response) for row in batch]
assert [row["run_id"] for row in rows if row["attempt"] == 2] == expected_runs
- assert (manager.staged is not None) == expected_reconciled
+ # mypy keeps the narrowing from the assertion before the sync ran.
+ staged = cast(DepotReconciliationCursor | None, manager.staged)
+ assert staged is not None
+ assert (staged.reconciled_at == NOW.isoformat()) == expected_reconciled
+ if expected_runs:
+ assert staged.scanned_through == runs[0]["createdAt"]
def test_incomplete_reconciliation_does_not_advance_the_cursor(self) -> None:
source = DepotSource()
manager = SourceCursorManager(source.cursor_class(), None, source)
- inputs = mock.MagicMock(should_use_incremental_field=False, history_start=None, source_cursor=manager)
+ inputs = mock.MagicMock(
+ should_use_incremental_field=False, history_start=None, source_cursor=manager, schema_name=JOB_ATTEMPTS
+ )
runs = [_run("recent", dt.timedelta(days=2)), _run("old", dt.timedelta(days=14))]
session = _fake_session(runs)
post = session.post.side_effect
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/depot/tests/test_depot_source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/depot/tests/test_depot_source.py
index cce130c51b2f..6df69b2ed5cf 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/depot/tests/test_depot_source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/depot/tests/test_depot_source.py
@@ -66,7 +66,7 @@ class TestDepotSource:
source = DepotSource()
session = mock.MagicMock()
session.post.return_value = _response(status, reason, body)
- inputs = mock.MagicMock(should_use_incremental_field=False, history_start=None)
+ inputs = mock.MagicMock(should_use_incremental_field=False, history_start=None, schema_name="job_attempts")
with mock.patch(f"{MODULE}.make_tracked_session", return_value=session):
response = source.source_for_pipeline(DepotSourceConfig(api_token="token", repository="a/b"), inputs)