Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): add pipeline, transform, experiment, and monitoring tables to aws sagemaker (#118509)

Tom Owers3dea60741262
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index e4507752781a..4d0af1127925 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -1273,12 +1273,12 @@ Diffed against: <https://raw.githubusercontent.com/boto/botocore/develop/botocor
- [x] `Model packages (ListModelPackages and DescribeModelPackage, POST /)` — resolve model package references in models and training jobs to registered versions, approval status, and evaluation results (high)
- [x] `Hyperparameter tuning jobs (ListHyperParameterTuningJobs and DescribeHyperParameterTuningJob, POST /)` — resolve TuningJobArn on training jobs and compare tuning objectives, outcomes, and resource use (high)
- [x] `AutoML jobs (ListAutoMLJobs, DescribeAutoMLJob and DescribeAutoMLJobV2, POST /)` — resolve AutoMLJobArn on training jobs and compare automated training outcomes and best candidates (high)
-- [ ] `Experiments, trials, and trial components (ListExperiments, ListTrials, ListTrialComponents and their Describe operations, POST /)` — resolve ExperimentConfig references on training jobs and compare experiment runs, parameters, and metrics (high)
-- [ ] `Transform jobs (ListTransformJobs and DescribeTransformJob, POST /)` — analyze batch inference status, duration, failures, and model use (high)
-- [ ] `Pipeline executions (ListPipelineExecutions and DescribePipelineExecution, POST /)` — measure workflow success rates, execution duration, and repeated failures (high)
+- [x] `Experiments, trials, and trial components (ListExperiments, ListTrials, ListTrialComponents and their Describe operations, POST /)` — resolve ExperimentConfig references on training jobs and compare experiment runs, parameters, and metrics (high)
+- [x] `Transform jobs (ListTransformJobs and DescribeTransformJob, POST /)` — analyze batch inference status, duration, failures, and model use (high)
+- [x] `Pipeline executions (ListPipelineExecutions and DescribePipelineExecution, POST /)` — measure workflow success rates, execution duration, and repeated failures (high)
- [ ] `Pipeline execution steps (ListPipelineExecutionSteps, POST /)` — locate slow or failed steps and connect pipeline runs to training and processing jobs (medium)
- [ ] `Pipelines (ListPipelines and DescribePipeline, POST /)` — identify pipeline owners and definitions when comparing workflow execution results (medium)
-- [ ] `Monitoring executions (ListMonitoringExecutions, POST /)` — track model monitoring failures and execution status for deployed endpoints (medium)
+- [x] `Monitoring executions (ListMonitoringExecutions, POST /)` — track model monitoring failures and execution status for deployed endpoints (medium)
Note: The official AWS SDK model supplies operation names, HTTP methods, required inputs, and response fields. The four existing tables use list and describe operations; training metrics and status transitions are already included. Read operations use POST / and require operation names in X-Amz-Target. Pipelines, steps, experiments, and model packages need nested requests or describe calls; unrelated settings and write operations are excluded.
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_sagemaker/aws_sagemaker.py b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_sagemaker/aws_sagemaker.py
index ac4a2b118f67..b7bbda2f8508 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_sagemaker/aws_sagemaker.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_sagemaker/aws_sagemaker.py
@@ -16,6 +16,8 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.aws_sagema
ENDPOINTS,
MAX_RESULTS,
TARGET_PREFIXES,
+ SageMakerEndpoint,
+ SageMakerParent,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
from products.warehouse_sources.backend.temporal.data_imports.sources.common.http.transport import BoundedRetry
@@ -54,6 +56,10 @@ _TIMESTAMP_FIELDS = {
"ProcessingEndTime",
"HyperParameterTuningEndTime",
"EndTime",
+ "StartTime",
+ "ScheduledTime",
+ "TransformStartTime",
+ "TransformEndTime",
}
@@ -62,6 +68,7 @@ class AwsSagemakerResumeConfig:
next_token: str | None = None
watermark: float | None = None
complete: bool = False
+ parent_name: str | None = None
class AwsSagemakerError(Exception):
@@ -167,6 +174,83 @@ def timestamp(value: dt.datetime | str | int | float | None) -> float | None:
return (parsed if parsed.tzinfo else parsed.replace(tzinfo=dt.UTC)).timestamp()
+def parent_names(
+ client: AwsSagemakerClient,
+ parent: SageMakerParent,
+ manager: ResumableSourceManager[AwsSagemakerResumeConfig],
+) -> list[str]:
+ names: list[str] = []
+ token: str | None = None
+ while True:
+ payload: dict[str, Any] = {"MaxResults": MAX_RESULTS, "SortBy": "Name", "SortOrder": "Ascending"}
+ if token:
+ payload["NextToken"] = token
+ page = client.request(parent.list_operation, payload)
+ names.extend(item[parent.name_field] for item in page[parent.result_key])
+ manager.safe_point()
+ next_token = page.get("NextToken") or None
+ if next_token is None:
+ break
+ if next_token == token:
+ raise ValueError("AWS SageMaker returned a repeated pagination token.")
+ token = next_token
+ # Sorting here keeps the resume order independent of how AWS collates names.
+ return sorted(set(names))
+
+
+def list_pages(
+ client: AwsSagemakerClient,
+ settings: SageMakerEndpoint,
+ manager: ResumableSourceManager[AwsSagemakerResumeConfig],
+ watermark: float | None,
+ incremental: bool,
+ token: str | None,
+ parent_name: str | None = None,
+ following_parent: str | None = None,
+) -> Iterator[list[dict[str, Any]]]:
+ while True:
+ payload: dict[str, Any] = {
+ "MaxResults": MAX_RESULTS,
+ "SortBy": "CreationTime",
+ "SortOrder": "Ascending",
+ **dict(settings.list_filters),
+ }
+ if settings.parent and parent_name:
+ payload[settings.parent.name_field] = parent_name
+ if watermark is not None and incremental and settings.incremental:
+ payload["CreationTimeAfter"] = watermark
+ if token:
+ payload["NextToken"] = token
+ page = client.request(settings.list_operation, payload)
+ next_token = page.get("NextToken") or None
+ if next_token == token and next_token is not None:
+ raise ValueError("AWS SageMaker returned a repeated pagination token.")
+ describe_operation = settings.describe_operation
+ rows = [
+ normalize_row(
+ client.request(describe_operation, settings.describe_payload(item)) if describe_operation else item
+ )
+ for item in page[settings.result_key]
+ ]
+ for row in rows:
+ if any(not row.get(key) for key in settings.primary_keys):
+ raise ValueError("AWS SageMaker returned a resource without its identifier.")
+ manager.save_state(
+ AwsSagemakerResumeConfig(
+ next_token=next_token,
+ watermark=watermark,
+ complete=next_token is None and following_parent is None,
+ parent_name=parent_name if next_token else following_parent,
+ )
+ )
+ if rows:
+ yield rows
+ manager.safe_point()
+ if next_token is None:
+ break
+ token = next_token
+
+
def get_rows(
config: AwsSagemakerSourceConfig,
endpoint: str,
@@ -183,37 +267,32 @@ def get_rows(
token = state.next_token if state else None
client = AwsSagemakerClient(config, api_version)
try:
- while True:
- payload: dict[str, Any] = {
- "MaxResults": MAX_RESULTS,
- "SortBy": "CreationTime",
- "SortOrder": "Ascending",
- **dict(settings.list_filters),
- }
- if watermark is not None and incremental and settings.incremental:
- payload["CreationTimeAfter"] = watermark
- if token:
- payload["NextToken"] = token
- page = client.request(settings.list_operation, payload)
- next_token = page.get("NextToken") or None
- if next_token == token and next_token is not None:
- raise ValueError("AWS SageMaker returned a repeated pagination token.")
- rows = [
- normalize_row(client.request(settings.describe_operation, settings.describe_payload(item)))
- for item in page[settings.result_key]
- ]
- for row in rows:
- if not row.get(settings.primary_key):
- raise ValueError("AWS SageMaker returned a resource without its ARN.")
- manager.save_state(
- AwsSagemakerResumeConfig(next_token=next_token, watermark=watermark, complete=next_token is None)
- )
- if rows:
- yield rows
- manager.safe_point()
- if next_token is None:
- break
- token = next_token
+ if settings.parent is None:
+ yield from list_pages(client, settings, manager, watermark, incremental, token)
+ return
+ start = state.parent_name if state else None
+ parents = [name for name in parent_names(client, settings.parent, manager) if start is None or name >= start]
+ for index, name in enumerate(parents):
+ following = parents[index + 1] if index + 1 < len(parents) else None
+ try:
+ yield from list_pages(
+ client,
+ settings,
+ manager,
+ watermark,
+ incremental,
+ token if name == start else None,
+ parent_name=name,
+ following_parent=following,
+ )
+ except AwsSagemakerError as error:
+ # A parent deleted during the walk must not stop the walk over the remaining parents.
+ if error.code != "ResourceNotFound":
+ raise
+ manager.save_state(
+ AwsSagemakerResumeConfig(watermark=watermark, complete=following is None, parent_name=following)
+ )
+ manager.safe_point()
finally:
client.close()
@@ -229,17 +308,26 @@ def validate_credentials(
except ValueError as error:
return False, str(error)
try:
- page = client.request(settings.list_operation, {"MaxResults": 1, **dict(settings.list_filters)})
- if schema_name and page.get(settings.result_key):
+ filters = dict(settings.list_filters)
+ if settings.parent:
+ parents = client.request(settings.parent.list_operation, {"MaxResults": 1})[settings.parent.result_key]
+ if not parents:
+ return True, None
+ filters[settings.parent.name_field] = parents[0][settings.parent.name_field]
+ page = client.request(settings.list_operation, {"MaxResults": 1, **filters})
+ if schema_name and settings.describe_operation and page.get(settings.result_key):
client.request(settings.describe_operation, settings.describe_payload(page[settings.result_key][0]))
except AwsSagemakerError as error:
if error.code in {"AccessDenied", "AccessDeniedException"}:
if schema_name is None:
return True, None
- return (
- False,
- f"Grant sagemaker:{settings.list_operation} and sagemaker:{settings.describe_operation} for this table.",
- )
+ operations = [
+ settings.parent.list_operation if settings.parent else None,
+ settings.list_operation,
+ settings.describe_operation,
+ ]
+ permissions = ", ".join(f"sagemaker:{operation}" for operation in operations if operation)
+ return False, f"Grant these permissions for this table: {permissions}."
for code, message in ERROR_MESSAGES.items():
if error.code.startswith(code):
return False, message
@@ -263,7 +351,7 @@ def aws_sagemaker_source(
return SourceResponse(
name=endpoint,
items=lambda: get_rows(config, endpoint, api_version, manager, incremental, last_value),
- primary_keys=[settings.primary_key],
+ primary_keys=list(settings.primary_keys),
partition_keys=["creation_time"],
partition_mode="datetime",
partition_format="month",
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_sagemaker/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_sagemaker/canonical_descriptions.py
index ac713d068b27..4419e288c0aa 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_sagemaker/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_sagemaker/canonical_descriptions.py
@@ -125,4 +125,90 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"failure_reason": "The reason the AutoML job failed.",
},
},
+ "transform_jobs": {
+ "description": "SageMaker batch transform jobs, including their status, model, data settings, and run times.",
+ "docs_url": "https://docs.aws.amazon.com/sagemaker/latest/APIReference/API_DescribeTransformJob.html",
+ "columns": {
+ "transform_job_arn": "The Amazon Resource Name of the transform job.",
+ "transform_job_name": "The name of the transform job.",
+ "transform_job_status": "The status of the transform job.",
+ "failure_reason": "The reason the transform job failed, if it failed.",
+ "model_name": "The name of the model that the transform job uses.",
+ "creation_time": "The time AWS created the transform job.",
+ "transform_start_time": "The time the transform job started.",
+ "transform_end_time": "The time the transform job ended.",
+ "transform_resources": "The ML compute instances that the transform job uses.",
+ "experiment_config": "The experiment, trial, and trial component that the transform job is associated with.",
+ },
+ },
+ "pipeline_executions": {
+ "description": "SageMaker pipeline executions for every pipeline in the region, including status and run times.",
+ "docs_url": "https://docs.aws.amazon.com/sagemaker/latest/APIReference/API_DescribePipelineExecution.html",
+ "columns": {
+ "pipeline_execution_arn": "The Amazon Resource Name of the pipeline execution.",
+ "pipeline_arn": "The Amazon Resource Name of the pipeline that ran.",
+ "pipeline_execution_display_name": "The display name of the pipeline execution.",
+ "pipeline_execution_status": "The status of the pipeline execution.",
+ "failure_reason": "The reason the pipeline execution failed, if it failed.",
+ "creation_time": "The time AWS created the pipeline execution.",
+ "last_modified_time": "The time AWS last modified the pipeline execution.",
+ "pipeline_experiment_config": "The experiment and trial that the pipeline execution is associated with.",
+ },
+ },
+ "experiments": {
+ "description": "SageMaker experiments, which group trials for comparison.",
+ "docs_url": "https://docs.aws.amazon.com/sagemaker/latest/APIReference/API_DescribeExperiment.html",
+ "columns": {
+ "experiment_arn": "The Amazon Resource Name of the experiment.",
+ "experiment_name": "The name of the experiment.",
+ "display_name": "The name of the experiment as displayed.",
+ "source": "The resource that created the experiment, if any.",
+ "description": "The description of the experiment.",
+ "creation_time": "The time AWS created the experiment.",
+ "last_modified_time": "The time AWS last modified the experiment.",
+ },
+ },
+ "trials": {
+ "description": "SageMaker trials, which group the trial components of one experiment run.",
+ "docs_url": "https://docs.aws.amazon.com/sagemaker/latest/APIReference/API_DescribeTrial.html",
+ "columns": {
+ "trial_arn": "The Amazon Resource Name of the trial.",
+ "trial_name": "The name of the trial.",
+ "experiment_name": "The name of the experiment that the trial is part of.",
+ "display_name": "The name of the trial as displayed.",
+ "source": "The resource that created the trial, if any.",
+ "creation_time": "The time AWS created the trial.",
+ "last_modified_time": "The time AWS last modified the trial.",
+ },
+ },
+ "trial_components": {
+ "description": "SageMaker trial components, the steps of a trial with their parameters, artifacts, and metrics.",
+ "docs_url": "https://docs.aws.amazon.com/sagemaker/latest/APIReference/API_DescribeTrialComponent.html",
+ "columns": {
+ "trial_component_arn": "The Amazon Resource Name of the trial component.",
+ "trial_component_name": "The name of the trial component.",
+ "status": "The status of the trial component.",
+ "start_time": "The time the trial component started.",
+ "end_time": "The time the trial component ended.",
+ "parameters": "The hyperparameters of the trial component.",
+ "input_artifacts": "The input artifacts of the trial component.",
+ "output_artifacts": "The output artifacts of the trial component.",
+ "metrics": "The summary statistics of the metrics of the trial component.",
+ "creation_time": "The time AWS created the trial component.",
+ },
+ },
+ "monitoring_executions": {
+ "description": "SageMaker model monitoring executions, one row for each scheduled run of a monitoring schedule.",
+ "docs_url": "https://docs.aws.amazon.com/sagemaker/latest/APIReference/API_MonitoringExecutionSummary.html",
+ "columns": {
+ "monitoring_schedule_name": "The name of the monitoring schedule.",
+ "scheduled_time": "The time the monitoring job was scheduled.",
+ "monitoring_execution_status": "The status of the monitoring job.",
+ "processing_job_arn": "The Amazon Resource Name of the processing job that ran the monitoring job.",
+ "endpoint_name": "The name of the monitored endpoint.",
+ "failure_reason": "The reason the monitoring job failed, if it failed.",
+ "monitoring_type": "The type of monitoring job.",
+ "creation_time": "The time AWS created the monitoring job.",
+ },
+ },
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_sagemaker/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_sagemaker/settings.py
index 388f62a5b542..121cf50a41bf 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_sagemaker/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_sagemaker/settings.py
@@ -11,15 +11,25 @@ MAX_RESULTS = 100
@frozen
-class SageMakerEndpoint:
+class SageMakerParent:
list_operation: str
result_key: str
- describe_operation: str
name_field: str
- primary_key: str
+
+
+@frozen
+class SageMakerEndpoint:
+ list_operation: str
+ result_key: str
+ primary_keys: tuple[str, ...]
+ # Without a describe operation, the list summary is the row.
+ describe_operation: str | None = None
+ name_field: str = ""
incremental: bool = False
describe_key: str | None = None
list_filters: tuple[tuple[str, str], ...] = ()
+ # The list operation requires a parent name, so the source lists each parent's children.
+ parent: SageMakerParent | None = None
def describe_payload(self, item: dict[str, Any]) -> dict[str, Any]:
return {self.name_field: item[self.describe_key or self.name_field]}
@@ -31,28 +41,28 @@ ENDPOINTS: dict[str, SageMakerEndpoint] = {
result_key="TrainingJobSummaries",
describe_operation="DescribeTrainingJob",
name_field="TrainingJobName",
- primary_key="training_job_arn",
+ primary_keys=("training_job_arn",),
),
"processing_jobs": SageMakerEndpoint(
list_operation="ListProcessingJobs",
result_key="ProcessingJobSummaries",
describe_operation="DescribeProcessingJob",
name_field="ProcessingJobName",
- primary_key="processing_job_arn",
+ primary_keys=("processing_job_arn",),
),
"endpoints": SageMakerEndpoint(
list_operation="ListEndpoints",
result_key="Endpoints",
describe_operation="DescribeEndpoint",
name_field="EndpointName",
- primary_key="endpoint_arn",
+ primary_keys=("endpoint_arn",),
),
"models": SageMakerEndpoint(
list_operation="ListModels",
result_key="Models",
describe_operation="DescribeModel",
name_field="ModelName",
- primary_key="model_arn",
+ primary_keys=("model_arn",),
incremental=True,
),
"endpoint_configs": SageMakerEndpoint(
@@ -60,7 +70,7 @@ ENDPOINTS: dict[str, SageMakerEndpoint] = {
result_key="EndpointConfigs",
describe_operation="DescribeEndpointConfig",
name_field="EndpointConfigName",
- primary_key="endpoint_config_arn",
+ primary_keys=("endpoint_config_arn",),
incremental=True,
),
"model_packages": SageMakerEndpoint(
@@ -68,7 +78,7 @@ ENDPOINTS: dict[str, SageMakerEndpoint] = {
result_key="ModelPackageSummaryList",
describe_operation="DescribeModelPackage",
name_field="ModelPackageName",
- primary_key="model_package_arn",
+ primary_keys=("model_package_arn",),
# Versioned packages share their group's name, so only the ARN identifies one version.
describe_key="ModelPackageArn",
# The API lists only unversioned packages by default, which hides Model Registry versions.
@@ -79,14 +89,60 @@ ENDPOINTS: dict[str, SageMakerEndpoint] = {
result_key="HyperParameterTuningJobSummaries",
describe_operation="DescribeHyperParameterTuningJob",
name_field="HyperParameterTuningJobName",
- primary_key="hyper_parameter_tuning_job_arn",
+ primary_keys=("hyper_parameter_tuning_job_arn",),
),
"automl_jobs": SageMakerEndpoint(
list_operation="ListAutoMLJobs",
result_key="AutoMLJobSummaries",
describe_operation="DescribeAutoMLJobV2",
name_field="AutoMLJobName",
- primary_key="auto_ml_job_arn",
+ primary_keys=("auto_ml_job_arn",),
+ ),
+ "transform_jobs": SageMakerEndpoint(
+ list_operation="ListTransformJobs",
+ result_key="TransformJobSummaries",
+ describe_operation="DescribeTransformJob",
+ name_field="TransformJobName",
+ primary_keys=("transform_job_arn",),
+ ),
+ "pipeline_executions": SageMakerEndpoint(
+ list_operation="ListPipelineExecutions",
+ result_key="PipelineExecutionSummaries",
+ describe_operation="DescribePipelineExecution",
+ name_field="PipelineExecutionArn",
+ primary_keys=("pipeline_execution_arn",),
+ parent=SageMakerParent(
+ list_operation="ListPipelines",
+ result_key="PipelineSummaries",
+ name_field="PipelineName",
+ ),
+ ),
+ "experiments": SageMakerEndpoint(
+ list_operation="ListExperiments",
+ result_key="ExperimentSummaries",
+ describe_operation="DescribeExperiment",
+ name_field="ExperimentName",
+ primary_keys=("experiment_arn",),
+ ),
+ "trials": SageMakerEndpoint(
+ list_operation="ListTrials",
+ result_key="TrialSummaries",
+ describe_operation="DescribeTrial",
+ name_field="TrialName",
+ primary_keys=("trial_arn",),
+ ),
+ "trial_components": SageMakerEndpoint(
+ list_operation="ListTrialComponents",
+ result_key="TrialComponentSummaries",
+ describe_operation="DescribeTrialComponent",
+ name_field="TrialComponentName",
+ primary_keys=("trial_component_arn",),
+ ),
+ "monitoring_executions": SageMakerEndpoint(
+ list_operation="ListMonitoringExecutions",
+ result_key="MonitoringExecutionSummaries",
+ # Executions have no ARN, and a schedule runs at most once per scheduled time.
+ primary_keys=("monitoring_schedule_name", "scheduled_time"),
),
}
@@ -104,4 +160,10 @@ ENDPOINT_DESCRIPTIONS = {
"model_packages": "Model packages and Model Registry versions with approval status, metrics, and inference settings.",
"hyperparameter_tuning_jobs": "Hyperparameter tuning jobs with objectives, best training jobs, and consumed resources.",
"automl_jobs": "AutoML jobs with problem type, objective, status, and best candidate.",
+ "transform_jobs": "Batch transform jobs with status, model, input and output settings, and run times.",
+ "pipeline_executions": "Pipeline executions with status, failure reason, and run times for every pipeline.",
+ "experiments": "Experiments with source, display name, and creation details.",
+ "trials": "Experiment trials with their experiment, source, and metadata.",
+ "trial_components": "Trial components with status, parameters, artifacts, and metric summaries.",
+ "monitoring_executions": "Model monitoring executions with status, schedule, endpoint, and failure reason.",
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_sagemaker/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_sagemaker/source.py
index b557ffc150d7..8b5575419c86 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_sagemaker/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_sagemaker/source.py
@@ -139,8 +139,14 @@ class AwsSagemakerSource(ResumableSource[AwsSagemakerSourceConfig, AwsSagemakerR
"`sagemaker:ListEndpointConfigs`, `sagemaker:DescribeEndpointConfig`, "
"`sagemaker:ListModelPackages`, `sagemaker:DescribeModelPackage`, "
"`sagemaker:ListHyperParameterTuningJobs`, `sagemaker:DescribeHyperParameterTuningJob`, "
- "`sagemaker:ListAutoMLJobs`, and `sagemaker:DescribeAutoMLJobV2`. "
- "Jobs, endpoints, and model packages use full refresh to capture status changes. "
+ "`sagemaker:ListAutoMLJobs`, `sagemaker:DescribeAutoMLJobV2`, "
+ "`sagemaker:ListTransformJobs`, `sagemaker:DescribeTransformJob`, "
+ "`sagemaker:ListPipelines`, `sagemaker:ListPipelineExecutions`, `sagemaker:DescribePipelineExecution`, "
+ "`sagemaker:ListExperiments`, `sagemaker:DescribeExperiment`, "
+ "`sagemaker:ListTrials`, `sagemaker:DescribeTrial`, "
+ "`sagemaker:ListTrialComponents`, `sagemaker:DescribeTrialComponent`, "
+ "and `sagemaker:ListMonitoringExecutions`. "
+ "Jobs, executions, experiments, endpoints, and model packages use full refresh to capture status changes. "
"Models and endpoint configurations also support incremental sync by creation time."
),
iconPath="/static/services/aws_sagemaker.png",
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_sagemaker/tests/test_aws_sagemaker.py b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_sagemaker/tests/test_aws_sagemaker.py
index daf84bb3d3c0..966ff19da170 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_sagemaker/tests/test_aws_sagemaker.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_sagemaker/tests/test_aws_sagemaker.py
@@ -203,6 +203,50 @@ def test_signed_json_request(
{},
{"AutoMLJobName": "example"},
),
+ (
+ "transform_jobs",
+ "ListTransformJobs",
+ "TransformJobSummaries",
+ "DescribeTransformJob",
+ "TransformJobName",
+ "TransformJobArn",
+ "transform_job_arn",
+ {},
+ {"TransformJobName": "example"},
+ ),
+ (
+ "experiments",
+ "ListExperiments",
+ "ExperimentSummaries",
+ "DescribeExperiment",
+ "ExperimentName",
+ "ExperimentArn",
+ "experiment_arn",
+ {},
+ {"ExperimentName": "example"},
+ ),
+ (
+ "trials",
+ "ListTrials",
+ "TrialSummaries",
+ "DescribeTrial",
+ "TrialName",
+ "TrialArn",
+ "trial_arn",
+ {},
+ {"TrialName": "example"},
+ ),
+ (
+ "trial_components",
+ "ListTrialComponents",
+ "TrialComponentSummaries",
+ "DescribeTrialComponent",
+ "TrialComponentName",
+ "TrialComponentArn",
+ "trial_component_arn",
+ {},
+ {"TrialComponentName": "example"},
+ ),
],
)
def test_pagination_describes_rows_and_stages_only_complete_pages(
@@ -265,6 +309,131 @@ def test_pagination_describes_rows_and_stages_only_complete_pages(
transport.close.assert_called_once()
+def execution(arn: str) -> ScriptedResponse:
+ return ScriptedResponse(json={"PipelineExecutionArn": arn, "CreationTime": 1704067200})
+
+
+ "resume_state,script,expected_targets,expected_rows,expected_states",
+ [
+ (
+ None,
+ [
+ ScriptedResponse(json={"PipelineSummaries": [{"PipelineName": "zeta"}], "NextToken": "p2"}),
+ ScriptedResponse(json={"PipelineSummaries": [{"PipelineName": "alpha"}]}),
+ ScriptedResponse(
+ json={"PipelineExecutionSummaries": [{"PipelineExecutionArn": "arn-a1"}], "NextToken": "a2"}
+ ),
+ execution("arn-a1"),
+ ScriptedResponse(json={"PipelineExecutionSummaries": []}),
+ ScriptedResponse(json={"PipelineExecutionSummaries": [{"PipelineExecutionArn": "arn-z1"}]}),
+ execution("arn-z1"),
+ ],
+ [
+ ("ListPipelines", {"MaxResults": 100, "SortBy": "Name", "SortOrder": "Ascending"}),
+ ("ListPipelines", {"NextToken": "p2"}),
+ ("ListPipelineExecutions", {"PipelineName": "alpha"}),
+ ("DescribePipelineExecution", {"PipelineExecutionArn": "arn-a1"}),
+ ("ListPipelineExecutions", {"PipelineName": "alpha", "NextToken": "a2"}),
+ ("ListPipelineExecutions", {"PipelineName": "zeta"}),
+ ("DescribePipelineExecution", {"PipelineExecutionArn": "arn-z1"}),
+ ],
+ ["arn-a1", "arn-z1"],
+ [
+ AwsSagemakerResumeConfig(next_token="a2", parent_name="alpha"),
+ AwsSagemakerResumeConfig(parent_name="zeta"),
+ AwsSagemakerResumeConfig(complete=True),
+ ],
+ ),
+ (
+ AwsSagemakerResumeConfig(next_token="m2", parent_name="middle"),
+ [
+ ScriptedResponse(
+ json={"PipelineSummaries": [{"PipelineName": "zeta"}, {"PipelineName": "alpha"}]},
+ ),
+ ScriptedResponse(json={"PipelineExecutionSummaries": [{"PipelineExecutionArn": "arn-z1"}]}),
+ execution("arn-z1"),
+ ],
+ [
+ ("ListPipelines", {}),
+ ("ListPipelineExecutions", {"PipelineName": "zeta"}),
+ ("DescribePipelineExecution", {"PipelineExecutionArn": "arn-z1"}),
+ ],
+ ["arn-z1"],
+ [AwsSagemakerResumeConfig(complete=True)],
+ ),
+ (
+ None,
+ [
+ ScriptedResponse(json={"PipelineSummaries": [{"PipelineName": "zeta"}, {"PipelineName": "alpha"}]}),
+ ScriptedResponse(status=400, json={"__type": "ResourceNotFound"}),
+ ScriptedResponse(json={"PipelineExecutionSummaries": [{"PipelineExecutionArn": "arn-z1"}]}),
+ execution("arn-z1"),
+ ],
+ [
+ ("ListPipelines", {}),
+ ("ListPipelineExecutions", {"PipelineName": "alpha"}),
+ ("ListPipelineExecutions", {"PipelineName": "zeta"}),
+ ("DescribePipelineExecution", {"PipelineExecutionArn": "arn-z1"}),
+ ],
+ ["arn-z1"],
+ [AwsSagemakerResumeConfig(parent_name="zeta"), AwsSagemakerResumeConfig(complete=True)],
+ ),
+ ],
+)
+def test_pipeline_executions_walk_every_pipeline_and_resume_from_the_saved_one(
+ config: AwsSagemakerSourceConfig,
+ resume_state: AwsSagemakerResumeConfig | None,
+ script: list[ScriptedResponse],
+ expected_targets: list[tuple[str, dict[str, object]]],
+ expected_rows: list[str],
+ expected_states: list[AwsSagemakerResumeConfig],
+) -> None:
+ result = driver(config).run("pipeline_executions", script, resume_state=resume_state)
+ assert result.raised is None
+ assert [row["pipeline_execution_arn"] for row in result.rows] == expected_rows
+ assert len(result.requests) == len(expected_targets)
+ for sent, (operation, payload) in zip(result.requests, expected_targets):
+ assert sent.headers["x-amz-target"] == f"SageMaker.{operation}"
+ assert payload.items() <= sent.json().items()
+ assert result.saved_states == expected_states
+
+
+def test_monitoring_executions_use_list_summaries_as_rows(config: AwsSagemakerSourceConfig) -> None:
+ summary = {
+ "MonitoringScheduleName": "schedule",
+ "ScheduledTime": 1704067200,
+ "CreationTime": 1704067260,
+ "LastModifiedTime": 1704067320,
+ "MonitoringExecutionStatus": "Failed",
+ "FailureReason": "Baseline missing",
+ }
+ result = driver(config).run(
+ "monitoring_executions",
+ [ScriptedResponse(json={"MonitoringExecutionSummaries": [summary, {**summary, "ScheduledTime": None}]})],
+ )
+ assert isinstance(result.raised, ValueError)
+ assert result.saved_states == []
+
+ result = driver(config).run(
+ "monitoring_executions", [ScriptedResponse(json={"MonitoringExecutionSummaries": [summary]})]
+ )
+ assert result.raised is None
+ assert len(result.requests) == 1
+ assert result.requests[0].headers["x-amz-target"] == "SageMaker.ListMonitoringExecutions"
+ assert result.rows == [
+ {
+ "monitoring_schedule_name": "schedule",
+ "scheduled_time": dt.datetime(2024, 1, 1, tzinfo=dt.UTC),
+ "creation_time": dt.datetime(2024, 1, 1, 0, 1, tzinfo=dt.UTC),
+ "last_modified_time": dt.datetime(2024, 1, 1, 0, 2, tzinfo=dt.UTC),
+ "monitoring_execution_status": "Failed",
+ "failure_reason": "Baseline missing",
+ }
+ ]
+ assert result.response and result.response.primary_keys == ["monitoring_schedule_name", "scheduled_time"]
+
+
@pytest.mark.parametrize("incremental", [True, False])
@pytest.mark.parametrize(
"last_value",
@@ -398,6 +567,47 @@ def test_validation_probes_describe_only_for_selected_table(
assert network.requests_log[1].json() == {"ModelName": "example"}
+ "schema_name,script,expected_targets",
+ [
+ (
+ "pipeline_executions",
+ [
+ ScriptedResponse(json={"PipelineSummaries": [{"PipelineName": "pipeline"}]}),
+ ScriptedResponse(json={"PipelineExecutionSummaries": [{"PipelineExecutionArn": "arn-1"}]}),
+ ScriptedResponse(json={"PipelineExecutionArn": "arn-1"}),
+ ],
+ [
+ ("ListPipelines", {"MaxResults": 1}),
+ ("ListPipelineExecutions", {"MaxResults": 1, "PipelineName": "pipeline"}),
+ ("DescribePipelineExecution", {"PipelineExecutionArn": "arn-1"}),
+ ],
+ ),
+ (
+ "pipeline_executions",
+ [ScriptedResponse(json={"PipelineSummaries": []})],
+ [("ListPipelines", {"MaxResults": 1})],
+ ),
+ (
+ "monitoring_executions",
+ [ScriptedResponse(json={"MonitoringExecutionSummaries": [{"MonitoringScheduleName": "schedule"}]})],
+ [("ListMonitoringExecutions", {"MaxResults": 1})],
+ ),
+ ],
+)
+def test_validation_probes_parent_and_skips_missing_describe(
+ config: AwsSagemakerSourceConfig,
+ schema_name: str,
+ script: list[ScriptedResponse],
+ expected_targets: list[tuple[str, dict[str, object]]],
+) -> None:
+ with scripted_network(script) as network:
+ assert validate_credentials(config, schema_name, "2017-07-24") == (True, None)
+ assert [(sent.headers["x-amz-target"], sent.json()) for sent in network.requests_log] == [
+ (f"SageMaker.{operation}", payload) for operation, payload in expected_targets
+ ]
+
+
@pytest.mark.parametrize(
"field,value,message",
[