Skip to content

Ministack: Free, open-source local AWS emulator - 60+ services, Terraform compatible, real databases. Free forever. MIT licensed.

fix(glue): give each Spark job script its --JOB_NAME and --JOB_RUN_ID (#2426)

mishukdutta-cz73221a268eb7
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 7b781381..50a629cc 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -8,6 +8,7 @@ Versioning follows [Semantic Versioning](https://semver.org/).
## [Unreleased]
### Fixed
+- **Glue — Spark job scripts get `--JOB_NAME` and `--JOB_RUN_ID`** — a Spark job run passed only the job's `DefaultArguments` and the run's `Arguments`, so a script that reads `JOB_NAME` or `JOB_RUN_ID` with `getResolvedOptions` failed; both now reach the script, as on AWS. They replace a value in `DefaultArguments`, and a value in the run's `Arguments` replaces them. `pythonshell` scripts don't get them, as on AWS. `GetJobRun` still shows only the arguments the caller gave.
- **Step Functions — JSONata fields on an inline `Map`** — `Items`, `ItemSelector`, `MaxConcurrency`, `Output` and `Assign` were ignored or read as JSONPath, so the Map iterated over the whole input, iterations got literal `{% ... %}` text, a `MaxConcurrency` expression failed with `States.Runtime`, and `Assign` variables were never set; all five are now evaluated, as on AWS. `Items` that is not an array fails with `States.QueryEvaluationError`.
- **Step Functions — JSONata `Catch` on `Map` and `Parallel` states** — a catcher's `Output` and `Assign` were ignored, so the caught error passed on unchanged and the catcher's variables were never set; both now apply, as on a `Task` state.
- **Step Functions — JSONata variables with the value `null`** — a variable assigned `null` read back as undefined, so `{% $var %}` failed with `States.QueryEvaluationError`; it now reads back as `null`, as on AWS. An unset variable still fails.
diff --git a/ministack/services/glue.py b/ministack/services/glue.py
index c67dc312..e7443b02 100644
--- a/ministack/services/glue.py
+++ b/ministack/services/glue.py
@@ -1706,6 +1706,15 @@ def _resolve_script(script_location):
return None
+def _script_args(job, job_name, run_id, run_arguments):
+ """Return the arguments that the job script gets."""
+ default_arguments = job.get("DefaultArguments", {})
+ if not _is_spark_job(job):
+ return {**default_arguments, **run_arguments}
+ # Glue gives a Spark script its job name and run ID. They replace DefaultArguments, and the run's Arguments replace them.
+ return {**default_arguments, "--JOB_NAME": job_name, "--JOB_RUN_ID": run_id, **run_arguments}
+
+
def _start_job_run(data):
job_name = data.get("JobName")
if job_name not in _jobs:
@@ -1740,6 +1749,8 @@ def _start_job_run(data):
_job_runs[job_name] = []
_job_runs[job_name].append(run)
+ script_args = _script_args(job, job_name, run_id, data.get("Arguments", {}))
+
def _execute():
run["JobRunState"] = "RUNNING"
run["LastModifiedOn"] = int(time.time())
@@ -1758,11 +1769,11 @@ def _start_job_run(data):
logger.info("Glue: image %s not available — stubbing job %s", image, job_name)
if use_docker:
- _execute_spark_docker(run, job, job_name, args, resolved, docker_client)
+ _execute_spark_docker(run, job, job_name, script_args, resolved, docker_client)
elif _is_spark_job(job):
run["JobRunState"] = "SUCCEEDED"
elif resolved and resolved.endswith(".py"):
- _execute_subprocess(run, job, args, resolved)
+ _execute_subprocess(run, job, script_args, resolved)
else:
run["JobRunState"] = "SUCCEEDED"
diff --git a/tests/test_glue.py b/tests/test_glue.py
index 34fdb4f8..72963e53 100644
--- a/tests/test_glue.py
+++ b/tests/test_glue.py
@@ -1387,6 +1387,34 @@ def test_glue_spark_skips_docker_when_image_missing(glue):
), f"Job should reach terminal state without Docker image. Got: {run['JobRunState']}"
+_USER_IDS = {"--JOB_NAME": "user-value", "--JOB_RUN_ID": "user-value"}
+_DEFAULT_IDS = {"--JOB_NAME": "default-value", "--JOB_RUN_ID": "default-value"}
+_GLUE_IDS = {"--JOB_NAME": "my-job", "--JOB_RUN_ID": "jr_1"}
+
+
[email protected]("command,default_arguments,run_arguments,expected", [
+ ("glueetl", {}, {}, _GLUE_IDS),
+ ("glueetl", {}, _USER_IDS, _USER_IDS),
+ ("glueetl", _DEFAULT_IDS, {}, _GLUE_IDS),
+ ("gluestreaming", {}, {}, _GLUE_IDS),
+ ("gluestreaming", {}, _USER_IDS, _USER_IDS),
+ ("gluestreaming", _DEFAULT_IDS, {}, _GLUE_IDS),
+ ("pythonshell", {}, {}, {}),
+ ("pythonshell", {}, _USER_IDS, _USER_IDS),
+ ("pythonshell", _DEFAULT_IDS, {}, _DEFAULT_IDS),
+], ids=["etl", "etl-run-args", "etl-default-args", "streaming", "streaming-run-args", "streaming-default-args",
+ "shell", "shell-run-args", "shell-default-args"])
+def test_glue_script_args_job_name_and_run_id(command, default_arguments, run_arguments, expected):
+ """A Spark script gets --JOB_NAME and --JOB_RUN_ID from Glue. They replace DefaultArguments, and the run's
+ Arguments replace them. A pythonshell script gets only the arguments that the caller gave."""
+ from ministack.services import glue as _glue
+
+ job = {"Command": {"Name": command}, "DefaultArguments": {"--extra": "x", **default_arguments}}
+ args = _glue._script_args(job, "my-job", "jr_1", run_arguments)
+
+ assert args == {"--extra": "x", **expected}
+
+
def test_glue_spark_image_for_version_maps_to_official_aws_image():
"""`GlueVersion: 4.0` and `3.0` map to the canonical `amazon/aws-glue-libs`
images real AWS Glue uses for Spark. Override via `GLUE_DOCKER_IMAGE`."""