Ministack: Free, open-source local AWS emulator - 60+ services, Terraform compatible, real databases. Free forever. MIT licensed.
Python awsaws-emulatoraws-localaws-sdkdevtoolsdockerdynamodbec2emulatorlambdalocalstacklocalstack-alternativeministackmock-awsopen-sourcepythons3sqsterraform
fix(glue): give each Spark job script its --JOB_NAME and --JOB_RUN_ID (#2426)
mishukdutta-cz73221a268eb7
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 7b781381..50a629cc 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -8,6 +8,7 @@ Versioning follows [Semantic Versioning](https://semver.org/).## [Unreleased]### Fixed+- **Glue — Spark job scripts get `--JOB_NAME` and `--JOB_RUN_ID`** — a Spark job run passed only the job's `DefaultArguments` and the run's `Arguments`, so a script that reads `JOB_NAME` or `JOB_RUN_ID` with `getResolvedOptions` failed; both now reach the script, as on AWS. They replace a value in `DefaultArguments`, and a value in the run's `Arguments` replaces them. `pythonshell` scripts don't get them, as on AWS. `GetJobRun` still shows only the arguments the caller gave.- **Step Functions — JSONata fields on an inline `Map`** — `Items`, `ItemSelector`, `MaxConcurrency`, `Output` and `Assign` were ignored or read as JSONPath, so the Map iterated over the whole input, iterations got literal `{% ... %}` text, a `MaxConcurrency` expression failed with `States.Runtime`, and `Assign` variables were never set; all five are now evaluated, as on AWS. `Items` that is not an array fails with `States.QueryEvaluationError`.- **Step Functions — JSONata `Catch` on `Map` and `Parallel` states** — a catcher's `Output` and `Assign` were ignored, so the caught error passed on unchanged and the catcher's variables were never set; both now apply, as on a `Task` state.- **Step Functions — JSONata variables with the value `null`** — a variable assigned `null` read back as undefined, so `{% $var %}` failed with `States.QueryEvaluationError`; it now reads back as `null`, as on AWS. An unset variable still fails.diff --git a/ministack/services/glue.py b/ministack/services/glue.pyindex c67dc312..e7443b02 100644--- a/ministack/services/glue.py+++ b/ministack/services/glue.py@@ -1706,6 +1706,15 @@ def _resolve_script(script_location):return None+def _script_args(job, job_name, run_id, run_arguments):+ """Return the arguments that the job script gets."""+ default_arguments = job.get("DefaultArguments", {})+ if not _is_spark_job(job):+ return {**default_arguments, **run_arguments}+ # Glue gives a Spark script its job name and run ID. They replace DefaultArguments, and the run's Arguments replace them.+ return {**default_arguments, "--JOB_NAME": job_name, "--JOB_RUN_ID": run_id, **run_arguments}++def _start_job_run(data):job_name = data.get("JobName")if job_name not in _jobs:@@ -1740,6 +1749,8 @@ def _start_job_run(data):_job_runs[job_name] = []_job_runs[job_name].append(run)+ script_args = _script_args(job, job_name, run_id, data.get("Arguments", {}))+def _execute():run["JobRunState"] = "RUNNING"run["LastModifiedOn"] = int(time.time())@@ -1758,11 +1769,11 @@ def _start_job_run(data):logger.info("Glue: image %s not available — stubbing job %s", image, job_name)if use_docker:- _execute_spark_docker(run, job, job_name, args, resolved, docker_client)+ _execute_spark_docker(run, job, job_name, script_args, resolved, docker_client)elif _is_spark_job(job):run["JobRunState"] = "SUCCEEDED"elif resolved and resolved.endswith(".py"):- _execute_subprocess(run, job, args, resolved)+ _execute_subprocess(run, job, script_args, resolved)else:run["JobRunState"] = "SUCCEEDED"diff --git a/tests/test_glue.py b/tests/test_glue.pyindex 34fdb4f8..72963e53 100644--- a/tests/test_glue.py+++ b/tests/test_glue.py@@ -1387,6 +1387,34 @@ def test_glue_spark_skips_docker_when_image_missing(glue):), f"Job should reach terminal state without Docker image. Got: {run['JobRunState']}"+_USER_IDS = {"--JOB_NAME": "user-value", "--JOB_RUN_ID": "user-value"}+_DEFAULT_IDS = {"--JOB_NAME": "default-value", "--JOB_RUN_ID": "default-value"}+_GLUE_IDS = {"--JOB_NAME": "my-job", "--JOB_RUN_ID": "jr_1"}++[email protected]("command,default_arguments,run_arguments,expected", [+ ("glueetl", {}, {}, _GLUE_IDS),+ ("glueetl", {}, _USER_IDS, _USER_IDS),+ ("glueetl", _DEFAULT_IDS, {}, _GLUE_IDS),+ ("gluestreaming", {}, {}, _GLUE_IDS),+ ("gluestreaming", {}, _USER_IDS, _USER_IDS),+ ("gluestreaming", _DEFAULT_IDS, {}, _GLUE_IDS),+ ("pythonshell", {}, {}, {}),+ ("pythonshell", {}, _USER_IDS, _USER_IDS),+ ("pythonshell", _DEFAULT_IDS, {}, _DEFAULT_IDS),+], ids=["etl", "etl-run-args", "etl-default-args", "streaming", "streaming-run-args", "streaming-default-args",+ "shell", "shell-run-args", "shell-default-args"])+def test_glue_script_args_job_name_and_run_id(command, default_arguments, run_arguments, expected):+ """A Spark script gets --JOB_NAME and --JOB_RUN_ID from Glue. They replace DefaultArguments, and the run's+ Arguments replace them. A pythonshell script gets only the arguments that the caller gave."""+ from ministack.services import glue as _glue++ job = {"Command": {"Name": command}, "DefaultArguments": {"--extra": "x", **default_arguments}}+ args = _glue._script_args(job, "my-job", "jr_1", run_arguments)++ assert args == {"--extra": "x", **expected}++def test_glue_spark_image_for_version_maps_to_official_aws_image():"""`GlueVersion: 4.0` and `3.0` map to the canonical `amazon/aws-glue-libs`images real AWS Glue uses for Spark. Override via `GLUE_DOCKER_IMAGE`."""