Skip to content

Graph-Native Infrastructure for Context and Accountable AI Systems

fix(cli): repair the pipeline command group (#2019)

Nilay Mallikca6c9f1918ad
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 61c12f51..74e717aa 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -23,6 +23,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Fixed
+- **The `semantica pipeline` command group called APIs that do not exist** (fixes #2012) by @nilaymallikk
+ - `init`, `run`, `status` and `stop` all raised `AttributeError` before doing any work, calling `PipelineTemplateManager.scaffold()` and `ExecutionEngine.run()` / `.status()` / `.stop()`, none of which exist. `validate` reported `Pipeline config valid` for every input, including a file with a missing dependency, because it was handed the file *path string* where `PipelineValidator.validate()` expects a `Pipeline` — a `str` has no steps to complain about
+ - `init` now calls `create_pipeline_from_template()` and serialises the built `Pipeline` to the output file. Its `--template` choices and default were five names the manager has never registered, so each one would have reached `Template not found` next; they are the four real templates now, pinned against `PipelineTemplateManager.list_templates()` by a test
+ - `validate` and `run` both load the file through a new shared `_load_pipeline_file()` helper. `run` hands the `Pipeline` to `execute_pipeline()`, maps `--parallel` onto `pipeline.config["parallelism"]` and `--incremental` onto per-step delta mode, and refuses a pipeline whose steps have no handler instead of reporting success for a run that did nothing — a file-loaded pipeline can never carry handlers, so every step would pass data straight through. `--watch` has no engine implementation and now says so
+ - `status` and `stop` explain that the engine keeps pipelines in an in-memory dict, so neither can observe a pipeline owned by another process
+
- **`semantica backup create` reported success for an archive with no store data** (fixes #1820) by @costajohnt
- With no file-based `graph_db` / `vector_store` / `triplet_store` path in the config, `backup create` wrote a manifest with `"files": []` and exited 0. It now fails with an error that points to `semantica backup info`, before asking for a passphrase or confirmation and before writing anything.
diff --git a/semantica/cli.py b/semantica/cli.py
index d8d9e0af..019bf4ae 100644
--- a/semantica/cli.py
+++ b/semantica/cli.py
@@ -4851,14 +4851,42 @@ _viz_command("analytics", "visualize_analytics", "Render centrality and communit
# ─── Orchestration ────────────────────────────────────────────────────────────
+# Must stay in step with PipelineTemplateManager.list_templates(). The CLI used to
+# offer five names the manager has never registered, so every `pipeline init`
+# reached "Template not found" once the wrapper called the real API. Pinned by
+# tests/test_cli_commands.py::TestPipeline::test_init_templates_match_the_manager.
_PIPELINE_TEMPLATES = [
- "ingest-extract-kg", "rag", "ontology-build", "decision-track", "full",
+ "document_processing", "rag_pipeline", "kg_construction", "ontology_generation",
]
+def _load_pipeline_file(path: str) -> Any:
+ """Load a pipeline config file into a ``Pipeline`` object.
+
+ Accepts YAML, or JSON (which ``yaml.safe_load`` parses as a subset). Step
+ handlers are process-local and are never restored from a file, so every step
+ of a loaded pipeline has ``handler is None`` — see
+ ``PipelineSerializer.deserialize_pipeline``.
+ """
+ try:
+ from .pipeline import PipelineSerializer
+ except ImportError as exc:
+ raise click.ClickException(f"Pipeline module not available: {exc}") from exc
+ try:
+ payload = yaml.safe_load(Path(path).read_text(encoding="utf-8"))
+ except yaml.YAMLError as exc:
+ raise click.ClickException(f"Could not parse {path} as YAML: {exc}") from exc
+ if not isinstance(payload, dict):
+ raise click.ClickException(
+ f"{path} is not a pipeline definition: expected a mapping with a "
+ f"'steps' list, got {type(payload).__name__}."
+ )
+ return PipelineSerializer().deserialize_pipeline(payload)
+
+
@pipeline.command("init")
@click.option("--template", type=click.Choice(_PIPELINE_TEMPLATES),
- default="ingest-extract-kg", show_default=True)
+ default="document_processing", show_default=True)
@click.option("--output", default="pipeline.yaml", show_default=True, type=click.Path())
@click.option("--dry-run", "local_dry", is_flag=True, default=False)
@click.pass_obj
@@ -4867,7 +4895,7 @@ def pipeline_init(cli_ctx: CLIContext, template: str, output: str, local_dry: bo
\b
Example:
- semantica pipeline init --template ingest-extract-kg --output pipeline.yaml
+ semantica pipeline init --template document_processing --output pipeline.yaml
"""
cli_ctx = _require_ctx(cli_ctx)
@@ -4876,12 +4904,18 @@ def pipeline_init(cli_ctx: CLIContext, template: str, output: str, local_dry: bo
_dry(cli_ctx, "init pipeline", template=template, output=output)
return
try:
- from .pipeline import PipelineTemplateManager
- mgr = PipelineTemplateManager()
- content = mgr.scaffold(template)
- except ImportError:
- content = f"# Pipeline template: {template}\nsteps: []\n"
- Path(output).write_text(content, encoding="utf-8")
+ from .pipeline import PipelineSerializer, PipelineTemplateManager
+ except ImportError as exc:
+ raise click.ClickException(f"Pipeline module not available: {exc}") from exc
+ builder = PipelineTemplateManager().create_pipeline_from_template(template)
+ # serialize_pipeline(..., format="yaml") returns the dict form rather than
+ # a YAML string, so the dump happens here.
+ payload = PipelineSerializer().serialize_pipeline(
+ builder.build(), format="yaml",
+ )
+ Path(output).write_text(
+ yaml.safe_dump(payload, sort_keys=False), encoding="utf-8",
+ )
_ok(cli_ctx, f"Created {output} from template '{template}'")
_run_with_error_handling(_action)
@@ -4896,16 +4930,36 @@ def pipeline_validate(cli_ctx: CLIContext, pipeline_file: str, local_json: bool)
cli_ctx = _require_ctx(cli_ctx)
def _action() -> None:
+ pipeline = _load_pipeline_file(pipeline_file)
try:
from .pipeline import PipelineValidator
- v = PipelineValidator()
- result = v.validate(pipeline_file)
except ImportError as exc:
raise click.ClickException(f"Pipeline module not available: {exc}") from exc
+ # The old wrapper passed the file path straight to validate(). A str has no
+ # steps, so it type-checked nothing and reported valid for any input.
+ result = PipelineValidator().validate(pipeline)
+ errors = [str(e) for e in (getattr(result, "errors", None) or [])]
+ warnings_found = [str(w) for w in (getattr(result, "warnings", None) or [])]
if _is_json(cli_ctx, local_json):
- _jecho(result if isinstance(result, dict) else {"valid": bool(result)})
+ _jecho({
+ "valid": bool(getattr(result, "valid", False)),
+ "errors": errors,
+ "warnings": warnings_found,
+ "pipeline": pipeline_file,
+ })
+ elif not getattr(result, "valid", False):
+ for err in errors:
+ _warn(cli_ctx, err)
+ raise click.ClickException(
+ f"Pipeline config invalid: {pipeline_file} "
+ f"({len(errors)} error(s), {len(warnings_found)} warning(s))"
+ )
else:
- _ok(cli_ctx, f"Pipeline config valid: {pipeline_file}")
+ _ok(
+ cli_ctx,
+ f"Pipeline config valid: {pipeline_file} "
+ f"({len(warnings_found)} warning(s))",
+ )
_run_with_error_handling(_action)
@@ -4934,16 +4988,52 @@ def pipeline_run(cli_ctx: CLIContext, pipeline_file: str, parallel: int,
if _is_dry(cli_ctx, local_dry):
_dry(cli_ctx, "run pipeline", file=pipeline_file, parallel=parallel)
return
+ if watch:
+ raise click.ClickException(
+ "--watch has no implementation in the execution engine yet; "
+ "re-run the pipeline after each change instead."
+ )
+ pipeline = _load_pipeline_file(pipeline_file)
+ # _execute_step() returns its input untouched when step.handler is None, so
+ # an all-unhandled pipeline "succeeds" having done nothing. Refuse instead
+ # of reporting a green no-op.
+ unhandled = [
+ step.name for step in pipeline.steps
+ if getattr(step, "handler", None) is None
+ ]
+ if unhandled:
+ raise click.ClickException(
+ f"Cannot execute {pipeline_file}: {len(unhandled)} step(s) have no "
+ f"handler ({', '.join(unhandled)}). Step handlers are Python "
+ "callables and are never restored from a config file, so the engine "
+ "would pass data through every step and still report success. "
+ "Register them with PipelineBuilder.register_step_handler() and run "
+ "the pipeline in Python, or use `semantica pipeline validate` to "
+ "check the config."
+ )
+ # --parallel and --incremental have no run-level switch in the engine:
+ # parallelism is read from pipeline.config and delta mode is per step.
+ pipeline.config["parallelism"] = parallel
+ if incremental:
+ for step in pipeline.steps:
+ step.delta_mode = True
try:
from .pipeline import ExecutionEngine
- engine = ExecutionEngine(config=cli_ctx.config.to_dict())
- result = engine.run(
- pipeline_file, workers=parallel,
- incremental=incremental, watch=watch,
- )
except ImportError as exc:
raise click.ClickException(f"Pipeline module not available: {exc}") from exc
- _ok(cli_ctx, f"Pipeline completed: {result}")
+ engine = ExecutionEngine(config=cli_ctx.config.to_dict())
+ result = engine.execute_pipeline(pipeline)
+ if not result.success:
+ errors = [str(e) for e in (result.errors or [])]
+ raise click.ClickException(
+ f"Pipeline failed: {'; '.join(errors) if errors else 'no error detail'}"
+ )
+ elapsed = result.metadata.get("execution_time")
+ pipeline_id = result.metadata.get("pipeline_id", pipeline.name)
+ summary = f"Pipeline completed: {pipeline_id}"
+ if isinstance(elapsed, (int, float)):
+ summary += f" in {elapsed:.2f}s"
+ _ok(cli_ctx, summary)
_run_with_error_handling(_action)
@@ -4956,16 +5046,16 @@ def pipeline_status(cli_ctx: CLIContext, local_json: bool) -> None:
cli_ctx = _require_ctx(cli_ctx)
def _action() -> None:
- try:
- from .pipeline import ExecutionEngine
- engine = ExecutionEngine(config=cli_ctx.config.to_dict())
- result = engine.status()
- except ImportError as exc:
- raise click.ClickException(f"Pipeline module not available: {exc}") from exc
- if _is_json(cli_ctx, local_json):
- _jecho(result if isinstance(result, dict) else {"status": str(result)})
- else:
- _pprint(cli_ctx, result)
+ # ExecutionEngine.pipeline_status is an in-memory dict and
+ # get_pipeline_status() only reads it, so a pipeline started by another
+ # process is invisible here. There is no id to accept either: ids are
+ # pipeline.name and are never persisted.
+ raise click.ClickException(
+ "`pipeline status` has no source of truth: the execution engine keeps "
+ "pipeline status in memory, so a pipeline started by a separate "
+ "`semantica pipeline run` process cannot be observed from here. Track it "
+ "in the process that called ExecutionEngine.execute_pipeline() instead."
+ )
_run_with_error_handling(_action)
@@ -4977,13 +5067,17 @@ def pipeline_stop(cli_ctx: CLIContext) -> None:
cli_ctx = _require_ctx(cli_ctx)
def _action() -> None:
- try:
- from .pipeline import ExecutionEngine
- engine = ExecutionEngine(config=cli_ctx.config.to_dict())
- engine.stop()
- except ImportError as exc:
- raise click.ClickException(f"Pipeline module not available: {exc}") from exc
- _ok(cli_ctx, "Pipeline stopped.")
+ # stop_pipeline() does set a flag the run loop honours (execution_engine.py
+ # :273 and :320), but it takes a pipeline id and that flag lives on the
+ # engine instance doing the stopping, so it cannot reach a pipeline owned by
+ # another process.
+ raise click.ClickException(
+ "`pipeline stop` has nothing to stop: the execution engine tracks running "
+ "pipelines in memory only, so a pipeline started by a separate "
+ "`semantica pipeline run` process is not reachable from here. Call "
+ "ExecutionEngine.stop_pipeline(pipeline_id) in the process running the "
+ "pipeline instead."
+ )
_run_with_error_handling(_action)
diff --git a/tests/test_cli_commands.py b/tests/test_cli_commands.py
index 6cabeb2e..3b4ef954 100644
--- a/tests/test_cli_commands.py
+++ b/tests/test_cli_commands.py
@@ -2695,21 +2695,47 @@ class TestPipeline:
result = runner.invoke(cli_module.main, ["pipeline", "init", "--dry-run"])
_ok(result)
- def test_init_creates_file(self, runner, monkeypatch):
- fake_pl = _fake_module(
- PipelineTemplateManager=lambda: MagicMock(scaffold=lambda t: "steps: []\n"),
- )
- monkeypatch.setitem(__import__("sys").modules, "semantica.pipeline", fake_pl)
+ def test_init_templates_match_the_manager(self, runner):
+ # The CLI hardcoded five template names the manager has never registered,
+ # so every `pipeline init` would have reached "Template not found" once the
+ # wrapper called the real API. Keep the two lists in step.
+ from semantica.pipeline import PipelineTemplateManager
+
+ manager = PipelineTemplateManager()
+ assert cli_module._PIPELINE_TEMPLATES == manager.list_templates()
+
+ def test_init_creates_a_loadable_pipeline(self, runner):
with runner.isolated_filesystem():
- result = runner.invoke(cli_module.main, ["pipeline", "init",
- "--template", "rag", "--output", "pl.yaml"])
+ result = runner.invoke(
+ cli_module.main,
+ ["pipeline", "init", "--template", "rag_pipeline",
+ "--output", "pl.yaml"],
+ )
_ok(result)
- assert os.path.exists("pl.yaml")
+ pipeline = cli_module._load_pipeline_file("pl.yaml")
+ assert [s.name for s in pipeline.steps] == [
+ "ingest", "chunk", "embed", "store_vectors",
+ ]
- def test_init_template_choices(self, runner):
+ def test_init_writes_the_pipeline_not_a_placeholder(self, runner):
+ # The old ImportError path silently wrote "# Pipeline template: X\nsteps: []".
+ with runner.isolated_filesystem():
+ result = runner.invoke(
+ cli_module.main,
+ ["pipeline", "init", "--output", "pl.yaml"],
+ )
+ _ok(result)
+ text = open("pl.yaml").read()
+ assert "steps: []" not in text
+ assert "type: ingest" in text
+
+ def test_init_help_offers_only_real_templates(self, runner):
result = runner.invoke(cli_module.main, ["pipeline", "init", "--help"])
- for t in ["ingest-extract-kg", "rag", "ontology-build", "decision-track", "full"]:
- assert t in result.output
+ _ok(result)
+ for gone in ["ingest-extract-kg", "decision-track", "ontology-build"]:
+ assert gone not in result.output
+ for real in ["document_processing", "rag_pipeline"]:
+ assert real in result.output
def test_validate_requires_file(self, runner):
result = runner.invoke(cli_module.main, ["pipeline", "validate"])
@@ -2719,6 +2745,54 @@ class TestPipeline:
result = runner.invoke(cli_module.main, ["pipeline", "validate", "no_such.yaml"])
assert result.exit_code != 0
+ def test_validate_reads_the_file_instead_of_the_path(self, runner):
+ # Regression: validate() was handed the *path string*. A str has no steps,
+ # so it type-checked nothing and reported valid for any input.
+ with runner.isolated_filesystem():
+ with open("pl.yaml", "w") as f:
+ f.write("steps:\n - name: a\n type: ingest\n")
+ result = runner.invoke(
+ cli_module.main,
+ ["pipeline", "validate", "pl.yaml", "--json"],
+ )
+ _ok(result)
+ payload = json.loads([
+ line for line in result.output.splitlines() if line.startswith("{")
+ ][-1])
+ assert payload["valid"] is True
+ assert payload["pipeline"] == "pl.yaml"
+ # The loader leaves handlers off, and the validator says so.
+ assert any("no handler" in w for w in payload["warnings"])
+
+ def test_validate_reports_a_missing_dependency(self, runner):
+ # Before the fix this exact file was reported valid, because it was never
+ # loaded.
+ with runner.isolated_filesystem():
+ with open("pl.yaml", "w") as f:
+ f.write(
+ "steps:\n"
+ " - name: a\n"
+ " type: ingest\n"
+ " dependencies: [nope]\n"
+ )
+ result = runner.invoke(
+ cli_module.main,
+ ["pipeline", "validate", "pl.yaml"],
+ )
+ assert result.exit_code != 0
+ assert "Missing dependency 'nope'" in _flatten(result.output)
+
+ def test_validate_rejects_a_file_that_is_not_a_pipeline(self, runner):
+ with runner.isolated_filesystem():
+ with open("pl.yaml", "w") as f:
+ f.write("just a string\n")
+ result = runner.invoke(
+ cli_module.main,
+ ["pipeline", "validate", "pl.yaml"],
+ )
+ assert result.exit_code != 0
+ assert "not a pipeline definition" in _flatten(result.output)
+
def test_run_dry_run(self, runner):
with runner.isolated_filesystem():
with open("pl.yaml", "w") as f:
@@ -2730,22 +2804,110 @@ class TestPipeline:
result = runner.invoke(cli_module.main, ["pipeline", "run"])
assert result.exit_code != 0
- def test_status_exits_0(self, runner):
- with patch("builtins.__import__", side_effect=lambda n, *a, **k: (
- (_ for _ in ()).throw(ImportError(n))
- if "pipeline" in n else __import__(n, *a, **k)
- )):
- result = runner.invoke(cli_module.main, ["pipeline", "status"])
+ def test_run_refuses_steps_that_have_no_handler(self, runner):
+ # A pipeline loaded from a file can never carry handlers, and
+ # _execute_step() returns its input untouched when handler is None, so
+ # executing would report success having done nothing.
+ with runner.isolated_filesystem():
+ with open("pl.yaml", "w") as f:
+ f.write("steps:\n - name: ingest\n type: ingest\n")
+ result = runner.invoke(
+ cli_module.main,
+ ["pipeline", "run", "pl.yaml"],
+ )
+ assert result.exit_code != 0
+ flat = _flatten(result.output)
+ assert "no handler" in flat
+ assert "ingest" in flat
+
+ def test_run_executes_a_pipeline_with_handlers(self, runner, monkeypatch):
+ # The wiring itself: execute_pipeline() has to receive the loaded Pipeline,
+ # and --parallel has to land on pipeline.config, the only place the engine
+ # reads parallelism from.
+ from semantica.pipeline import PipelineBuilder
+
+ builder = PipelineBuilder()
+ builder.register_step_handler("ingest", lambda data, **kw: data)
+ builder.add_step("ingest", "ingest")
+ handled = builder.build()
+
+ seen = {}
+
+ class _Engine:
+ def __init__(self, config=None):
+ seen["config"] = config
+
+ def execute_pipeline(self, pipeline, **options):
+ seen["pipeline"] = pipeline
+ return _fake_module(
+ success=True,
+ errors=[],
+ metadata={"pipeline_id": pipeline.name, "execution_time": 0.5},
+ )
+
+ monkeypatch.setattr(cli_module, "_load_pipeline_file", lambda path: handled)
+ monkeypatch.setitem(__import__("sys").modules, "semantica.pipeline",
+ _fake_module(ExecutionEngine=_Engine))
+ with runner.isolated_filesystem():
+ with open("pl.yaml", "w") as f:
+ f.write("steps: []\n")
+ result = runner.invoke(
+ cli_module.main,
+ ["pipeline", "run", "pl.yaml", "--parallel", "7"],
+ )
+ _ok(result)
+ assert seen["pipeline"] is handled
+ assert seen["pipeline"].config["parallelism"] == 7
+ assert "Pipeline completed" in _flatten(result.output)
+
+ def test_run_reports_a_failed_pipeline(self, runner, monkeypatch):
+ from semantica.pipeline import PipelineBuilder
+
+ builder = PipelineBuilder()
+ builder.register_step_handler("ingest", lambda data, **kw: data)
+ builder.add_step("ingest", "ingest")
+ handled = builder.build()
+
+ class _Engine:
+ def __init__(self, config=None):
+ pass
+
+ def execute_pipeline(self, pipeline, **options):
+ return _fake_module(success=False, errors=["boom"], metadata={})
+
+ monkeypatch.setattr(cli_module, "_load_pipeline_file", lambda path: handled)
+ monkeypatch.setitem(__import__("sys").modules, "semantica.pipeline",
+ _fake_module(ExecutionEngine=_Engine))
+ with runner.isolated_filesystem():
+ with open("pl.yaml", "w") as f:
+ f.write("steps: []\n")
+ result = runner.invoke(cli_module.main, ["pipeline", "run", "pl.yaml"])
+ assert result.exit_code != 0
+ assert "boom" in _flatten(result.output)
+
+ def test_run_rejects_watch(self, runner):
+ with runner.isolated_filesystem():
+ with open("pl.yaml", "w") as f:
+ f.write("steps:\n - name: ingest\n type: ingest\n")
+ result = runner.invoke(
+ cli_module.main,
+ ["pipeline", "run", "pl.yaml", "--watch"],
+ )
assert result.exit_code != 0
+ assert "--watch" in _flatten(result.output)
+
+ def test_status_reports_it_cannot_observe_another_process(self, runner):
+ # The engine keeps pipeline status in memory, so a separate `pipeline run`
+ # process is not observable from here.
+ result = runner.invoke(cli_module.main, ["pipeline", "status"])
+ assert result.exit_code != 0
+ assert "no source of truth" in _flatten(result.output)
assert "Traceback" not in result.output
- def test_stop_exits_cleanly_on_import_error(self, runner):
- with patch("builtins.__import__", side_effect=lambda n, *a, **k: (
- (_ for _ in ()).throw(ImportError(n))
- if "pipeline" in n else __import__(n, *a, **k)
- )):
- result = runner.invoke(cli_module.main, ["pipeline", "stop"])
+ def test_stop_reports_it_cannot_reach_another_process(self, runner):
+ result = runner.invoke(cli_module.main, ["pipeline", "stop"])
assert result.exit_code != 0
+ assert "nothing to stop" in _flatten(result.output)
assert "Traceback" not in result.output