Graph-Native Infrastructure for Context and Accountable AI Systems
Python agent-memoryaiai-governanceai-infrastructureartificial-intelligencecontext-engineeringcontext-graphsdata-engineeringdecision-intelligencedeveloper-toolsexplainable-aigenerative-aigraph-ragknowledge-graphllmontologyprovenancepythonreasoningsemantic-search
fix(cli): repair the pipeline command group (#2019)
Nilay Mallikca6c9f1918ad
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 61c12f51..74e717aa 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -23,6 +23,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0### Fixed+- **The `semantica pipeline` command group called APIs that do not exist** (fixes #2012) by @nilaymallikk+ - `init`, `run`, `status` and `stop` all raised `AttributeError` before doing any work, calling `PipelineTemplateManager.scaffold()` and `ExecutionEngine.run()` / `.status()` / `.stop()`, none of which exist. `validate` reported `Pipeline config valid` for every input, including a file with a missing dependency, because it was handed the file *path string* where `PipelineValidator.validate()` expects a `Pipeline` — a `str` has no steps to complain about+ - `init` now calls `create_pipeline_from_template()` and serialises the built `Pipeline` to the output file. Its `--template` choices and default were five names the manager has never registered, so each one would have reached `Template not found` next; they are the four real templates now, pinned against `PipelineTemplateManager.list_templates()` by a test+ - `validate` and `run` both load the file through a new shared `_load_pipeline_file()` helper. `run` hands the `Pipeline` to `execute_pipeline()`, maps `--parallel` onto `pipeline.config["parallelism"]` and `--incremental` onto per-step delta mode, and refuses a pipeline whose steps have no handler instead of reporting success for a run that did nothing — a file-loaded pipeline can never carry handlers, so every step would pass data straight through. `--watch` has no engine implementation and now says so+ - `status` and `stop` explain that the engine keeps pipelines in an in-memory dict, so neither can observe a pipeline owned by another process+- **`semantica backup create` reported success for an archive with no store data** (fixes #1820) by @costajohnt- With no file-based `graph_db` / `vector_store` / `triplet_store` path in the config, `backup create` wrote a manifest with `"files": []` and exited 0. It now fails with an error that points to `semantica backup info`, before asking for a passphrase or confirmation and before writing anything.diff --git a/semantica/cli.py b/semantica/cli.pyindex d8d9e0af..019bf4ae 100644--- a/semantica/cli.py+++ b/semantica/cli.py@@ -4851,14 +4851,42 @@ _viz_command("analytics", "visualize_analytics", "Render centrality and communit# ─── Orchestration ────────────────────────────────────────────────────────────+# Must stay in step with PipelineTemplateManager.list_templates(). The CLI used to+# offer five names the manager has never registered, so every `pipeline init`+# reached "Template not found" once the wrapper called the real API. Pinned by+# tests/test_cli_commands.py::TestPipeline::test_init_templates_match_the_manager._PIPELINE_TEMPLATES = [- "ingest-extract-kg", "rag", "ontology-build", "decision-track", "full",+ "document_processing", "rag_pipeline", "kg_construction", "ontology_generation",]+def _load_pipeline_file(path: str) -> Any:+ """Load a pipeline config file into a ``Pipeline`` object.++ Accepts YAML, or JSON (which ``yaml.safe_load`` parses as a subset). Step+ handlers are process-local and are never restored from a file, so every step+ of a loaded pipeline has ``handler is None`` — see+ ``PipelineSerializer.deserialize_pipeline``.+ """+ try:+ from .pipeline import PipelineSerializer+ except ImportError as exc:+ raise click.ClickException(f"Pipeline module not available: {exc}") from exc+ try:+ payload = yaml.safe_load(Path(path).read_text(encoding="utf-8"))+ except yaml.YAMLError as exc:+ raise click.ClickException(f"Could not parse {path} as YAML: {exc}") from exc+ if not isinstance(payload, dict):+ raise click.ClickException(+ f"{path} is not a pipeline definition: expected a mapping with a "+ f"'steps' list, got {type(payload).__name__}."+ )+ return PipelineSerializer().deserialize_pipeline(payload)++@pipeline.command("init")@click.option("--template", type=click.Choice(_PIPELINE_TEMPLATES),- default="ingest-extract-kg", show_default=True)+ default="document_processing", show_default=True)@click.option("--output", default="pipeline.yaml", show_default=True, type=click.Path())@click.option("--dry-run", "local_dry", is_flag=True, default=False)@click.pass_obj@@ -4867,7 +4895,7 @@ def pipeline_init(cli_ctx: CLIContext, template: str, output: str, local_dry: bo\bExample:- semantica pipeline init --template ingest-extract-kg --output pipeline.yaml+ semantica pipeline init --template document_processing --output pipeline.yaml"""cli_ctx = _require_ctx(cli_ctx)@@ -4876,12 +4904,18 @@ def pipeline_init(cli_ctx: CLIContext, template: str, output: str, local_dry: bo_dry(cli_ctx, "init pipeline", template=template, output=output)returntry:- from .pipeline import PipelineTemplateManager- mgr = PipelineTemplateManager()- content = mgr.scaffold(template)- except ImportError:- content = f"# Pipeline template: {template}\nsteps: []\n"- Path(output).write_text(content, encoding="utf-8")+ from .pipeline import PipelineSerializer, PipelineTemplateManager+ except ImportError as exc:+ raise click.ClickException(f"Pipeline module not available: {exc}") from exc+ builder = PipelineTemplateManager().create_pipeline_from_template(template)+ # serialize_pipeline(..., format="yaml") returns the dict form rather than+ # a YAML string, so the dump happens here.+ payload = PipelineSerializer().serialize_pipeline(+ builder.build(), format="yaml",+ )+ Path(output).write_text(+ yaml.safe_dump(payload, sort_keys=False), encoding="utf-8",+ )_ok(cli_ctx, f"Created {output} from template '{template}'")_run_with_error_handling(_action)@@ -4896,16 +4930,36 @@ def pipeline_validate(cli_ctx: CLIContext, pipeline_file: str, local_json: bool)cli_ctx = _require_ctx(cli_ctx)def _action() -> None:+ pipeline = _load_pipeline_file(pipeline_file)try:from .pipeline import PipelineValidator- v = PipelineValidator()- result = v.validate(pipeline_file)except ImportError as exc:raise click.ClickException(f"Pipeline module not available: {exc}") from exc+ # The old wrapper passed the file path straight to validate(). A str has no+ # steps, so it type-checked nothing and reported valid for any input.+ result = PipelineValidator().validate(pipeline)+ errors = [str(e) for e in (getattr(result, "errors", None) or [])]+ warnings_found = [str(w) for w in (getattr(result, "warnings", None) or [])]if _is_json(cli_ctx, local_json):- _jecho(result if isinstance(result, dict) else {"valid": bool(result)})+ _jecho({+ "valid": bool(getattr(result, "valid", False)),+ "errors": errors,+ "warnings": warnings_found,+ "pipeline": pipeline_file,+ })+ elif not getattr(result, "valid", False):+ for err in errors:+ _warn(cli_ctx, err)+ raise click.ClickException(+ f"Pipeline config invalid: {pipeline_file} "+ f"({len(errors)} error(s), {len(warnings_found)} warning(s))"+ )else:- _ok(cli_ctx, f"Pipeline config valid: {pipeline_file}")+ _ok(+ cli_ctx,+ f"Pipeline config valid: {pipeline_file} "+ f"({len(warnings_found)} warning(s))",+ )_run_with_error_handling(_action)@@ -4934,16 +4988,52 @@ def pipeline_run(cli_ctx: CLIContext, pipeline_file: str, parallel: int,if _is_dry(cli_ctx, local_dry):_dry(cli_ctx, "run pipeline", file=pipeline_file, parallel=parallel)return+ if watch:+ raise click.ClickException(+ "--watch has no implementation in the execution engine yet; "+ "re-run the pipeline after each change instead."+ )+ pipeline = _load_pipeline_file(pipeline_file)+ # _execute_step() returns its input untouched when step.handler is None, so+ # an all-unhandled pipeline "succeeds" having done nothing. Refuse instead+ # of reporting a green no-op.+ unhandled = [+ step.name for step in pipeline.steps+ if getattr(step, "handler", None) is None+ ]+ if unhandled:+ raise click.ClickException(+ f"Cannot execute {pipeline_file}: {len(unhandled)} step(s) have no "+ f"handler ({', '.join(unhandled)}). Step handlers are Python "+ "callables and are never restored from a config file, so the engine "+ "would pass data through every step and still report success. "+ "Register them with PipelineBuilder.register_step_handler() and run "+ "the pipeline in Python, or use `semantica pipeline validate` to "+ "check the config."+ )+ # --parallel and --incremental have no run-level switch in the engine:+ # parallelism is read from pipeline.config and delta mode is per step.+ pipeline.config["parallelism"] = parallel+ if incremental:+ for step in pipeline.steps:+ step.delta_mode = Truetry:from .pipeline import ExecutionEngine- engine = ExecutionEngine(config=cli_ctx.config.to_dict())- result = engine.run(- pipeline_file, workers=parallel,- incremental=incremental, watch=watch,- )except ImportError as exc:raise click.ClickException(f"Pipeline module not available: {exc}") from exc- _ok(cli_ctx, f"Pipeline completed: {result}")+ engine = ExecutionEngine(config=cli_ctx.config.to_dict())+ result = engine.execute_pipeline(pipeline)+ if not result.success:+ errors = [str(e) for e in (result.errors or [])]+ raise click.ClickException(+ f"Pipeline failed: {'; '.join(errors) if errors else 'no error detail'}"+ )+ elapsed = result.metadata.get("execution_time")+ pipeline_id = result.metadata.get("pipeline_id", pipeline.name)+ summary = f"Pipeline completed: {pipeline_id}"+ if isinstance(elapsed, (int, float)):+ summary += f" in {elapsed:.2f}s"+ _ok(cli_ctx, summary)_run_with_error_handling(_action)@@ -4956,16 +5046,16 @@ def pipeline_status(cli_ctx: CLIContext, local_json: bool) -> None:cli_ctx = _require_ctx(cli_ctx)def _action() -> None:- try:- from .pipeline import ExecutionEngine- engine = ExecutionEngine(config=cli_ctx.config.to_dict())- result = engine.status()- except ImportError as exc:- raise click.ClickException(f"Pipeline module not available: {exc}") from exc- if _is_json(cli_ctx, local_json):- _jecho(result if isinstance(result, dict) else {"status": str(result)})- else:- _pprint(cli_ctx, result)+ # ExecutionEngine.pipeline_status is an in-memory dict and+ # get_pipeline_status() only reads it, so a pipeline started by another+ # process is invisible here. There is no id to accept either: ids are+ # pipeline.name and are never persisted.+ raise click.ClickException(+ "`pipeline status` has no source of truth: the execution engine keeps "+ "pipeline status in memory, so a pipeline started by a separate "+ "`semantica pipeline run` process cannot be observed from here. Track it "+ "in the process that called ExecutionEngine.execute_pipeline() instead."+ )_run_with_error_handling(_action)@@ -4977,13 +5067,17 @@ def pipeline_stop(cli_ctx: CLIContext) -> None:cli_ctx = _require_ctx(cli_ctx)def _action() -> None:- try:- from .pipeline import ExecutionEngine- engine = ExecutionEngine(config=cli_ctx.config.to_dict())- engine.stop()- except ImportError as exc:- raise click.ClickException(f"Pipeline module not available: {exc}") from exc- _ok(cli_ctx, "Pipeline stopped.")+ # stop_pipeline() does set a flag the run loop honours (execution_engine.py+ # :273 and :320), but it takes a pipeline id and that flag lives on the+ # engine instance doing the stopping, so it cannot reach a pipeline owned by+ # another process.+ raise click.ClickException(+ "`pipeline stop` has nothing to stop: the execution engine tracks running "+ "pipelines in memory only, so a pipeline started by a separate "+ "`semantica pipeline run` process is not reachable from here. Call "+ "ExecutionEngine.stop_pipeline(pipeline_id) in the process running the "+ "pipeline instead."+ )_run_with_error_handling(_action)diff --git a/tests/test_cli_commands.py b/tests/test_cli_commands.pyindex 6cabeb2e..3b4ef954 100644--- a/tests/test_cli_commands.py+++ b/tests/test_cli_commands.py@@ -2695,21 +2695,47 @@ class TestPipeline:result = runner.invoke(cli_module.main, ["pipeline", "init", "--dry-run"])_ok(result)- def test_init_creates_file(self, runner, monkeypatch):- fake_pl = _fake_module(- PipelineTemplateManager=lambda: MagicMock(scaffold=lambda t: "steps: []\n"),- )- monkeypatch.setitem(__import__("sys").modules, "semantica.pipeline", fake_pl)+ def test_init_templates_match_the_manager(self, runner):+ # The CLI hardcoded five template names the manager has never registered,+ # so every `pipeline init` would have reached "Template not found" once the+ # wrapper called the real API. Keep the two lists in step.+ from semantica.pipeline import PipelineTemplateManager++ manager = PipelineTemplateManager()+ assert cli_module._PIPELINE_TEMPLATES == manager.list_templates()++ def test_init_creates_a_loadable_pipeline(self, runner):with runner.isolated_filesystem():- result = runner.invoke(cli_module.main, ["pipeline", "init",- "--template", "rag", "--output", "pl.yaml"])+ result = runner.invoke(+ cli_module.main,+ ["pipeline", "init", "--template", "rag_pipeline",+ "--output", "pl.yaml"],+ )_ok(result)- assert os.path.exists("pl.yaml")+ pipeline = cli_module._load_pipeline_file("pl.yaml")+ assert [s.name for s in pipeline.steps] == [+ "ingest", "chunk", "embed", "store_vectors",+ ]- def test_init_template_choices(self, runner):+ def test_init_writes_the_pipeline_not_a_placeholder(self, runner):+ # The old ImportError path silently wrote "# Pipeline template: X\nsteps: []".+ with runner.isolated_filesystem():+ result = runner.invoke(+ cli_module.main,+ ["pipeline", "init", "--output", "pl.yaml"],+ )+ _ok(result)+ text = open("pl.yaml").read()+ assert "steps: []" not in text+ assert "type: ingest" in text++ def test_init_help_offers_only_real_templates(self, runner):result = runner.invoke(cli_module.main, ["pipeline", "init", "--help"])- for t in ["ingest-extract-kg", "rag", "ontology-build", "decision-track", "full"]:- assert t in result.output+ _ok(result)+ for gone in ["ingest-extract-kg", "decision-track", "ontology-build"]:+ assert gone not in result.output+ for real in ["document_processing", "rag_pipeline"]:+ assert real in result.outputdef test_validate_requires_file(self, runner):result = runner.invoke(cli_module.main, ["pipeline", "validate"])@@ -2719,6 +2745,54 @@ class TestPipeline:result = runner.invoke(cli_module.main, ["pipeline", "validate", "no_such.yaml"])assert result.exit_code != 0+ def test_validate_reads_the_file_instead_of_the_path(self, runner):+ # Regression: validate() was handed the *path string*. A str has no steps,+ # so it type-checked nothing and reported valid for any input.+ with runner.isolated_filesystem():+ with open("pl.yaml", "w") as f:+ f.write("steps:\n - name: a\n type: ingest\n")+ result = runner.invoke(+ cli_module.main,+ ["pipeline", "validate", "pl.yaml", "--json"],+ )+ _ok(result)+ payload = json.loads([+ line for line in result.output.splitlines() if line.startswith("{")+ ][-1])+ assert payload["valid"] is True+ assert payload["pipeline"] == "pl.yaml"+ # The loader leaves handlers off, and the validator says so.+ assert any("no handler" in w for w in payload["warnings"])++ def test_validate_reports_a_missing_dependency(self, runner):+ # Before the fix this exact file was reported valid, because it was never+ # loaded.+ with runner.isolated_filesystem():+ with open("pl.yaml", "w") as f:+ f.write(+ "steps:\n"+ " - name: a\n"+ " type: ingest\n"+ " dependencies: [nope]\n"+ )+ result = runner.invoke(+ cli_module.main,+ ["pipeline", "validate", "pl.yaml"],+ )+ assert result.exit_code != 0+ assert "Missing dependency 'nope'" in _flatten(result.output)++ def test_validate_rejects_a_file_that_is_not_a_pipeline(self, runner):+ with runner.isolated_filesystem():+ with open("pl.yaml", "w") as f:+ f.write("just a string\n")+ result = runner.invoke(+ cli_module.main,+ ["pipeline", "validate", "pl.yaml"],+ )+ assert result.exit_code != 0+ assert "not a pipeline definition" in _flatten(result.output)+def test_run_dry_run(self, runner):with runner.isolated_filesystem():with open("pl.yaml", "w") as f:@@ -2730,22 +2804,110 @@ class TestPipeline:result = runner.invoke(cli_module.main, ["pipeline", "run"])assert result.exit_code != 0- def test_status_exits_0(self, runner):- with patch("builtins.__import__", side_effect=lambda n, *a, **k: (- (_ for _ in ()).throw(ImportError(n))- if "pipeline" in n else __import__(n, *a, **k)- )):- result = runner.invoke(cli_module.main, ["pipeline", "status"])+ def test_run_refuses_steps_that_have_no_handler(self, runner):+ # A pipeline loaded from a file can never carry handlers, and+ # _execute_step() returns its input untouched when handler is None, so+ # executing would report success having done nothing.+ with runner.isolated_filesystem():+ with open("pl.yaml", "w") as f:+ f.write("steps:\n - name: ingest\n type: ingest\n")+ result = runner.invoke(+ cli_module.main,+ ["pipeline", "run", "pl.yaml"],+ )+ assert result.exit_code != 0+ flat = _flatten(result.output)+ assert "no handler" in flat+ assert "ingest" in flat++ def test_run_executes_a_pipeline_with_handlers(self, runner, monkeypatch):+ # The wiring itself: execute_pipeline() has to receive the loaded Pipeline,+ # and --parallel has to land on pipeline.config, the only place the engine+ # reads parallelism from.+ from semantica.pipeline import PipelineBuilder++ builder = PipelineBuilder()+ builder.register_step_handler("ingest", lambda data, **kw: data)+ builder.add_step("ingest", "ingest")+ handled = builder.build()++ seen = {}++ class _Engine:+ def __init__(self, config=None):+ seen["config"] = config++ def execute_pipeline(self, pipeline, **options):+ seen["pipeline"] = pipeline+ return _fake_module(+ success=True,+ errors=[],+ metadata={"pipeline_id": pipeline.name, "execution_time": 0.5},+ )++ monkeypatch.setattr(cli_module, "_load_pipeline_file", lambda path: handled)+ monkeypatch.setitem(__import__("sys").modules, "semantica.pipeline",+ _fake_module(ExecutionEngine=_Engine))+ with runner.isolated_filesystem():+ with open("pl.yaml", "w") as f:+ f.write("steps: []\n")+ result = runner.invoke(+ cli_module.main,+ ["pipeline", "run", "pl.yaml", "--parallel", "7"],+ )+ _ok(result)+ assert seen["pipeline"] is handled+ assert seen["pipeline"].config["parallelism"] == 7+ assert "Pipeline completed" in _flatten(result.output)++ def test_run_reports_a_failed_pipeline(self, runner, monkeypatch):+ from semantica.pipeline import PipelineBuilder++ builder = PipelineBuilder()+ builder.register_step_handler("ingest", lambda data, **kw: data)+ builder.add_step("ingest", "ingest")+ handled = builder.build()++ class _Engine:+ def __init__(self, config=None):+ pass++ def execute_pipeline(self, pipeline, **options):+ return _fake_module(success=False, errors=["boom"], metadata={})++ monkeypatch.setattr(cli_module, "_load_pipeline_file", lambda path: handled)+ monkeypatch.setitem(__import__("sys").modules, "semantica.pipeline",+ _fake_module(ExecutionEngine=_Engine))+ with runner.isolated_filesystem():+ with open("pl.yaml", "w") as f:+ f.write("steps: []\n")+ result = runner.invoke(cli_module.main, ["pipeline", "run", "pl.yaml"])+ assert result.exit_code != 0+ assert "boom" in _flatten(result.output)++ def test_run_rejects_watch(self, runner):+ with runner.isolated_filesystem():+ with open("pl.yaml", "w") as f:+ f.write("steps:\n - name: ingest\n type: ingest\n")+ result = runner.invoke(+ cli_module.main,+ ["pipeline", "run", "pl.yaml", "--watch"],+ )assert result.exit_code != 0+ assert "--watch" in _flatten(result.output)++ def test_status_reports_it_cannot_observe_another_process(self, runner):+ # The engine keeps pipeline status in memory, so a separate `pipeline run`+ # process is not observable from here.+ result = runner.invoke(cli_module.main, ["pipeline", "status"])+ assert result.exit_code != 0+ assert "no source of truth" in _flatten(result.output)assert "Traceback" not in result.output- def test_stop_exits_cleanly_on_import_error(self, runner):- with patch("builtins.__import__", side_effect=lambda n, *a, **k: (- (_ for _ in ()).throw(ImportError(n))- if "pipeline" in n else __import__(n, *a, **k)- )):- result = runner.invoke(cli_module.main, ["pipeline", "stop"])+ def test_stop_reports_it_cannot_reach_another_process(self, runner):+ result = runner.invoke(cli_module.main, ["pipeline", "stop"])assert result.exit_code != 0+ assert "nothing to stop" in _flatten(result.output)assert "Traceback" not in result.output