"CLI-Anything: Making ALL Software Agent-Native" -- CLI-Hub: https://clianything.cc/
Python
fix(openrefine): support piped REPL workflows (#385)
asahikiko53c56535dae6
diff --git a/openrefine/agent-harness/README.md b/openrefine/agent-harness/README.mdindex bf2436798..52ac4263a 100644--- a/openrefine/agent-harness/README.md+++ b/openrefine/agent-harness/README.md@@ -15,6 +15,12 @@ cli-anything-openrefine --helpcli-anything-openrefine```+Replay a scripted REPL user journey in CI or a shell pipeline:++```bash+printf 'help\nexit\n' | cli-anything-openrefine+```+Start OpenRefine first for backend commands:```bashdiff --git a/openrefine/agent-harness/cli_anything/openrefine/README.md b/openrefine/agent-harness/cli_anything/openrefine/README.mdindex 0eac1021b..66cf2737f 100644--- a/openrefine/agent-harness/cli_anything/openrefine/README.md+++ b/openrefine/agent-harness/cli_anything/openrefine/README.md@@ -12,6 +12,21 @@ cli-anything-openrefine --json data export clean.csvRun `cli-anything-openrefine` with no arguments for the REPL.+The REPL also accepts newline-separated commands from stdin, so user journeys+can be replayed in CI without an interactive terminal:++```bash+printf 'help\nexit\n' | cli-anything-openrefine+```++Interactive terminals keep the Unicode banner, prompt history, and styling;+redirected input uses ASCII-only output for cross-platform reliability,+including Windows environments configured with legacy encodings.+Unicode command payloads are preserved as ASCII backslash escapes in this+mode, so values remain identifiable without triggering encoding failures.+If any piped command fails, the REPL continues consuming the script but exits+nonzero at `exit` or EOF so CI can reject the failed user journey.+Start OpenRefine first:```bashdiff --git a/openrefine/agent-harness/cli_anything/openrefine/__init__.py b/openrefine/agent-harness/cli_anything/openrefine/__init__.pyindex f22f701a0..42dce4ea7 100644--- a/openrefine/agent-harness/cli_anything/openrefine/__init__.py+++ b/openrefine/agent-harness/cli_anything/openrefine/__init__.py@@ -1,3 +1,3 @@"""CLI-Anything harness for OpenRefine."""-__version__ = "1.0.0"+__version__ = "1.0.1"diff --git a/openrefine/agent-harness/cli_anything/openrefine/openrefine_cli.py b/openrefine/agent-harness/cli_anything/openrefine/openrefine_cli.pyindex 4b0ac449a..2b579e18a 100644--- a/openrefine/agent-harness/cli_anything/openrefine/openrefine_cli.py+++ b/openrefine/agent-harness/cli_anything/openrefine/openrefine_cli.py@@ -25,27 +25,57 @@ def _service(ctx: click.Context) -> OpenRefineService:return OpenRefineService(OpenRefineBackend(base_url, timeout=ctx.obj["timeout"]), store)-def _emit(data: Any, as_json: bool) -> None:+def _ascii_safe(value: Any) -> str:+ """Render text as reversible ASCII escapes for legacy output streams."""+ return str(value).encode("ascii", errors="backslashreplace").decode("ascii")+++def _emit(data: Any, as_json: bool, ascii_safe: bool = False) -> None:if as_json:click.echo(json.dumps(data, indent=2, sort_keys=True))elif isinstance(data, dict):for key, value in data.items():- click.echo(f"{key}: {value}")+ text = f"{key}: {value}"+ click.echo(_ascii_safe(text) if ascii_safe else text)else:- click.echo(str(data))+ text = str(data)+ click.echo(_ascii_safe(text) if ascii_safe else text)def _handle(ctx: click.Context, func, *args, **kwargs) -> None:try:- _emit(func(*args, **kwargs), ctx.obj["json"])+ _emit(+ func(*args, **kwargs),+ ctx.obj["json"],+ ascii_safe=ctx.obj.get("ascii_output", False),+ )except (OpenRefineError, ValueError, OSError) as exc:if ctx.obj["json"]:click.echo(json.dumps({"error": str(exc), "ok": False}, indent=2, sort_keys=True), err=True)else:- click.echo(f"Error: {exc}", err=True)+ message = f"Error: {exc}"+ if ctx.obj.get("ascii_output", False):+ message = _ascii_safe(message)+ click.echo(message, err=True)raise click.exceptions.Exit(1)+def _supports_interactive_prompt(stdin=None, stdout=None) -> bool:+ """Return whether prompt_toolkit can safely attach to the current terminal.++ Redirected streams are common in CI, Click's ``CliRunner``, and shell+ pipelines that replay a user workflow. On Windows prompt_toolkit raises+ ``NoConsoleScreenBufferError`` for those streams instead of reading the+ piped commands, so keep its rich prompt for real terminals only.+ """+ stdin = sys.stdin if stdin is None else stdin+ stdout = sys.stdout if stdout is None else stdout+ try:+ return bool(stdin.isatty() and stdout.isatty())+ except (AttributeError, OSError):+ return False++@click.group(invoke_without_command=True)@click.option("--base-url", default=None, help="OpenRefine URL. Defaults to OPENREFINE_URL, then session state, then http://127.0.0.1:3333.")@click.option("--session", "session_path", type=click.Path(dir_okay=False), default=None, help="Session JSON path.")@@ -68,8 +98,16 @@ def repl(ctx: click.Context) -> None:"""Start the interactive REPL."""history_file = _repl_history_file(ctx)skin = ReplSkin("openrefine", version=__version__, history_file=history_file)- skin.print_banner()- prompt = skin.create_prompt_session()+ interactive = _supports_interactive_prompt()+ if interactive:+ skin.print_banner()+ prompt = skin.create_prompt_session()+ else:+ # Keep redirected workflows ASCII-only. Windows pipes and CI runners+ # may expose legacy encodings such as cp1252, which cannot represent+ # the skin's box-drawing banner, prompt arrow, or status icons.+ prompt = None+ ctx.obj["ascii_output"] = Truecommands = {"status": "Check backend and session","projects": "List OpenRefine projects",@@ -80,37 +118,76 @@ def repl(ctx: click.Context) -> None:"undo / redo": "Use OpenRefine undo-redo where possible","exit": "Quit",}+ had_error = Falsewhile True:try:state = SessionStore(ctx.obj["session"]).load()- line = skin.get_input(prompt, project_name=state.project_name)+ if interactive:+ line = skin.get_input(prompt, project_name=state.project_name)+ else:+ line = input().strip()except (EOFError, KeyboardInterrupt):- skin.print_goodbye()+ if interactive:+ skin.print_goodbye()+ else:+ click.echo("Goodbye!")+ if had_error:+ raise click.exceptions.Exit(1)returntry:parts = shlex.split(line)except (IndexError, ValueError) as exc:- skin.error(str(exc))+ if interactive:+ skin.error(str(exc))+ else:+ click.echo(f"Error: {_ascii_safe(exc)}", err=True)+ had_error = Truecontinueif not parts:continuetry:args = _repl_to_args(parts)except (IndexError, ValueError) as exc:- skin.error(str(exc))+ if interactive:+ skin.error(str(exc))+ else:+ click.echo(f"Error: {_ascii_safe(exc)}", err=True)+ had_error = Truecontinueif parts[0] in {"exit", "quit"}:- skin.print_goodbye()+ if interactive:+ skin.print_goodbye()+ else:+ click.echo("Goodbye!")+ if had_error:+ raise click.exceptions.Exit(1)returnif parts[0] == "help":- skin.help(commands)+ if interactive:+ skin.help(commands)+ else:+ for command, description in commands.items():+ click.echo(f"{command}: {description}")continuetry:- cli.main(args=_global_args(ctx) + args, prog_name="cli-anything-openrefine", obj=ctx.obj, standalone_mode=False)- except SystemExit:- pass+ result = cli.main(+ args=_global_args(ctx) + args,+ prog_name="cli-anything-openrefine",+ obj=ctx.obj,+ standalone_mode=False,+ )+ if isinstance(result, int) and result != 0:+ had_error = True+ except (SystemExit, click.exceptions.Exit) as exc:+ exit_code = getattr(exc, "exit_code", getattr(exc, "code", 0))+ if exit_code:+ had_error = Trueexcept Exception as exc:- skin.error(str(exc))+ if interactive:+ skin.error(str(exc))+ else:+ click.echo(f"Error: {_ascii_safe(exc)}", err=True)+ had_error = Truedef _repl_to_args(parts: list[str]) -> list[str]:diff --git a/openrefine/agent-harness/cli_anything/openrefine/skills/SKILL.md b/openrefine/agent-harness/cli_anything/openrefine/skills/SKILL.mdindex 2ef724c88..5ed836da9 100644--- a/openrefine/agent-harness/cli_anything/openrefine/skills/SKILL.md+++ b/openrefine/agent-harness/cli_anything/openrefine/skills/SKILL.md@@ -51,6 +51,22 @@ cli-anything-openrefine --json --session run/session.json session redoRun `cli-anything-openrefine` with no subcommand to enter the REPL.+For automated user journeys, pipe newline-separated REPL commands through+stdin. Redirected streams automatically use an ASCII-only input/output path+while interactive terminals retain the Unicode banner, prompt history, and+styling:++```bash+printf 'help\nexit\n' | cli-anything-openrefine+```++Unicode payloads are rendered as reversible ASCII backslash escapes when the+REPL is redirected, preventing legacy Windows output encodings from turning a+successful command into a failed journey.++Piped workflows return a nonzero exit status at `exit` or EOF if any command+failed, allowing CI to detect an unsuccessful user journey.+## Error HandlingWhen `--json` is set, command failures write a JSON object to stderr with `ok: false`.diff --git a/openrefine/agent-harness/cli_anything/openrefine/tests/TEST.md b/openrefine/agent-harness/cli_anything/openrefine/tests/TEST.mdindex 8190673fb..4ad9695c4 100644--- a/openrefine/agent-harness/cli_anything/openrefine/tests/TEST.md+++ b/openrefine/agent-harness/cli_anything/openrefine/tests/TEST.md@@ -2,8 +2,8 @@## Test Inventory Plan-- `test_core.py`: 76 backend-free unit and CLI tests planned.-- `test_full_e2e.py`: 12 real-backend E2E tests planned.+- `test_core.py`: 81 backend-free unit and CLI tests planned.+- `test_full_e2e.py`: 14 subprocess and real-backend E2E tests planned.## Unit Test Plan@@ -11,7 +11,7 @@- `core.session`: default state, atomic save/load, record, undo, redo, empty-stack errors.- `core.project`: service orchestration with fake backend, import/open/apply/export/rows, local and backend undo/redo behavior.- `utils.openrefine_backend`: small pure helpers and error types.-- `openrefine_cli`: help output, default REPL entry, JSON operation builder commands, session show, REPL command mapping.+- `openrefine_cli`: help output, terminal capability detection, piped multi-step REPL journeys, JSON operation builder commands, session show, REPL command mapping.## E2E Test Plan@@ -24,6 +24,7 @@ It intentionally fails loudly when the backend is unavailable.- **Cleaning operation history**: apply `core/text-transform` and verify exported CSV no longer contains padded names.- **Normalization operation history**: apply `core/mass-edit` to city values and verify exported content.- **Agent subprocess workflow**: run the installed or module CLI with `--json`, import data, inspect rows, export CSV, and parse exported rows with Python `csv`.+- **Piped REPL workflow**: replay newline-separated user commands through the real CLI subprocess and verify the plain-input fallback exits cleanly on Windows and CI.- **Operation file workflow**: build an operation-history JSON file via CLI, apply it to a backend project, and verify operation count.- **State persistence**: verify session JSON persists current project and action history across subprocess calls.- **Undo/redo recovery**: apply a backend operation and exercise OpenRefine undo/redo endpoints.@@ -36,9 +37,17 @@ Unit suite run:```text$ python -m pytest cli_anything/openrefine/tests/test_core.py -q-........................................................................ [ 94%]-.... [100%]-76 passed in 0.42s+........................................................................ [ 88%]+......... [100%]+81 passed in 0.34s+```++Backend-independent subprocess checks:++```text+$ python -m pytest cli_anything/openrefine/tests/test_full_e2e.py -q -k "piped_user_commands or failed_piped_command or cli_help_subprocess"+... [100%]+3 passed, 11 deselected in 1.18s```Previous full suite run with OpenRefine 3.10.1 running at `http://127.0.0.1:3333`:@@ -123,7 +132,7 @@ Collection check:```text$ python -m pytest cli_anything/openrefine/tests/ --collect-only -q-88 tests collected in 0.17s+95 tests collected in 0.06s```Setup metadata check:@@ -132,18 +141,18 @@ Setup metadata check:$ python setup.py --namecli-anything-openrefine$ python setup.py --version-1.0.0+1.0.1```## Summary Statistics-- Total collected tests: 88-- Backend-free unit tests: 76 passing-- E2E tests: 12 collected and previously passing against a real OpenRefine 3.10.1 local HTTP backend+- Total collected tests: 95+- Backend-free unit tests: 81 passing+- E2E tests: 14 collected; the 3 backend-independent subprocess checks pass locally, and the original 12-test suite previously passed against a real OpenRefine 3.10.1 local HTTP backend- Minimum validator thresholds met: 50+ pytest tests and 10+ E2E pytest tests## Coverage Notes- Unit tests cover operation JSON builders, session persistence, fake-backend service orchestration, CLI JSON output, and default REPL entry.-- E2E tests cover real backend import, metadata, row reads, operation application, CSV export verification, subprocess CLI workflows, session persistence, undo/redo, JSON error handling, and cleanup recovery.+- E2E tests cover piped REPL user input, real backend import, metadata, row reads, operation application, CSV export verification, subprocess CLI workflows, session persistence, undo/redo, JSON error handling, and cleanup recovery.- Reconciliation workflows are documented as a limitation and currently require applying exported OpenRefine reconciliation operation histories.diff --git a/openrefine/agent-harness/cli_anything/openrefine/tests/test_core.py b/openrefine/agent-harness/cli_anything/openrefine/tests/test_core.pyindex 37ac3e0f8..d36b3ce55 100644--- a/openrefine/agent-harness/cli_anything/openrefine/tests/test_core.py+++ b/openrefine/agent-harness/cli_anything/openrefine/tests/test_core.py@@ -17,7 +17,7 @@ from cli_anything.openrefine.core.operations import (from cli_anything.openrefine.core.project import OpenRefineService, _extract_project_idfrom cli_anything.openrefine.core.session import SessionState, SessionStorefrom cli_anything.openrefine import openrefine_cli-from cli_anything.openrefine.openrefine_cli import _repl_to_args, cli+from cli_anything.openrefine.openrefine_cli import _ascii_safe, _repl_to_args, _supports_interactive_prompt, clifrom cli_anything.openrefine.utils.openrefine_backend import OpenRefineBackend, OpenRefineError, _coerce_json_or_text@@ -457,8 +457,67 @@ def test_cli_session_show_json_uses_custom_path(tmp_path):def test_cli_default_enters_repl_and_exits():result = CliRunner().invoke(cli, input="exit\n")assert result.exit_code == 0- assert "cli-anything" in result.output- assert "Openrefine" in result.output+ assert result.output == "Goodbye!\n"+++def test_prompt_toolkit_is_reserved_for_real_terminals():+ class Stream:+ def __init__(self, is_tty):+ self.is_tty = is_tty++ def isatty(self):+ return self.is_tty++ assert _supports_interactive_prompt(Stream(True), Stream(True))+ assert not _supports_interactive_prompt(Stream(False), Stream(True))+ assert not _supports_interactive_prompt(Stream(True), Stream(False))+++def test_ascii_safe_output_preserves_unicode_as_escapes():+ assert _ascii_safe("Project 😀 café") == r"Project \U0001f600 caf\xe9"+++def test_cli_repl_accepts_piped_multi_step_user_journey(tmp_path, monkeypatch):+ session = tmp_path / "session.json"+ output = tmp_path / "clean export.csv"+ monkeypatch.setattr(openrefine_cli, "OpenRefineBackend", FakeBackend)++ commands = "\n".join(+ [+ "open 123",+ "rows 2",+ f'export "{output}" csv',+ "exit",+ "",+ ]+ )+ result = CliRunner().invoke(cli, ["--session", str(session)], input=commands)++ assert result.exit_code == 0, result.output+ assert "project_id: 123" in result.output+ assert "Alice" in result.output+ assert output.read_text(encoding="utf-8").startswith("name,value")+ state = SessionStore(session).load()+ assert state.project_id == "123"+ assert state.last_export == str(output)+++def test_cli_repl_piped_errors_are_ascii_safe():+ result = CliRunner().invoke(cli, input='import "unterminated\nexit\n')++ assert result.exit_code == 1+ assert "Error: No closing quotation" in result.stderr+ assert result.output.endswith("Goodbye!\n")+++def test_cli_repl_propagates_piped_command_failures(tmp_path):+ for script in ("rows\n", "definitely-not-a-command\n"):+ session = tmp_path / f"session-{len(script)}.json"+ result = CliRunner().invoke(cli, ["--session", str(session)], input=script)++ assert result.exit_code == 1+ assert "Error:" in result.stderr+ assert result.output.endswith("Goodbye!\n")def test_openrefine_error_is_runtime_error():diff --git a/openrefine/agent-harness/cli_anything/openrefine/tests/test_full_e2e.py b/openrefine/agent-harness/cli_anything/openrefine/tests/test_full_e2e.pyindex 3f4ea1551..d3b82d2d3 100644--- a/openrefine/agent-harness/cli_anything/openrefine/tests/test_full_e2e.py+++ b/openrefine/agent-harness/cli_anything/openrefine/tests/test_full_e2e.py@@ -7,11 +7,10 @@ import shutilimport subprocessimport sysimport time-from pathlib import Pathimport pytest-from cli_anything.openrefine.utils.openrefine_backend import INSTALL_INSTRUCTIONS, OpenRefineBackend, OpenRefineError+from cli_anything.openrefine.utils.openrefine_backend import INSTALL_INSTRUCTIONS, OpenRefineBackenddef _resolve_cli(name):@@ -163,6 +162,54 @@ def test_e2e_cli_help_subprocess(cli_base):assert "data" in result.stdout+def test_e2e_cli_repl_accepts_piped_user_commands(cli_base, tmp_path):+ env = os.environ.copy()+ env.update({"NO_COLOR": "1", "PYTHONIOENCODING": "cp1252"})+ session = tmp_path / "unicode-session.json"+ session.write_text(+ json.dumps({"project_name": "Project 😀"}, ensure_ascii=False),+ encoding="utf-8",+ )+ result = subprocess.run(+ cli_base + ["--session", str(session)],+ input="session show\nhelp\nexit\n",+ capture_output=True,+ text=True,+ encoding="cp1252",+ check=False,+ timeout=30,+ env=env,+ )+ print("STDOUT:", result.stdout)+ print("STDERR:", result.stderr)+ assert result.returncode == 0+ assert r"Project \U0001f600" in result.stdout+ assert "List OpenRefine projects" in result.stdout+ assert "Goodbye!" in result.stdout+ assert "NoConsoleScreenBufferError" not in result.stderr+ assert "UnicodeEncodeError" not in result.stderr+++def test_e2e_cli_repl_returns_nonzero_after_failed_piped_command(cli_base):+ env = os.environ.copy()+ env.update({"NO_COLOR": "1", "PYTHONIOENCODING": "cp1252"})+ result = subprocess.run(+ cli_base,+ input="definitely-not-a-command\n",+ capture_output=True,+ text=True,+ encoding="cp1252",+ check=False,+ timeout=30,+ env=env,+ )+ print("STDOUT:", result.stdout)+ print("STDERR:", result.stderr)+ assert result.returncode == 1+ assert "No such command 'definitely-not-a-command'" in result.stderr+ assert result.stdout.endswith("Goodbye!\n")++def test_e2e_cli_json_import_rows_export_workflow(backend, cli_base, sample_csv, tmp_path, base_url):session = tmp_path / "session.json"imported = _run(cli_base, ["--json", "--base-url", base_url, "--session", str(session), "project", "import", str(sample_csv), "--name", "cli-anything-e2e-cli"])diff --git a/openrefine/agent-harness/setup.py b/openrefine/agent-harness/setup.pyindex 3e779ef0c..42781ce7f 100644--- a/openrefine/agent-harness/setup.py+++ b/openrefine/agent-harness/setup.py@@ -3,7 +3,7 @@ from setuptools import find_namespace_packages, setupsetup(name="cli-anything-openrefine",- version="1.0.0",+ version="1.0.1",description="CLI-Anything harness for OpenRefine data wrangling workflows",long_description="Agent-native Click CLI for OpenRefine's local HTTP API, operation histories, exports, and sessions.",author="CLI-Anything-Team",diff --git a/registry.json b/registry.jsonindex c6a17cc1f..18c1f4354 100644--- a/registry.json+++ b/registry.json@@ -27,7 +27,7 @@{"name": "openrefine","display_name": "OpenRefine",- "version": "1.0.0",+ "version": "1.0.1","description": "Agent-native CLI for OpenRefine import, operation-history cleaning, row inspection, export, and session undo/redo through the real local HTTP API.","requires": "OpenRefine 3.10.x or newer running as a local web server","homepage": "https://openrefine.org/",diff --git a/skills/cli-anything-openrefine/SKILL.md b/skills/cli-anything-openrefine/SKILL.mdindex 2ef724c88..5ed836da9 100644--- a/skills/cli-anything-openrefine/SKILL.md+++ b/skills/cli-anything-openrefine/SKILL.md@@ -51,6 +51,22 @@ cli-anything-openrefine --json --session run/session.json session redoRun `cli-anything-openrefine` with no subcommand to enter the REPL.+For automated user journeys, pipe newline-separated REPL commands through+stdin. Redirected streams automatically use an ASCII-only input/output path+while interactive terminals retain the Unicode banner, prompt history, and+styling:++```bash+printf 'help\nexit\n' | cli-anything-openrefine+```++Unicode payloads are rendered as reversible ASCII backslash escapes when the+REPL is redirected, preventing legacy Windows output encodings from turning a+successful command into a failed journey.++Piped workflows return a nonzero exit status at `exit` or EOF if any command+failed, allowing CI to detect an unsuccessful user journey.+## Error HandlingWhen `--json` is set, command failures write a JSON object to stderr with `ok: false`.