Skip to content

"CLI-Anything: Making ALL Software Agent-Native" -- CLI-Hub: https://clianything.cc/

Python

fix(openrefine): support piped REPL workflows (#385)

asahikiko53c56535dae6
diff --git a/openrefine/agent-harness/README.md b/openrefine/agent-harness/README.md
index bf2436798..52ac4263a 100644
--- a/openrefine/agent-harness/README.md
+++ b/openrefine/agent-harness/README.md
@@ -15,6 +15,12 @@ cli-anything-openrefine --help
cli-anything-openrefine
```
+Replay a scripted REPL user journey in CI or a shell pipeline:
+
+```bash
+printf 'help\nexit\n' | cli-anything-openrefine
+```
+
Start OpenRefine first for backend commands:
```bash
diff --git a/openrefine/agent-harness/cli_anything/openrefine/README.md b/openrefine/agent-harness/cli_anything/openrefine/README.md
index 0eac1021b..66cf2737f 100644
--- a/openrefine/agent-harness/cli_anything/openrefine/README.md
+++ b/openrefine/agent-harness/cli_anything/openrefine/README.md
@@ -12,6 +12,21 @@ cli-anything-openrefine --json data export clean.csv
Run `cli-anything-openrefine` with no arguments for the REPL.
+The REPL also accepts newline-separated commands from stdin, so user journeys
+can be replayed in CI without an interactive terminal:
+
+```bash
+printf 'help\nexit\n' | cli-anything-openrefine
+```
+
+Interactive terminals keep the Unicode banner, prompt history, and styling;
+redirected input uses ASCII-only output for cross-platform reliability,
+including Windows environments configured with legacy encodings.
+Unicode command payloads are preserved as ASCII backslash escapes in this
+mode, so values remain identifiable without triggering encoding failures.
+If any piped command fails, the REPL continues consuming the script but exits
+nonzero at `exit` or EOF so CI can reject the failed user journey.
+
Start OpenRefine first:
```bash
diff --git a/openrefine/agent-harness/cli_anything/openrefine/__init__.py b/openrefine/agent-harness/cli_anything/openrefine/__init__.py
index f22f701a0..42dce4ea7 100644
--- a/openrefine/agent-harness/cli_anything/openrefine/__init__.py
+++ b/openrefine/agent-harness/cli_anything/openrefine/__init__.py
@@ -1,3 +1,3 @@
"""CLI-Anything harness for OpenRefine."""
-__version__ = "1.0.0"
+__version__ = "1.0.1"
diff --git a/openrefine/agent-harness/cli_anything/openrefine/openrefine_cli.py b/openrefine/agent-harness/cli_anything/openrefine/openrefine_cli.py
index 4b0ac449a..2b579e18a 100644
--- a/openrefine/agent-harness/cli_anything/openrefine/openrefine_cli.py
+++ b/openrefine/agent-harness/cli_anything/openrefine/openrefine_cli.py
@@ -25,27 +25,57 @@ def _service(ctx: click.Context) -> OpenRefineService:
return OpenRefineService(OpenRefineBackend(base_url, timeout=ctx.obj["timeout"]), store)
-def _emit(data: Any, as_json: bool) -> None:
+def _ascii_safe(value: Any) -> str:
+ """Render text as reversible ASCII escapes for legacy output streams."""
+ return str(value).encode("ascii", errors="backslashreplace").decode("ascii")
+
+
+def _emit(data: Any, as_json: bool, ascii_safe: bool = False) -> None:
if as_json:
click.echo(json.dumps(data, indent=2, sort_keys=True))
elif isinstance(data, dict):
for key, value in data.items():
- click.echo(f"{key}: {value}")
+ text = f"{key}: {value}"
+ click.echo(_ascii_safe(text) if ascii_safe else text)
else:
- click.echo(str(data))
+ text = str(data)
+ click.echo(_ascii_safe(text) if ascii_safe else text)
def _handle(ctx: click.Context, func, *args, **kwargs) -> None:
try:
- _emit(func(*args, **kwargs), ctx.obj["json"])
+ _emit(
+ func(*args, **kwargs),
+ ctx.obj["json"],
+ ascii_safe=ctx.obj.get("ascii_output", False),
+ )
except (OpenRefineError, ValueError, OSError) as exc:
if ctx.obj["json"]:
click.echo(json.dumps({"error": str(exc), "ok": False}, indent=2, sort_keys=True), err=True)
else:
- click.echo(f"Error: {exc}", err=True)
+ message = f"Error: {exc}"
+ if ctx.obj.get("ascii_output", False):
+ message = _ascii_safe(message)
+ click.echo(message, err=True)
raise click.exceptions.Exit(1)
+def _supports_interactive_prompt(stdin=None, stdout=None) -> bool:
+ """Return whether prompt_toolkit can safely attach to the current terminal.
+
+ Redirected streams are common in CI, Click's ``CliRunner``, and shell
+ pipelines that replay a user workflow. On Windows prompt_toolkit raises
+ ``NoConsoleScreenBufferError`` for those streams instead of reading the
+ piped commands, so keep its rich prompt for real terminals only.
+ """
+ stdin = sys.stdin if stdin is None else stdin
+ stdout = sys.stdout if stdout is None else stdout
+ try:
+ return bool(stdin.isatty() and stdout.isatty())
+ except (AttributeError, OSError):
+ return False
+
+
@click.group(invoke_without_command=True)
@click.option("--base-url", default=None, help="OpenRefine URL. Defaults to OPENREFINE_URL, then session state, then http://127.0.0.1:3333.")
@click.option("--session", "session_path", type=click.Path(dir_okay=False), default=None, help="Session JSON path.")
@@ -68,8 +98,16 @@ def repl(ctx: click.Context) -> None:
"""Start the interactive REPL."""
history_file = _repl_history_file(ctx)
skin = ReplSkin("openrefine", version=__version__, history_file=history_file)
- skin.print_banner()
- prompt = skin.create_prompt_session()
+ interactive = _supports_interactive_prompt()
+ if interactive:
+ skin.print_banner()
+ prompt = skin.create_prompt_session()
+ else:
+ # Keep redirected workflows ASCII-only. Windows pipes and CI runners
+ # may expose legacy encodings such as cp1252, which cannot represent
+ # the skin's box-drawing banner, prompt arrow, or status icons.
+ prompt = None
+ ctx.obj["ascii_output"] = True
commands = {
"status": "Check backend and session",
"projects": "List OpenRefine projects",
@@ -80,37 +118,76 @@ def repl(ctx: click.Context) -> None:
"undo / redo": "Use OpenRefine undo-redo where possible",
"exit": "Quit",
}
+ had_error = False
while True:
try:
state = SessionStore(ctx.obj["session"]).load()
- line = skin.get_input(prompt, project_name=state.project_name)
+ if interactive:
+ line = skin.get_input(prompt, project_name=state.project_name)
+ else:
+ line = input().strip()
except (EOFError, KeyboardInterrupt):
- skin.print_goodbye()
+ if interactive:
+ skin.print_goodbye()
+ else:
+ click.echo("Goodbye!")
+ if had_error:
+ raise click.exceptions.Exit(1)
return
try:
parts = shlex.split(line)
except (IndexError, ValueError) as exc:
- skin.error(str(exc))
+ if interactive:
+ skin.error(str(exc))
+ else:
+ click.echo(f"Error: {_ascii_safe(exc)}", err=True)
+ had_error = True
continue
if not parts:
continue
try:
args = _repl_to_args(parts)
except (IndexError, ValueError) as exc:
- skin.error(str(exc))
+ if interactive:
+ skin.error(str(exc))
+ else:
+ click.echo(f"Error: {_ascii_safe(exc)}", err=True)
+ had_error = True
continue
if parts[0] in {"exit", "quit"}:
- skin.print_goodbye()
+ if interactive:
+ skin.print_goodbye()
+ else:
+ click.echo("Goodbye!")
+ if had_error:
+ raise click.exceptions.Exit(1)
return
if parts[0] == "help":
- skin.help(commands)
+ if interactive:
+ skin.help(commands)
+ else:
+ for command, description in commands.items():
+ click.echo(f"{command}: {description}")
continue
try:
- cli.main(args=_global_args(ctx) + args, prog_name="cli-anything-openrefine", obj=ctx.obj, standalone_mode=False)
- except SystemExit:
- pass
+ result = cli.main(
+ args=_global_args(ctx) + args,
+ prog_name="cli-anything-openrefine",
+ obj=ctx.obj,
+ standalone_mode=False,
+ )
+ if isinstance(result, int) and result != 0:
+ had_error = True
+ except (SystemExit, click.exceptions.Exit) as exc:
+ exit_code = getattr(exc, "exit_code", getattr(exc, "code", 0))
+ if exit_code:
+ had_error = True
except Exception as exc:
- skin.error(str(exc))
+ if interactive:
+ skin.error(str(exc))
+ else:
+ click.echo(f"Error: {_ascii_safe(exc)}", err=True)
+ had_error = True
def _repl_to_args(parts: list[str]) -> list[str]:
diff --git a/openrefine/agent-harness/cli_anything/openrefine/skills/SKILL.md b/openrefine/agent-harness/cli_anything/openrefine/skills/SKILL.md
index 2ef724c88..5ed836da9 100644
--- a/openrefine/agent-harness/cli_anything/openrefine/skills/SKILL.md
+++ b/openrefine/agent-harness/cli_anything/openrefine/skills/SKILL.md
@@ -51,6 +51,22 @@ cli-anything-openrefine --json --session run/session.json session redo
Run `cli-anything-openrefine` with no subcommand to enter the REPL.
+For automated user journeys, pipe newline-separated REPL commands through
+stdin. Redirected streams automatically use an ASCII-only input/output path
+while interactive terminals retain the Unicode banner, prompt history, and
+styling:
+
+```bash
+printf 'help\nexit\n' | cli-anything-openrefine
+```
+
+Unicode payloads are rendered as reversible ASCII backslash escapes when the
+REPL is redirected, preventing legacy Windows output encodings from turning a
+successful command into a failed journey.
+
+Piped workflows return a nonzero exit status at `exit` or EOF if any command
+failed, allowing CI to detect an unsuccessful user journey.
+
## Error Handling
When `--json` is set, command failures write a JSON object to stderr with `ok: false`.
diff --git a/openrefine/agent-harness/cli_anything/openrefine/tests/TEST.md b/openrefine/agent-harness/cli_anything/openrefine/tests/TEST.md
index 8190673fb..4ad9695c4 100644
--- a/openrefine/agent-harness/cli_anything/openrefine/tests/TEST.md
+++ b/openrefine/agent-harness/cli_anything/openrefine/tests/TEST.md
@@ -2,8 +2,8 @@
## Test Inventory Plan
-- `test_core.py`: 76 backend-free unit and CLI tests planned.
-- `test_full_e2e.py`: 12 real-backend E2E tests planned.
+- `test_core.py`: 81 backend-free unit and CLI tests planned.
+- `test_full_e2e.py`: 14 subprocess and real-backend E2E tests planned.
## Unit Test Plan
@@ -11,7 +11,7 @@
- `core.session`: default state, atomic save/load, record, undo, redo, empty-stack errors.
- `core.project`: service orchestration with fake backend, import/open/apply/export/rows, local and backend undo/redo behavior.
- `utils.openrefine_backend`: small pure helpers and error types.
-- `openrefine_cli`: help output, default REPL entry, JSON operation builder commands, session show, REPL command mapping.
+- `openrefine_cli`: help output, terminal capability detection, piped multi-step REPL journeys, JSON operation builder commands, session show, REPL command mapping.
## E2E Test Plan
@@ -24,6 +24,7 @@ It intentionally fails loudly when the backend is unavailable.
- **Cleaning operation history**: apply `core/text-transform` and verify exported CSV no longer contains padded names.
- **Normalization operation history**: apply `core/mass-edit` to city values and verify exported content.
- **Agent subprocess workflow**: run the installed or module CLI with `--json`, import data, inspect rows, export CSV, and parse exported rows with Python `csv`.
+- **Piped REPL workflow**: replay newline-separated user commands through the real CLI subprocess and verify the plain-input fallback exits cleanly on Windows and CI.
- **Operation file workflow**: build an operation-history JSON file via CLI, apply it to a backend project, and verify operation count.
- **State persistence**: verify session JSON persists current project and action history across subprocess calls.
- **Undo/redo recovery**: apply a backend operation and exercise OpenRefine undo/redo endpoints.
@@ -36,9 +37,17 @@ Unit suite run:
```text
$ python -m pytest cli_anything/openrefine/tests/test_core.py -q
-........................................................................ [ 94%]
-.... [100%]
-76 passed in 0.42s
+........................................................................ [ 88%]
+......... [100%]
+81 passed in 0.34s
+```
+
+Backend-independent subprocess checks:
+
+```text
+$ python -m pytest cli_anything/openrefine/tests/test_full_e2e.py -q -k "piped_user_commands or failed_piped_command or cli_help_subprocess"
+... [100%]
+3 passed, 11 deselected in 1.18s
```
Previous full suite run with OpenRefine 3.10.1 running at `http://127.0.0.1:3333`:
@@ -123,7 +132,7 @@ Collection check:
```text
$ python -m pytest cli_anything/openrefine/tests/ --collect-only -q
-88 tests collected in 0.17s
+95 tests collected in 0.06s
```
Setup metadata check:
@@ -132,18 +141,18 @@ Setup metadata check:
$ python setup.py --name
cli-anything-openrefine
$ python setup.py --version
-1.0.0
+1.0.1
```
## Summary Statistics
-- Total collected tests: 88
-- Backend-free unit tests: 76 passing
-- E2E tests: 12 collected and previously passing against a real OpenRefine 3.10.1 local HTTP backend
+- Total collected tests: 95
+- Backend-free unit tests: 81 passing
+- E2E tests: 14 collected; the 3 backend-independent subprocess checks pass locally, and the original 12-test suite previously passed against a real OpenRefine 3.10.1 local HTTP backend
- Minimum validator thresholds met: 50+ pytest tests and 10+ E2E pytest tests
## Coverage Notes
- Unit tests cover operation JSON builders, session persistence, fake-backend service orchestration, CLI JSON output, and default REPL entry.
-- E2E tests cover real backend import, metadata, row reads, operation application, CSV export verification, subprocess CLI workflows, session persistence, undo/redo, JSON error handling, and cleanup recovery.
+- E2E tests cover piped REPL user input, real backend import, metadata, row reads, operation application, CSV export verification, subprocess CLI workflows, session persistence, undo/redo, JSON error handling, and cleanup recovery.
- Reconciliation workflows are documented as a limitation and currently require applying exported OpenRefine reconciliation operation histories.
diff --git a/openrefine/agent-harness/cli_anything/openrefine/tests/test_core.py b/openrefine/agent-harness/cli_anything/openrefine/tests/test_core.py
index 37ac3e0f8..d36b3ce55 100644
--- a/openrefine/agent-harness/cli_anything/openrefine/tests/test_core.py
+++ b/openrefine/agent-harness/cli_anything/openrefine/tests/test_core.py
@@ -17,7 +17,7 @@ from cli_anything.openrefine.core.operations import (
from cli_anything.openrefine.core.project import OpenRefineService, _extract_project_id
from cli_anything.openrefine.core.session import SessionState, SessionStore
from cli_anything.openrefine import openrefine_cli
-from cli_anything.openrefine.openrefine_cli import _repl_to_args, cli
+from cli_anything.openrefine.openrefine_cli import _ascii_safe, _repl_to_args, _supports_interactive_prompt, cli
from cli_anything.openrefine.utils.openrefine_backend import OpenRefineBackend, OpenRefineError, _coerce_json_or_text
@@ -457,8 +457,67 @@ def test_cli_session_show_json_uses_custom_path(tmp_path):
def test_cli_default_enters_repl_and_exits():
result = CliRunner().invoke(cli, input="exit\n")
assert result.exit_code == 0
- assert "cli-anything" in result.output
- assert "Openrefine" in result.output
+ assert result.output == "Goodbye!\n"
+
+
+def test_prompt_toolkit_is_reserved_for_real_terminals():
+ class Stream:
+ def __init__(self, is_tty):
+ self.is_tty = is_tty
+
+ def isatty(self):
+ return self.is_tty
+
+ assert _supports_interactive_prompt(Stream(True), Stream(True))
+ assert not _supports_interactive_prompt(Stream(False), Stream(True))
+ assert not _supports_interactive_prompt(Stream(True), Stream(False))
+
+
+def test_ascii_safe_output_preserves_unicode_as_escapes():
+ assert _ascii_safe("Project 😀 café") == r"Project \U0001f600 caf\xe9"
+
+
+def test_cli_repl_accepts_piped_multi_step_user_journey(tmp_path, monkeypatch):
+ session = tmp_path / "session.json"
+ output = tmp_path / "clean export.csv"
+ monkeypatch.setattr(openrefine_cli, "OpenRefineBackend", FakeBackend)
+
+ commands = "\n".join(
+ [
+ "open 123",
+ "rows 2",
+ f'export "{output}" csv',
+ "exit",
+ "",
+ ]
+ )
+ result = CliRunner().invoke(cli, ["--session", str(session)], input=commands)
+
+ assert result.exit_code == 0, result.output
+ assert "project_id: 123" in result.output
+ assert "Alice" in result.output
+ assert output.read_text(encoding="utf-8").startswith("name,value")
+ state = SessionStore(session).load()
+ assert state.project_id == "123"
+ assert state.last_export == str(output)
+
+
+def test_cli_repl_piped_errors_are_ascii_safe():
+ result = CliRunner().invoke(cli, input='import "unterminated\nexit\n')
+
+ assert result.exit_code == 1
+ assert "Error: No closing quotation" in result.stderr
+ assert result.output.endswith("Goodbye!\n")
+
+
+def test_cli_repl_propagates_piped_command_failures(tmp_path):
+ for script in ("rows\n", "definitely-not-a-command\n"):
+ session = tmp_path / f"session-{len(script)}.json"
+ result = CliRunner().invoke(cli, ["--session", str(session)], input=script)
+
+ assert result.exit_code == 1
+ assert "Error:" in result.stderr
+ assert result.output.endswith("Goodbye!\n")
def test_openrefine_error_is_runtime_error():
diff --git a/openrefine/agent-harness/cli_anything/openrefine/tests/test_full_e2e.py b/openrefine/agent-harness/cli_anything/openrefine/tests/test_full_e2e.py
index 3f4ea1551..d3b82d2d3 100644
--- a/openrefine/agent-harness/cli_anything/openrefine/tests/test_full_e2e.py
+++ b/openrefine/agent-harness/cli_anything/openrefine/tests/test_full_e2e.py
@@ -7,11 +7,10 @@ import shutil
import subprocess
import sys
import time
-from pathlib import Path
import pytest
-from cli_anything.openrefine.utils.openrefine_backend import INSTALL_INSTRUCTIONS, OpenRefineBackend, OpenRefineError
+from cli_anything.openrefine.utils.openrefine_backend import INSTALL_INSTRUCTIONS, OpenRefineBackend
def _resolve_cli(name):
@@ -163,6 +162,54 @@ def test_e2e_cli_help_subprocess(cli_base):
assert "data" in result.stdout
+def test_e2e_cli_repl_accepts_piped_user_commands(cli_base, tmp_path):
+ env = os.environ.copy()
+ env.update({"NO_COLOR": "1", "PYTHONIOENCODING": "cp1252"})
+ session = tmp_path / "unicode-session.json"
+ session.write_text(
+ json.dumps({"project_name": "Project 😀"}, ensure_ascii=False),
+ encoding="utf-8",
+ )
+ result = subprocess.run(
+ cli_base + ["--session", str(session)],
+ input="session show\nhelp\nexit\n",
+ capture_output=True,
+ text=True,
+ encoding="cp1252",
+ check=False,
+ timeout=30,
+ env=env,
+ )
+ print("STDOUT:", result.stdout)
+ print("STDERR:", result.stderr)
+ assert result.returncode == 0
+ assert r"Project \U0001f600" in result.stdout
+ assert "List OpenRefine projects" in result.stdout
+ assert "Goodbye!" in result.stdout
+ assert "NoConsoleScreenBufferError" not in result.stderr
+ assert "UnicodeEncodeError" not in result.stderr
+
+
+def test_e2e_cli_repl_returns_nonzero_after_failed_piped_command(cli_base):
+ env = os.environ.copy()
+ env.update({"NO_COLOR": "1", "PYTHONIOENCODING": "cp1252"})
+ result = subprocess.run(
+ cli_base,
+ input="definitely-not-a-command\n",
+ capture_output=True,
+ text=True,
+ encoding="cp1252",
+ check=False,
+ timeout=30,
+ env=env,
+ )
+ print("STDOUT:", result.stdout)
+ print("STDERR:", result.stderr)
+ assert result.returncode == 1
+ assert "No such command 'definitely-not-a-command'" in result.stderr
+ assert result.stdout.endswith("Goodbye!\n")
+
+
def test_e2e_cli_json_import_rows_export_workflow(backend, cli_base, sample_csv, tmp_path, base_url):
session = tmp_path / "session.json"
imported = _run(cli_base, ["--json", "--base-url", base_url, "--session", str(session), "project", "import", str(sample_csv), "--name", "cli-anything-e2e-cli"])
diff --git a/openrefine/agent-harness/setup.py b/openrefine/agent-harness/setup.py
index 3e779ef0c..42781ce7f 100644
--- a/openrefine/agent-harness/setup.py
+++ b/openrefine/agent-harness/setup.py
@@ -3,7 +3,7 @@ from setuptools import find_namespace_packages, setup
setup(
name="cli-anything-openrefine",
- version="1.0.0",
+ version="1.0.1",
description="CLI-Anything harness for OpenRefine data wrangling workflows",
long_description="Agent-native Click CLI for OpenRefine's local HTTP API, operation histories, exports, and sessions.",
author="CLI-Anything-Team",
diff --git a/registry.json b/registry.json
index c6a17cc1f..18c1f4354 100644
--- a/registry.json
+++ b/registry.json
@@ -27,7 +27,7 @@
{
"name": "openrefine",
"display_name": "OpenRefine",
- "version": "1.0.0",
+ "version": "1.0.1",
"description": "Agent-native CLI for OpenRefine import, operation-history cleaning, row inspection, export, and session undo/redo through the real local HTTP API.",
"requires": "OpenRefine 3.10.x or newer running as a local web server",
"homepage": "https://openrefine.org/",
diff --git a/skills/cli-anything-openrefine/SKILL.md b/skills/cli-anything-openrefine/SKILL.md
index 2ef724c88..5ed836da9 100644
--- a/skills/cli-anything-openrefine/SKILL.md
+++ b/skills/cli-anything-openrefine/SKILL.md
@@ -51,6 +51,22 @@ cli-anything-openrefine --json --session run/session.json session redo
Run `cli-anything-openrefine` with no subcommand to enter the REPL.
+For automated user journeys, pipe newline-separated REPL commands through
+stdin. Redirected streams automatically use an ASCII-only input/output path
+while interactive terminals retain the Unicode banner, prompt history, and
+styling:
+
+```bash
+printf 'help\nexit\n' | cli-anything-openrefine
+```
+
+Unicode payloads are rendered as reversible ASCII backslash escapes when the
+REPL is redirected, preventing legacy Windows output encodings from turning a
+successful command into a failed journey.
+
+Piped workflows return a nonzero exit status at `exit` or EOF if any command
+failed, allowing CI to detect an unsuccessful user journey.
+
## Error Handling
When `--json` is set, command failures write a JSON object to stderr with `ok: false`.