Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(evals): escape Markdown cell content in the report's slice labels

Aashish9dc4f7e756b4
diff --git a/laya/evals.py b/laya/evals.py
index 20f17a84..405dadc0 100644
--- a/laya/evals.py
+++ b/laya/evals.py
@@ -714,6 +714,27 @@ def _answered_model(result: Any) -> Optional[Any]:
return result.get("model")
+def _markdown_cell(value: Any) -> str:
+ """One slice label, written as a Markdown table cell rather than as table syntax.
+
+ The group key is caller data straight off the dataset: `tags` and question ids are any string
+ the author wrote, and `Example.from_dict` coerces them with `str` rather than restricting them.
+ A table reader takes an unescaped `|` as the start of the next column and a newline as the end
+ of the row, so a group labelled `risk|high` shifts every number in its row one header to the
+ left and drops the last, and a label containing a newline takes its number out of the table
+ altogether. The label is what a reader of `laya-evals run --markdown` matches a number against,
+ so a shifted row is a wrong figure rather than a typo -- and only that group's row moves, which
+ is what makes it hard to spot in a rendered report.
+
+ `\\` is escaped before `|` so a backslash at the end of a label cannot swallow the escape that
+ follows it. `\\|` is the repository's own convention for a literal pipe inside a cell: the
+ parity table in `BENCHMARKS.md` spells its `max |p_fast - p_stock|` columns that way, and
+ `tests/test_doc_tables.py` turns the escape back into a pipe to read the header.
+ """
+ return (str(value).replace("\\", "\\\\").replace("|", "\\|")
+ .replace("\r", " ").replace("\n", " "))
+
+
# --------------------------------------------------------------------------- evaluation
@dataclass
class EvalReport:
@@ -742,7 +763,7 @@ class EvalReport:
for value in sorted(self.slices[dimension]):
cells = ["%.4f" % self.slices[dimension][value].get(m, float("nan"))
if m in self.slices[dimension][value] else "" for m in metrics]
- lines.append("| %s | %s |" % (value, " | ".join(cells)))
+ lines.append("| %s | %s |" % (_markdown_cell(value), " | ".join(cells)))
return "\n".join(lines) + "\n"
def comparable_to(self, baseline: Dict[str, Any]) -> Tuple[bool, List[str]]:
diff --git a/tests/test_evals.py b/tests/test_evals.py
index f8ad6564..cca55881 100644
--- a/tests/test_evals.py
+++ b/tests/test_evals.py
@@ -777,6 +777,161 @@ def test_evaluate_overall_and_slices():
assert "choice_accuracy" in report.to_markdown()
+# --------------------------------------------------------------- markdown cells
+def _markdown_cells(line):
+ """Split one table row on its *unescaped* pipes, turning `\\|` back into a literal one.
+
+ Read left to right, a backslash escapes the character after it, so `\\\\\\|` is an escaped
+ backslash followed by a column boundary while `\\|` is a pipe inside the cell.
+ `str.split("|")` cannot tell the two apart -- it splits the escaped row this test is about,
+ which is the failure being reported -- so the scan is explicit here rather than in the product
+ code, where the writer has no reason to read its own output back.
+ """
+ cells, current, index = [], [], 0
+ while index < len(line):
+ char = line[index]
+ if char == "\\" and index + 1 < len(line):
+ current.append(line[index + 1])
+ index += 2
+ continue
+ if char == "|":
+ cells.append("".join(current))
+ current = []
+ index += 1
+ continue
+ current.append(char)
+ index += 1
+ cells.append("".join(current))
+ if line.startswith("|"):
+ cells = cells[1:] # the text before the first pipe is empty
+ if line.endswith("|"):
+ cells = cells[:-1] # ... and so is the text after the last
+ return [cell.strip() for cell in cells]
+
+
+def _markdown_slice_table(report, dimension):
+ """`{label: {metric: cell}}` for one `### by <dimension>` table, paired through its header."""
+ lines = report.to_markdown().splitlines()
+ body = lines[lines.index("### by %s" % dimension) + 1:]
+ header, rows = None, []
+ for line in body:
+ if line.startswith("###"):
+ break
+ if not line.startswith("|"):
+ continue
+ cells = _markdown_cells(line)
+ if all(cell and set(cell) <= set("-:") for cell in cells):
+ continue # the |---|---| separator row
+ if header is None:
+ header = cells
+ continue
+ assert len(cells) == len(header), \
+ "%r has %d cells for a %d-column header: %r" % (line, len(cells), len(header), cells)
+ rows.append(cells)
+ assert header and rows, "no %s table in the report" % dimension
+ return {cells[0]: dict(zip(header[1:], cells[1:])) for cells in rows}
+
+
+def _tagged_report(tags):
+ """A report whose `tag` slice has exactly these groups, two scored cases each.
+
+ Two cases per group land every metric in every cell (a single case leaves some of the
+ calibration columns empty), and each group gets its own confidence so the columns hold
+ different numbers -- a row shifted by one header then pairs a figure with the wrong name
+ instead of swapping two that happen to look alike.
+ """
+ examples, by_state = [], {}
+ for group, tag in enumerate(tags):
+ for case, expected in enumerate(("a", "b")):
+ state = "s%d%d" % (group, case)
+ # the group's first case answers right and its second wrong, so accuracy is 0.5
+ wrong = "b" if expected == "a" else "a"
+ examples.append(Example(state, Q, {"intent": expected}, tags=[tag]))
+ by_state[state] = {"intent": choice_answer(expected if case == 0 else wrong,
+ confidence=0.9 - 0.05 * group)}
+ report = evaluate(StubRunner(by_state), Dataset(examples),
+ evaluators=[ChoiceAccuracy(), MeanConfidence()])
+ assert sorted(report.slices["tag"]) == sorted(str(t) for t in tags)
+ return report
+
+
+def _assert_columns_pair(report, dimension, groups):
+ """Every figure sits under its own metric name, for every group in the table.
+
+ `groups` maps each rendered label to the key the slice is stored under; the two differ only for
+ a label whose line break was flattened into a space.
+ """
+ table = _markdown_slice_table(report, dimension)
+ assert sorted(table) == sorted(groups)
+ for label, data_label in groups.items():
+ row = table[label]
+ scores = report.slices[dimension][data_label]
+ assert sorted(row) == sorted(scores)
+ for metric, value in scores.items():
+ assert row[metric] == "%.4f" % value, "%s / %s" % (label, metric)
+
+
+def test_markdown_escapes_a_pipe_in_a_slice_label():
+ """A `|` in a group label is cell content, not a column boundary.
+
+ `tags` are any string the dataset author wrote -- `Example.from_dict` coerces them with `str`
+ rather than restricting them -- and this report is what a reader of `laya-evals run --markdown`
+ pairs numbers against. An unescaped pipe made that row one cell wider than its header, so every
+ figure after the label moved one header to the left and the last was dropped: `choice_accuracy`
+ showed the label's own second half, and the row that lost a value lost it silently.
+ """
+ report = _tagged_report(["risk|high", "plain"])
+ assert "| risk\\|high |" in report.to_markdown()
+ _assert_columns_pair(report, "tag", {"plain": "plain", "risk|high": "risk|high"})
+
+
+def test_markdown_keeps_a_label_with_a_line_break_on_one_row():
+ """A newline (or carriage return) in a label ends the row, taking its numbers out of the table.
+
+ `splitlines()` counts the physical rows the way a Markdown parser does: an unescaped break put
+ half a label on one line and the rest of the row, numbers included, on an unnumbered second
+ line that no header covers. Both breaks become spaces, which keeps the row intact and leaves
+ the label readable -- at the cell only, since the slice stays keyed by the label the dataset
+ wrote, so `--json` and `--markdown` still describe the same group.
+ """
+ report = _tagged_report(["two\nlines", "cr\rreturn", "plain"])
+ lines = report.to_markdown().splitlines()
+ table_rows = [line for line in lines[lines.index("### by tag"):] if line.startswith("|")]
+ assert len(table_rows) == 5, table_rows # header, separator, one row per group
+ _assert_columns_pair(report, "tag", {"two lines": "two\nlines",
+ "cr return": "cr\rreturn", "plain": "plain"})
+
+
+def test_markdown_escapes_a_backslash_before_the_pipe_escape():
+ """`\\` is doubled first, so a trailing backslash cannot swallow the escape that follows it.
+
+ A label ending in a backslash is ordinary data (`C:\\temp\\`), and writing only the pipe would
+ put that backslash in front of the escape for the *next* cell's boundary. The reader would then
+ take the boundary as a literal pipe and join the row -- so the doubling is what makes the pipe
+ escape mean one thing, and the round trip through `_markdown_cells` is the witness.
+ """
+ report = _tagged_report(["win\\path", "trailing\\", "back\\|pipe"])
+ markdown = report.to_markdown()
+ assert "| win\\\\path |" in markdown
+ assert "| trailing\\\\ |" in markdown
+ _assert_columns_pair(report, "tag", {"win\\path": "win\\path", "trailing\\": "trailing\\",
+ "back\\|pipe": "back\\|pipe"})
+
+
+def test_markdown_leaves_an_ordinary_label_untouched():
+ """The escape is invisible on the labels a report normally carries.
+
+ Nothing here rewrites a plain checkpoint name, language code or question id, so every existing
+ report -- and the doc-table parity checks that read the repository's own Markdown -- keeps the
+ bytes it had before.
+ """
+ report = _tagged_report(["english", "multilingual"])
+ markdown = report.to_markdown()
+ assert "| english |" in markdown and "| multilingual |" in markdown
+ assert "\\" not in markdown.split("### by tag")[1]
+ _assert_columns_pair(report, "tag", {"english": "english", "multilingual": "multilingual"})
+
+
def test_model_slice_follows_the_routed_checkpoint():
"""A Router records the checkpoint it chose under `routing`, not at the top level.