Skip to content

VectorCraft: An open-source, vector art tool written by scratch in pure Rust

refactor(engine): move cmd/variablecmds.rs items to cmd/variablecmds/{csv,library}.rs

aiongg4f7dd932681d
diff --git a/crates/engine/src/cmd/variablecmds.rs b/crates/engine/src/cmd/variablecmds.rs
index 0106e9895..be5fec64a 100644
--- a/crates/engine/src/cmd/variablecmds.rs
+++ b/crates/engine/src/cmd/variablecmds.rs
@@ -21,24 +21,14 @@ use super::fileio::{absolute_path, read_file_head, write_file};
use super::textedit::set_plain_text;
use super::*;
-/// The longest variable or dataset name, in characters (names arrive from MCP and files).
-const MAX_NAME: usize = 255;
-/// Cap on a graph variable's CSV (MCP and files are untrusted).
-const MAX_GRAPH_CSV: usize = 256 * 1024;
+mod csv;
+mod library;
-/// Caps on a datasets CSV (paths and MCP payloads are untrusted).
-const MAX_CSV_BYTES: usize = 1_048_576;
-const MAX_CSV_ROWS: usize = 1_000;
-const MAX_CSV_COLS: usize = 256;
+use csv::{MAX_GRAPH_CSV, dataset_export, dataset_import};
+use library::load_library;
-/// Caps on a variable library (paths and MCP payloads are untrusted): its size, its XML nodes, how
-/// deep its elements nest, and the entities, variables and data sets it declares.
-const MAX_LIBRARY_BYTES: usize = 8 << 20;
-const MAX_LIBRARY_ENTITIES: usize = 64;
-const MAX_LIBRARY_NODES: u32 = 2_000_000;
-const MAX_LIBRARY_NESTING: usize = 256;
-const MAX_LIBRARY_VARIABLES: usize = 256;
-const MAX_LIBRARY_DATASETS: usize = 1_000;
+/// The longest variable or dataset name, in characters (names arrive from MCP and files).
+const MAX_NAME: usize = 255;
/// Lock Variables refuses New Variable, Delete Variable, Rename Variable (Variable Options),
/// Make … Dynamic, an import whose header would define a variable and a Load Variable Library
@@ -545,346 +535,6 @@ fn dataset_select(s: &mut Session, p: &Value) -> Result<Value> {
apply_dataset(s, &var_name(p, "dataset.select")?)
}
-/// Bytes of a datasets CSV from `path`, `data`, `dataBase64` or `csv`.
-fn datasets_csv_bytes(p: &Value, cmd: &str) -> Result<Vec<u8>> {
- let bytes = match (str_param(p, "path"), str_param(p, "data"), str_param(p, "dataBase64"), str_param(p, "csv")) {
- // One byte past the cap is enough to refuse the file below.
- (Some(path), ..) => read_file_head(&absolute_path(path), MAX_CSV_BYTES + 1)?,
- (None, Some(text), ..) => text.as_bytes().to_vec(),
- (None, None, Some(b64), _) => vectorcraft_format::base64_decode(b64).ok_or_else(|| bad(cmd, "bad dataBase64"))?,
- (None, None, None, Some(csv)) => csv.as_bytes().to_vec(),
- _ => return Err(bad(cmd, "give `path`, `data`, `dataBase64` or `csv`")),
- };
- if bytes.len() > MAX_CSV_BYTES {
- return Err(bad(cmd, format!("CSV is larger than {MAX_CSV_BYTES} bytes")));
- }
- Ok(bytes)
-}
-
-/// A row of a datasets CSV: the file line it starts on, and its cells.
-type CsvRow = (usize, Vec<String>);
-
-/// Whether a datasets CSV separates its cells with tabs: its header row (the first line that
-/// isn't blank), read with tabs between cells, has a tab outside quotes and no comma outside
-/// quotes.
-fn tab_separated(text: &str) -> bool {
- // `start`: nothing but `\r` read yet in this cell. `blank`: no cell on this line holds
- // anything but white space yet.
- let (mut in_quotes, mut start, mut blank, mut tab) = (false, true, true, false);
- let mut chars = text.chars().peekable();
- while let Some(c) = chars.next() {
- match c {
- '"' if in_quotes => {
- if chars.peek() == Some(&'"') {
- chars.next();
- blank = false;
- } else {
- in_quotes = false;
- }
- }
- '"' if start => in_quotes = true,
- c if in_quotes => blank &= c.is_whitespace(),
- ',' => return false,
- '\t' => {
- tab = true;
- start = true;
- continue;
- }
- '\r' => continue,
- '\n' if blank => {
- tab = false;
- start = true;
- continue;
- }
- '\n' => return tab,
- c => blank &= c.is_whitespace(),
- }
- start = false;
- }
- tab
-}
-
-/// Parse a datasets CSV into rows of cells (no header interpretation yet). Cells are separated
-/// by commas, or by tabs when [`tab_separated`] says so. `"` quotes a cell (doubled inside),
-/// which may then hold separators and line breaks, as [`csv_escape`] writes them. An unquoted
-/// cell is trimmed. A quoted cell keeps what its quotes hold as written, spaces included, and
-/// drops white space after the closing quote at the end of the cell. Blank lines are skipped.
-fn parse_datasets_csv(text: &str, cmd: &str) -> Result<Vec<CsvRow>> {
- let text = text.strip_prefix('\u{feff}').unwrap_or(text);
- let sep = if tab_separated(text) { '\t' } else { ',' };
- let (mut rows, mut row, mut cur) = (Vec::new(), Vec::<String>::new(), String::new());
- // `closed`: where the closing quote of a quoted cell falls in `cur`. `start`: the line the
- // current row starts on.
- let (mut in_quotes, mut closed, mut line, mut start) = (false, None, 1usize, 1usize);
- let mut chars = text.chars().peekable();
- loop {
- let c = chars.next();
- match c {
- Some('"') if in_quotes => {
- if chars.peek() == Some(&'"') {
- cur.push('"');
- chars.next();
- } else {
- in_quotes = false;
- closed = Some(cur.len());
- }
- }
- Some('"') if cur.is_empty() => in_quotes = true,
- Some(ch) if ch == sep && !in_quotes => row.push(end_csv_cell(&mut cur, &mut closed)),
- Some('\r') if !in_quotes => {}
- Some('\n') | None if !in_quotes => {
- row.push(end_csv_cell(&mut cur, &mut closed));
- if row.iter().any(|c| !c.trim().is_empty()) {
- rows.push((start, std::mem::take(&mut row)));
- // +1 for the header.
- if rows.len() > MAX_CSV_ROWS + 1 {
- return Err(bad(cmd, format!("more than {MAX_CSV_ROWS} data rows")));
- }
- }
- row.clear();
- if c.is_none() {
- return Ok(rows);
- }
- line += 1;
- start = line;
- }
- None => return Err(bad(cmd, format!("line {line}: unclosed quote"))),
- Some(c) => {
- line += usize::from(c == '\n');
- cur.push(c);
- }
- }
- if row.len() > MAX_CSV_COLS {
- return Err(bad(cmd, format!("line {line}: more than {MAX_CSV_COLS} columns")));
- }
- }
-}
-
-/// Take the cell read so far out of `cur`. Without a closing quote (`closed` is `None`) the cell
-/// is trimmed; with one, the text before it is kept as written and the text after it is
-/// trimmed at the end.
-fn end_csv_cell(cur: &mut String, closed: &mut Option<usize>) -> String {
- let cell = std::mem::take(cur);
- match closed.take() {
- Some(at) => match cell.split_at_checked(at) {
- Some((quoted, after)) => format!("{quoted}{}", after.trim_end()),
- None => cell,
- },
- None => cell.trim().to_string(),
- }
-}
-
-fn csv_escape(s: &str) -> String {
- if s.contains([',', '"', '\n', '\r']) || s != s.trim() { format!("\"{}\"", s.replace('"', "\"\"")) } else { s.to_string() }
-}
-
-/// A visibility cell: true/false, 1/0, yes/no (ASCII, case-insensitive).
-fn parse_visible_cell(s: &str, var: &str, cmd: &str) -> Result<bool> {
- match s.trim().to_ascii_lowercase().as_str() {
- "true" | "1" | "yes" => Ok(true),
- "false" | "0" | "no" => Ok(false),
- _ => Err(bad(cmd, format!("value for `{var}` must be true or false"))),
- }
-}
-
-fn cell_to_value(kind: VariableKind, cell: &str, var: &str, cmd: &str) -> Result<Option<DataValue>> {
- // A text cell is not trimmed here (the parser trims unquoted cells); image, graph and
- // visibility cells are trimmed.
- let t = if kind == VariableKind::Text { cell } else { cell.trim() };
- if t.is_empty() {
- return Ok(None);
- }
- Ok(Some(match kind {
- VariableKind::Text => DataValue::Text(t.to_string()),
- VariableKind::Image => DataValue::Image(t.to_string()),
- VariableKind::Graph if t.len() <= MAX_GRAPH_CSV => DataValue::Graph(t.to_string()),
- VariableKind::Graph => return Err(bad(cmd, format!("value for `{var}` is longer than {MAX_GRAPH_CSV} bytes"))),
- VariableKind::Visibility => DataValue::Visible(parse_visible_cell(t, var, cmd)?),
- }))
-}
-
-fn value_to_cell(v: &DataValue) -> String {
- match v {
- DataValue::Text(t) | DataValue::Image(t) | DataValue::Graph(t) => t.clone(),
- DataValue::Visible(b) => if *b { "true" } else { "false" }.to_string(),
- }
-}
-
-/// Build the CSV of every dataset (header `name` then variables in definition order).
-fn datasets_to_csv(vars: &Variables) -> String {
- let mut out = String::from("name");
- for v in &vars.variables {
- out.push(',');
- out.push_str(&csv_escape(&v.name));
- }
- out.push('\n');
- for ds in &vars.datasets {
- out.push_str(&csv_escape(&ds.name));
- for v in &vars.variables {
- out.push(',');
- if let Some(val) = ds.values.get(&v.name) {
- out.push_str(&csv_escape(&value_to_cell(val)));
- }
- }
- out.push('\n');
- }
- out
-}
-
-/// The variable that a datasets CSV header cell (trimmed, not empty) in 1-based `column` names,
-/// and its kind. A cell that is the name of a variable in `kinds` or in `defined` (the variables
-/// that earlier columns define) names that variable. Otherwise `#` before the name makes it a
-/// visibility variable, `@` makes it an image variable, and a cell without a prefix names a text
-/// variable. A prefixed name that is already a variable must be one of the prefix's kind; a new
-/// name is added to `defined`. A `%` cell that names no variable (graph data files) is refused.
-fn header_variable(
- cell: &str,
- column: usize,
- kinds: &BTreeMap<String, VariableKind>,
- defined: &mut Vec<Variable>,
- cmd: &str,
-) -> Result<(String, VariableKind)> {
- let known = |name: &str, defined: &[Variable]| kinds.get(name).copied().or_else(|| defined.iter().find(|v| v.name == name).map(|v| v.kind));
- if let Some(kind) = known(cell, defined) {
- return Ok((cell.to_string(), kind));
- }
- let (name, kind) = if let Some(rest) = cell.strip_prefix('#') {
- (rest.trim(), VariableKind::Visibility)
- } else if let Some(rest) = cell.strip_prefix('@') {
- (rest.trim(), VariableKind::Image)
- } else if cell.starts_with('%') {
- return Err(bad(cmd, format!("column {column}: `{cell}` is a `%` column of graph data files, which are not read")));
- } else {
- (cell, VariableKind::Text)
- };
- if name.is_empty() {
- return Err(bad(cmd, format!("column {column}: `{cell}` names no variable")));
- }
- match known(name, defined) {
- Some(k) if k == kind => Ok((name.to_string(), kind)),
- Some(k) => Err(bad(cmd, format!("column {column}: `{cell}` names `{name}`, which is a {} variable", k.label()))),
- None if name.chars().count() > MAX_NAME => Err(bad(cmd, format!("column {column}: the variable name is longer than {MAX_NAME} characters"))),
- None => {
- defined.push(Variable { name: name.to_string(), kind });
- Ok((name.to_string(), kind))
- }
- }
-}
-
-/// Import datasets from CSV. A first header cell `name` makes the first column the dataset
-/// names; the other headers name variables, as [`header_variable`] reads them, and the
-/// variables the document doesn't define yet are defined, unbound. Without the `name` column,
-/// every header names a variable and the rows take the first free names "Data Set 1", "Data
-/// Set 2" and so on. Existing rows of the same name are replaced; new names are appended. One
-/// undo step.
-fn dataset_import(s: &mut Session, p: &Value) -> Result<Value> {
- const C: &str = "dataset.import";
- let bytes = datasets_csv_bytes(p, C)?;
- let text = std::str::from_utf8(&bytes).map_err(|_| bad(C, "CSV is not UTF-8"))?;
- let rows = parse_datasets_csv(text, C)?;
- let Some((_, header)) = rows.first() else {
- return Err(bad(C, "CSV is empty"));
- };
- let name_column = header.first().is_some_and(|h| h.trim().eq_ignore_ascii_case("name"));
- let (kinds, existing_names) = {
- let st = s.doc()?;
- let kinds: BTreeMap<String, VariableKind> = st.doc.variables.variables.iter().map(|v| (v.name.clone(), v.kind)).collect();
- let existing: BTreeSet<String> = st.doc.variables.datasets.iter().map(|d| d.name.clone()).collect();
- (kinds, existing)
- };
- let mut columns: Vec<(usize, String, VariableKind)> = Vec::new();
- let mut defined: Vec<Variable> = Vec::new();
- for (i, h) in header.iter().enumerate().skip(usize::from(name_column)) {
- let cell = h.trim();
- if cell.is_empty() {
- return Err(bad(C, format!("empty header in column {}", i + 1)));
- }
- let (name, kind) = header_variable(cell, i + 1, &kinds, &mut defined, C)?;
- columns.push((i, name, kind));
- }
- // Lock Variables refuses New Variable, so the import is refused when a header would define one.
- if !defined.is_empty() {
- variables_unlocked(s).map_err(|why| EngineError::Disabled(C.into(), why))?;
- }
- if rows.len() == 1 {
- return Err(bad(C, "CSV has a header but no data rows"));
- }
- let mut parsed: Vec<DataSet> = Vec::new();
- // The names of the rows read so far, and the last number given to a row without a name.
- let (mut seen, mut number) = (BTreeSet::new(), 0usize);
- for &(line, ref row) in rows.iter().skip(1) {
- let raw_name = if name_column { row.first().map_or("", |c| c.trim()) } else { "" };
- let name = if raw_name.is_empty() {
- // Same numbering as Capture Data Set when a row leaves the name blank or the file has
- // no name column. The search starts after the last number given.
- let (n, name) = (number + 1..)
- .map(|n| (n, format!("Data Set {n}")))
- .find(|(_, n)| !seen.contains(n) && !existing_names.contains(n))
- .unwrap_or_else(|| (line, format!("Data Set {line}")));
- number = n;
- name
- } else if raw_name.chars().count() > MAX_NAME {
- return Err(bad(C, format!("line {line}: `name` is longer than {MAX_NAME} characters")));
- } else {
- raw_name.to_string()
- };
- if !seen.insert(name.clone()) {
- return Err(bad(C, format!("line {line}: dataset `{name}` appears twice")));
- }
- let mut values = BTreeMap::new();
- for &(col, ref var, kind) in &columns {
- let cell = row.get(col).map(String::as_str).unwrap_or("");
- match cell_to_value(kind, cell, var, C) {
- Ok(Some(v)) => {
- values.insert(var.clone(), v);
- }
- Ok(None) => {}
- Err(_) => return Err(bad(C, format!("line {line}: value for `{var}` must match its variable kind"))),
- }
- }
- parsed.push(DataSet { name, values });
- }
- let names: Vec<String> = parsed.iter().map(|d| d.name.clone()).collect();
- let new_variables: Vec<String> = defined.iter().map(|v| v.name.clone()).collect();
- let (imported, updated) = s.edit("Import Data Sets", |d, _| {
- d.variables.variables.extend(defined);
- let mut imported = 0usize;
- let mut updated = 0usize;
- for row in parsed {
- if let Some(existing) = d.variables.datasets.iter_mut().find(|x| x.name == row.name) {
- *existing = row;
- updated += 1;
- } else {
- d.variables.datasets.push(row);
- imported += 1;
- }
- }
- Ok((imported, updated))
- })?;
- Ok(json!({"imported": imported, "updated": updated, "names": names, "defined": new_variables}))
-}
-
-/// Export every dataset as CSV (`name` then variables). With `path`, write the file; else return `{csv}`.
-fn dataset_export(s: &mut Session, p: &Value) -> Result<Value> {
- const C: &str = "dataset.export";
- let st = s.doc()?;
- if st.doc.variables.variables.is_empty() {
- return Err(bad(C, "define a variable first"));
- }
- let csv = datasets_to_csv(&st.doc.variables);
- if csv.len() > MAX_CSV_BYTES {
- return Err(bad(C, format!("CSV would be larger than {MAX_CSV_BYTES} bytes")));
- }
- match str_param(p, "path") {
- Some(path) => {
- let path = absolute_path(path);
- write_file(&path, csv.as_bytes())?;
- Ok(json!({"path": path, "datasets": st.doc.variables.datasets.len()}))
- }
- None => Ok(json!({"csv": csv, "datasets": st.doc.variables.datasets.len()})),
- }
-}
-
fn dataset_step(s: &mut Session, cmd: &str, forward: bool) -> Result<Value> {
let st = s.doc()?;
let n = st.doc.variables.datasets.len();
@@ -902,289 +552,6 @@ fn dataset_step(s: &mut Session, cmd: &str, forward: bool) -> Result<Value> {
apply_dataset(s, &name)
}
-/// Bytes of a variable library from `path`, `xml` or `dataBase64`.
-fn library_bytes(p: &Value, cmd: &str) -> Result<Vec<u8>> {
- let bytes = match (str_param(p, "path"), str_param(p, "xml"), str_param(p, "dataBase64")) {
- // One byte past the cap is enough to refuse the file below.
- (Some(path), ..) => read_file_head(&absolute_path(path), MAX_LIBRARY_BYTES + 1)?,
- (None, Some(xml), _) => xml.as_bytes().to_vec(),
- (None, None, Some(b64)) => vectorcraft_format::base64_decode(b64).ok_or_else(|| bad(cmd, "bad dataBase64"))?,
- _ => return Err(bad(cmd, "give `path`, `xml` or `dataBase64`")),
- };
- if bytes.len() > MAX_LIBRARY_BYTES {
- return Err(bad(cmd, format!("the variable library is larger than {MAX_LIBRARY_BYTES} bytes")));
- }
- Ok(bytes)
-}
-
-/// `xml` without the document type declaration of its prolog, with each reference to a general
-/// entity that the declaration defines replaced by the entity's value, its quotes and `>` escaped,
-/// outside CDATA sections, comments and processing instructions. The declaration may hold an
-/// external identifier and an internal subset of white space, comments, processing instructions and
-/// up to [`MAX_LIBRARY_ENTITIES`] entity declarations, each with a name of at most [`MAX_NAME`]
-/// bytes and a quoted value that holds no `<`, `&` or `%`. Anything else in it is refused, and so is
-/// a text that the replacements make longer than [`MAX_LIBRARY_BYTES`]. The XML parser then refuses
-/// any other document type declaration, so [`nesting`](super::place::docx::nesting) counts the
-/// elements the parser reads.
-fn without_doctype(xml: &str, cmd: &str) -> Result<String> {
- let b = xml.as_bytes();
- let past = |from: usize, end: &str| xml.get(from..).and_then(|rest| rest.find(end)).map(|at| from + at + end.len());
- let spaces = |from: usize| from + b.get(from..).map_or(0, |rest| rest.iter().take_while(|c| c.is_ascii_whitespace()).count());
- let at = |i: usize, s: &str| xml.get(i..).is_some_and(|rest| rest.starts_with(s));
- let refused = || bad(cmd, "the variable library's document type declaration holds something other than entities of plain text");
- // The prolog: an XML declaration, white space, comments and processing instructions.
- let mut i = 0;
- loop {
- i = spaces(i);
- if at(i, "<?") {
- i = past(i + 2, "?>").ok_or_else(refused)?;
- } else if at(i, "<!--") {
- i = past(i + 4, "-->").ok_or_else(refused)?;
- } else {
- break;
- }
- }
- if !at(i, "<!DOCTYPE") {
- return Ok(xml.to_string());
- }
- let start = i;
- // A quoted literal starting at `i` → the index past it and its text.
- let literal = |i: usize| -> Option<(usize, &str)> {
- let quote = *b.get(i).filter(|c| matches!(c, b'"' | b'\''))?;
- let len = b.get(i + 1..)?.iter().position(|c| *c == quote)?;
- Some((i + len + 2, xml.get(i + 1..i + 1 + len)?))
- };
- // `<!DOCTYPE` S Name (S ExternalID)? S? ('[' intSubset ']' S?)? '>'
- i = spaces(i + "<!DOCTYPE".len());
- i += b.get(i..).map_or(0, |rest| rest.iter().take_while(|c| !c.is_ascii_whitespace() && !matches!(c, b'[' | b'>')).count());
- i = spaces(i);
- for (id, literals) in [("SYSTEM", 1), ("PUBLIC", 2)] {
- if at(i, id) {
- i += id.len();
- for _ in 0..literals {
- i = literal(spaces(i)).ok_or_else(refused)?.0;
- }
- i = spaces(i);
- }
- }
- let mut entities: Vec<(&str, String)> = Vec::new();
- if at(i, "[") {
- i += 1;
- loop {
- i = spaces(i);
- if at(i, "]") {
- i = spaces(i + 1);
- break;
- } else if at(i, "<!--") {
- i = past(i + 4, "-->").ok_or_else(refused)?;
- } else if at(i, "<?") {
- i = past(i + 2, "?>").ok_or_else(refused)?;
- } else if at(i, "<!ENTITY") {
- i = spaces(i + "<!ENTITY".len());
- let len =
- b.get(i..).map_or(0, |rest| rest.iter().take_while(|c| !c.is_ascii_whitespace() && !matches!(c, b'"' | b'\'' | b'>')).count());
- let name = xml.get(i..i + len).filter(|n| !n.is_empty() && n.len() <= MAX_NAME && !n.starts_with('%')).ok_or_else(refused)?;
- let (end, value) = literal(spaces(i + len)).ok_or_else(refused)?;
- if value.contains(['<', '&', '%']) {
- return Err(refused());
- }
- i = spaces(end);
- if !at(i, ">") {
- return Err(refused());
- }
- i += 1;
- if entities.len() == MAX_LIBRARY_ENTITIES {
- return Err(bad(cmd, format!("the variable library declares more than {MAX_LIBRARY_ENTITIES} entities")));
- }
- entities.push((name, value.replace('"', "&quot;").replace('\'', "&apos;").replace('>', "&gt;")));
- } else {
- return Err(refused());
- }
- }
- }
- if !at(i, ">") {
- return Err(refused());
- }
- let mut out = String::with_capacity(xml.len());
- out.push_str(xml.get(..start).unwrap_or_default());
- // The rest, copied with the references replaced outside CDATA sections, comments and
- // processing instructions, where the parser reads none.
- let mut i = i + 1;
- while let Some(c) = xml.get(i..).and_then(|rest| rest.chars().next()) {
- let verbatim = [("<![CDATA[", "]]>"), ("<!--", "-->"), ("<?", "?>")].into_iter().find(|(open, _)| at(i, open));
- if let Some((open, close)) = verbatim {
- let end = past(i + open.len(), close).unwrap_or(xml.len());
- out.push_str(xml.get(i..end).unwrap_or_default());
- i = end;
- continue;
- }
- // Entity names are at most `MAX_NAME` bytes long, so the `;` is looked for that far.
- let entity = if c == '&' {
- let name = xml.get(i + 1..).and_then(|rest| rest.get(..rest.bytes().take(MAX_NAME + 1).position(|b| b == b';')?));
- name.and_then(|name| entities.iter().find(|(n, _)| *n == name))
- } else {
- None
- };
- match entity {
- Some((name, value)) => {
- out.push_str(value);
- i += name.len() + 2;
- if out.len() > MAX_LIBRARY_BYTES {
- return Err(bad(cmd, format!("the variable library's entities expand to more than {MAX_LIBRARY_BYTES} bytes")));
- }
- }
- None => {
- out.push(c);
- i += c.len_utf8();
- }
- }
- }
- if out.len() > MAX_LIBRARY_BYTES {
- return Err(bad(cmd, format!("the variable library's entities expand to more than {MAX_LIBRARY_BYTES} bytes")));
- }
- Ok(out)
-}
-
-/// A variable library as Load Variable Library reads it: the names of its text variables in the
-/// order it declares them, the name and `trait` of each variable of another kind, and its data
-/// sets with their text values.
-struct Library {
- text: Vec<String>,
- other: Vec<(String, String)>,
- datasets: Vec<DataSet>,
-}
-
-/// Read a variable library: XML whose `variableSets` element holds `variableSet` elements, each
-/// with a `variables` element of `variable` elements (`varName`, `trait`) and a `sampleDataSets`
-/// element of `sampleDataSet` elements (`dataSetName`). Elements are matched by their local names.
-/// A variable whose `trait` is `textcontent` is a text variable. In a data set, an element named
-/// after a text variable gives its value: the text of its `p` elements, one paragraph each, and
-/// none when it has no `p` element. Values for variables of other kinds are left out. A variable
-/// or data set declared twice, a value for a variable the library doesn't declare, and a library
-/// past the caps are refused. Entities that its document type declaration defines are read as
-/// [`without_doctype`] says.
-fn parse_library(xml: &str, cmd: &str) -> Result<Library> {
- let xml = without_doctype(xml, cmd)?;
- // roxmltree parses nested elements recursively.
- if super::place::docx::nesting(&xml) > MAX_LIBRARY_NESTING {
- return Err(bad(cmd, format!("the variable library nests elements more than {MAX_LIBRARY_NESTING} deep")));
- }
- let opts = roxmltree::ParsingOptions { nodes_limit: MAX_LIBRARY_NODES, ..roxmltree::ParsingOptions::default() };
- let doc =
- roxmltree::Document::parse_with_options(&xml, opts).map_err(|e| bad(cmd, format!("the variable library is not well-formed XML: {e}")))?;
- let is = |n: &roxmltree::Node, name: &str| n.is_element() && n.tag_name().name() == name;
- let Some(root) = doc.descendants().find(|n| is(n, "variableSets")) else {
- return Err(bad(cmd, "the file holds no `variableSets` element"));
- };
- let sets: Vec<roxmltree::Node> = root.children().filter(|n| is(n, "variableSet")).collect();
- let (mut text, mut other, mut declared) = (Vec::new(), Vec::new(), BTreeSet::new());
- for var in sets.iter().flat_map(|s| s.children()).filter(|n| is(n, "variables")).flat_map(|v| v.children()).filter(|n| is(n, "variable")) {
- let name = named(var.attribute("varName"), "varName", cmd)?;
- if !declared.insert(name.clone()) {
- return Err(bad(cmd, format!("variable `{name}` is declared twice")));
- }
- if declared.len() > MAX_LIBRARY_VARIABLES {
- return Err(bad(cmd, format!("the variable library declares more than {MAX_LIBRARY_VARIABLES} variables")));
- }
- match var.attribute("trait") {
- Some("textcontent") => text.push(name),
- Some(t) => other.push((name, t.chars().take(MAX_NAME).collect())),
- None => return Err(bad(cmd, format!("variable `{name}` has no `trait`"))),
- }
- }
- if declared.is_empty() {
- return Err(bad(cmd, "the variable library declares no variables"));
- }
- let text_names: BTreeSet<&str> = text.iter().map(String::as_str).collect();
- let (mut datasets, mut seen) = (Vec::new(), BTreeSet::new());
- for row in
- sets.iter().flat_map(|s| s.children()).filter(|n| is(n, "sampleDataSets")).flat_map(|d| d.children()).filter(|n| is(n, "sampleDataSet"))
- {
- let name = named(row.attribute("dataSetName"), "dataSetName", cmd)?;
- if !seen.insert(name.clone()) {
- return Err(bad(cmd, format!("data set `{name}` is declared twice")));
- }
- if seen.len() > MAX_LIBRARY_DATASETS {
- return Err(bad(cmd, format!("the variable library declares more than {MAX_LIBRARY_DATASETS} data sets")));
- }
- let (mut values, mut held) = (BTreeMap::new(), BTreeSet::new());
- for el in row.children().filter(roxmltree::Node::is_element) {
- let var = el.tag_name().name();
- if !declared.contains(var) {
- return Err(bad(cmd, format!("data set `{name}` holds a value for `{var}`, which the library doesn't declare")));
- }
- if !held.insert(var) {
- return Err(bad(cmd, format!("data set `{name}` holds two values for `{var}`")));
- }
- if !text_names.contains(var) {
- continue;
- }
- let paras: Vec<String> = el
- .children()
- .filter(|n| is(n, "p"))
- .map(|p| p.descendants().filter(roxmltree::Node::is_text).filter_map(|t| t.text()).collect())
- .collect();
- if !paras.is_empty() {
- values.insert(var.to_string(), DataValue::Text(paras.join("\n")));
- }
- }
- datasets.push(DataSet { name, values });
- }
- Ok(Library { text, other, datasets })
-}
-
-/// Load Variable Library: replace the document's variables and data sets with a variable library's
-/// ([`parse_library`]) in one undo step. Every variable of the document is deleted with its bindings
-/// and values, and every data set is deleted. The library's text variables are then defined,
-/// unbound, and its data sets are added. The library's variables of other kinds are left out, named
-/// in `skipped` and in a warning.
-fn load_library(s: &mut Session, p: &Value) -> Result<Value> {
- const C: &str = "variable.loadLibrary";
- let bytes = library_bytes(p, C)?;
- let xml = std::str::from_utf8(bytes.strip_prefix(b"\xEF\xBB\xBF").unwrap_or(&bytes)).map_err(|_| bad(C, "the variable library is not UTF-8"))?;
- let lib = parse_library(xml, C)?;
- let vars = &s.doc()?.doc.variables;
- let deleted: Vec<String> = vars.variables.iter().map(|v| v.name.clone()).collect();
- let deleted_sets: Vec<String> = vars.datasets.iter().map(|d| d.name.clone()).collect();
- // Lock Variables refuses New Variable and Delete Variable, so the load is refused when it would
- // delete or define a variable.
- if !deleted.is_empty() || !lib.text.is_empty() {
- variables_unlocked(s).map_err(|why| EngineError::Disabled(C.into(), why))?;
- }
- let added: Vec<Variable> = lib.text.iter().map(|n| Variable { name: n.clone(), kind: VariableKind::Text }).collect();
- let names: Vec<String> = lib.datasets.iter().map(|d| d.name.clone()).collect();
- s.edit("Load Variable Library", |d, _| {
- d.variables.prune(&deleted);
- d.variables.variables.extend(added);
- d.variables.datasets = lib.datasets;
- d.variables.active_dataset = None;
- Ok(())
- })?;
- // A highlight of a variable that is gone highlights nothing.
- let st = s.doc_mut()?;
- if st.variables_highlight.as_ref().is_some_and(|h| st.doc.variables.variable(h).is_none()) {
- st.variables_highlight = None;
- }
- st.revision += 1;
- let skipped: Vec<Value> = lib.other.iter().map(|(name, t)| json!({"name": name, "trait": t})).collect();
- let mut out = json!({
- "variables": lib.text, "deleted": deleted, "deletedDataSets": deleted_sets, "names": names, "skipped": skipped,
- });
- // Each `trait` once, in the order the library first uses it, with how many variables have it.
- let mut kinds: Vec<(&str, usize)> = Vec::new();
- for (_, t) in &lib.other {
- match kinds.iter_mut().find(|(k, _)| k == t) {
- Some((_, n)) => *n += 1,
- None => kinds.push((t, 1)),
- }
- }
- if !kinds.is_empty() {
- let kinds = kinds.iter().map(|(k, n)| format!("{k} ({n})")).collect::<Vec<_>>().join(", ");
- out["warning"] = json!(format!("variables that aren't text were left out: {kinds}"));
- }
- Ok(out)
-}
-
pub fn specs() -> Vec<CommandSpec> {
vec![
cmd!(
diff --git a/crates/engine/src/cmd/variablecmds/csv.rs b/crates/engine/src/cmd/variablecmds/csv.rs
new file mode 100644
index 000000000..6cb10284b
--- /dev/null
+++ b/crates/engine/src/cmd/variablecmds/csv.rs
@@ -0,0 +1,351 @@
+use super::*;
+
+/// Cap on a graph variable's CSV (MCP and files are untrusted).
+pub(super) const MAX_GRAPH_CSV: usize = 256 * 1024;
+
+/// Caps on a datasets CSV (paths and MCP payloads are untrusted).
+const MAX_CSV_BYTES: usize = 1_048_576;
+
+const MAX_CSV_ROWS: usize = 1_000;
+
+const MAX_CSV_COLS: usize = 256;
+
+/// Bytes of a datasets CSV from `path`, `data`, `dataBase64` or `csv`.
+fn datasets_csv_bytes(p: &Value, cmd: &str) -> Result<Vec<u8>> {
+ let bytes = match (str_param(p, "path"), str_param(p, "data"), str_param(p, "dataBase64"), str_param(p, "csv")) {
+ // One byte past the cap is enough to refuse the file below.
+ (Some(path), ..) => read_file_head(&absolute_path(path), MAX_CSV_BYTES + 1)?,
+ (None, Some(text), ..) => text.as_bytes().to_vec(),
+ (None, None, Some(b64), _) => vectorcraft_format::base64_decode(b64).ok_or_else(|| bad(cmd, "bad dataBase64"))?,
+ (None, None, None, Some(csv)) => csv.as_bytes().to_vec(),
+ _ => return Err(bad(cmd, "give `path`, `data`, `dataBase64` or `csv`")),
+ };
+ if bytes.len() > MAX_CSV_BYTES {
+ return Err(bad(cmd, format!("CSV is larger than {MAX_CSV_BYTES} bytes")));
+ }
+ Ok(bytes)
+}
+
+/// A row of a datasets CSV: the file line it starts on, and its cells.
+type CsvRow = (usize, Vec<String>);
+
+/// Whether a datasets CSV separates its cells with tabs: its header row (the first line that
+/// isn't blank), read with tabs between cells, has a tab outside quotes and no comma outside
+/// quotes.
+fn tab_separated(text: &str) -> bool {
+ // `start`: nothing but `\r` read yet in this cell. `blank`: no cell on this line holds
+ // anything but white space yet.
+ let (mut in_quotes, mut start, mut blank, mut tab) = (false, true, true, false);
+ let mut chars = text.chars().peekable();
+ while let Some(c) = chars.next() {
+ match c {
+ '"' if in_quotes => {
+ if chars.peek() == Some(&'"') {
+ chars.next();
+ blank = false;
+ } else {
+ in_quotes = false;
+ }
+ }
+ '"' if start => in_quotes = true,
+ c if in_quotes => blank &= c.is_whitespace(),
+ ',' => return false,
+ '\t' => {
+ tab = true;
+ start = true;
+ continue;
+ }
+ '\r' => continue,
+ '\n' if blank => {
+ tab = false;
+ start = true;
+ continue;
+ }
+ '\n' => return tab,
+ c => blank &= c.is_whitespace(),
+ }
+ start = false;
+ }
+ tab
+}
+
+/// Parse a datasets CSV into rows of cells (no header interpretation yet). Cells are separated
+/// by commas, or by tabs when [`tab_separated`] says so. `"` quotes a cell (doubled inside),
+/// which may then hold separators and line breaks, as [`csv_escape`] writes them. An unquoted
+/// cell is trimmed. A quoted cell keeps what its quotes hold as written, spaces included, and
+/// drops white space after the closing quote at the end of the cell. Blank lines are skipped.
+fn parse_datasets_csv(text: &str, cmd: &str) -> Result<Vec<CsvRow>> {
+ let text = text.strip_prefix('\u{feff}').unwrap_or(text);
+ let sep = if tab_separated(text) { '\t' } else { ',' };
+ let (mut rows, mut row, mut cur) = (Vec::new(), Vec::<String>::new(), String::new());
+ // `closed`: where the closing quote of a quoted cell falls in `cur`. `start`: the line the
+ // current row starts on.
+ let (mut in_quotes, mut closed, mut line, mut start) = (false, None, 1usize, 1usize);
+ let mut chars = text.chars().peekable();
+ loop {
+ let c = chars.next();
+ match c {
+ Some('"') if in_quotes => {
+ if chars.peek() == Some(&'"') {
+ cur.push('"');
+ chars.next();
+ } else {
+ in_quotes = false;
+ closed = Some(cur.len());
+ }
+ }
+ Some('"') if cur.is_empty() => in_quotes = true,
+ Some(ch) if ch == sep && !in_quotes => row.push(end_csv_cell(&mut cur, &mut closed)),
+ Some('\r') if !in_quotes => {}
+ Some('\n') | None if !in_quotes => {
+ row.push(end_csv_cell(&mut cur, &mut closed));
+ if row.iter().any(|c| !c.trim().is_empty()) {
+ rows.push((start, std::mem::take(&mut row)));
+ // +1 for the header.
+ if rows.len() > MAX_CSV_ROWS + 1 {
+ return Err(bad(cmd, format!("more than {MAX_CSV_ROWS} data rows")));
+ }
+ }
+ row.clear();
+ if c.is_none() {
+ return Ok(rows);
+ }
+ line += 1;
+ start = line;
+ }
+ None => return Err(bad(cmd, format!("line {line}: unclosed quote"))),
+ Some(c) => {
+ line += usize::from(c == '\n');
+ cur.push(c);
+ }
+ }
+ if row.len() > MAX_CSV_COLS {
+ return Err(bad(cmd, format!("line {line}: more than {MAX_CSV_COLS} columns")));
+ }
+ }
+}
+
+/// Take the cell read so far out of `cur`. Without a closing quote (`closed` is `None`) the cell
+/// is trimmed; with one, the text before it is kept as written and the text after it is
+/// trimmed at the end.
+fn end_csv_cell(cur: &mut String, closed: &mut Option<usize>) -> String {
+ let cell = std::mem::take(cur);
+ match closed.take() {
+ Some(at) => match cell.split_at_checked(at) {
+ Some((quoted, after)) => format!("{quoted}{}", after.trim_end()),
+ None => cell,
+ },
+ None => cell.trim().to_string(),
+ }
+}
+
+fn csv_escape(s: &str) -> String {
+ if s.contains([',', '"', '\n', '\r']) || s != s.trim() { format!("\"{}\"", s.replace('"', "\"\"")) } else { s.to_string() }
+}
+
+/// A visibility cell: true/false, 1/0, yes/no (ASCII, case-insensitive).
+fn parse_visible_cell(s: &str, var: &str, cmd: &str) -> Result<bool> {
+ match s.trim().to_ascii_lowercase().as_str() {
+ "true" | "1" | "yes" => Ok(true),
+ "false" | "0" | "no" => Ok(false),
+ _ => Err(bad(cmd, format!("value for `{var}` must be true or false"))),
+ }
+}
+
+fn cell_to_value(kind: VariableKind, cell: &str, var: &str, cmd: &str) -> Result<Option<DataValue>> {
+ // A text cell is not trimmed here (the parser trims unquoted cells); image, graph and
+ // visibility cells are trimmed.
+ let t = if kind == VariableKind::Text { cell } else { cell.trim() };
+ if t.is_empty() {
+ return Ok(None);
+ }
+ Ok(Some(match kind {
+ VariableKind::Text => DataValue::Text(t.to_string()),
+ VariableKind::Image => DataValue::Image(t.to_string()),
+ VariableKind::Graph if t.len() <= MAX_GRAPH_CSV => DataValue::Graph(t.to_string()),
+ VariableKind::Graph => return Err(bad(cmd, format!("value for `{var}` is longer than {MAX_GRAPH_CSV} bytes"))),
+ VariableKind::Visibility => DataValue::Visible(parse_visible_cell(t, var, cmd)?),
+ }))
+}
+
+fn value_to_cell(v: &DataValue) -> String {
+ match v {
+ DataValue::Text(t) | DataValue::Image(t) | DataValue::Graph(t) => t.clone(),
+ DataValue::Visible(b) => if *b { "true" } else { "false" }.to_string(),
+ }
+}
+
+/// Build the CSV of every dataset (header `name` then variables in definition order).
+fn datasets_to_csv(vars: &Variables) -> String {
+ let mut out = String::from("name");
+ for v in &vars.variables {
+ out.push(',');
+ out.push_str(&csv_escape(&v.name));
+ }
+ out.push('\n');
+ for ds in &vars.datasets {
+ out.push_str(&csv_escape(&ds.name));
+ for v in &vars.variables {
+ out.push(',');
+ if let Some(val) = ds.values.get(&v.name) {
+ out.push_str(&csv_escape(&value_to_cell(val)));
+ }
+ }
+ out.push('\n');
+ }
+ out
+}
+
+/// The variable that a datasets CSV header cell (trimmed, not empty) in 1-based `column` names,
+/// and its kind. A cell that is the name of a variable in `kinds` or in `defined` (the variables
+/// that earlier columns define) names that variable. Otherwise `#` before the name makes it a
+/// visibility variable, `@` makes it an image variable, and a cell without a prefix names a text
+/// variable. A prefixed name that is already a variable must be one of the prefix's kind; a new
+/// name is added to `defined`. A `%` cell that names no variable (graph data files) is refused.
+fn header_variable(
+ cell: &str,
+ column: usize,
+ kinds: &BTreeMap<String, VariableKind>,
+ defined: &mut Vec<Variable>,
+ cmd: &str,
+) -> Result<(String, VariableKind)> {
+ let known = |name: &str, defined: &[Variable]| kinds.get(name).copied().or_else(|| defined.iter().find(|v| v.name == name).map(|v| v.kind));
+ if let Some(kind) = known(cell, defined) {
+ return Ok((cell.to_string(), kind));
+ }
+ let (name, kind) = if let Some(rest) = cell.strip_prefix('#') {
+ (rest.trim(), VariableKind::Visibility)
+ } else if let Some(rest) = cell.strip_prefix('@') {
+ (rest.trim(), VariableKind::Image)
+ } else if cell.starts_with('%') {
+ return Err(bad(cmd, format!("column {column}: `{cell}` is a `%` column of graph data files, which are not read")));
+ } else {
+ (cell, VariableKind::Text)
+ };
+ if name.is_empty() {
+ return Err(bad(cmd, format!("column {column}: `{cell}` names no variable")));
+ }
+ match known(name, defined) {
+ Some(k) if k == kind => Ok((name.to_string(), kind)),
+ Some(k) => Err(bad(cmd, format!("column {column}: `{cell}` names `{name}`, which is a {} variable", k.label()))),
+ None if name.chars().count() > MAX_NAME => Err(bad(cmd, format!("column {column}: the variable name is longer than {MAX_NAME} characters"))),
+ None => {
+ defined.push(Variable { name: name.to_string(), kind });
+ Ok((name.to_string(), kind))
+ }
+ }
+}
+
+/// Import datasets from CSV. A first header cell `name` makes the first column the dataset
+/// names; the other headers name variables, as [`header_variable`] reads them, and the
+/// variables the document doesn't define yet are defined, unbound. Without the `name` column,
+/// every header names a variable and the rows take the first free names "Data Set 1", "Data
+/// Set 2" and so on. Existing rows of the same name are replaced; new names are appended. One
+/// undo step.
+pub(super) fn dataset_import(s: &mut Session, p: &Value) -> Result<Value> {
+ const C: &str = "dataset.import";
+ let bytes = datasets_csv_bytes(p, C)?;
+ let text = std::str::from_utf8(&bytes).map_err(|_| bad(C, "CSV is not UTF-8"))?;
+ let rows = parse_datasets_csv(text, C)?;
+ let Some((_, header)) = rows.first() else {
+ return Err(bad(C, "CSV is empty"));
+ };
+ let name_column = header.first().is_some_and(|h| h.trim().eq_ignore_ascii_case("name"));
+ let (kinds, existing_names) = {
+ let st = s.doc()?;
+ let kinds: BTreeMap<String, VariableKind> = st.doc.variables.variables.iter().map(|v| (v.name.clone(), v.kind)).collect();
+ let existing: BTreeSet<String> = st.doc.variables.datasets.iter().map(|d| d.name.clone()).collect();
+ (kinds, existing)
+ };
+ let mut columns: Vec<(usize, String, VariableKind)> = Vec::new();
+ let mut defined: Vec<Variable> = Vec::new();
+ for (i, h) in header.iter().enumerate().skip(usize::from(name_column)) {
+ let cell = h.trim();
+ if cell.is_empty() {
+ return Err(bad(C, format!("empty header in column {}", i + 1)));
+ }
+ let (name, kind) = header_variable(cell, i + 1, &kinds, &mut defined, C)?;
+ columns.push((i, name, kind));
+ }
+ // Lock Variables refuses New Variable, so the import is refused when a header would define one.
+ if !defined.is_empty() {
+ variables_unlocked(s).map_err(|why| EngineError::Disabled(C.into(), why))?;
+ }
+ if rows.len() == 1 {
+ return Err(bad(C, "CSV has a header but no data rows"));
+ }
+ let mut parsed: Vec<DataSet> = Vec::new();
+ // The names of the rows read so far, and the last number given to a row without a name.
+ let (mut seen, mut number) = (BTreeSet::new(), 0usize);
+ for &(line, ref row) in rows.iter().skip(1) {
+ let raw_name = if name_column { row.first().map_or("", |c| c.trim()) } else { "" };
+ let name = if raw_name.is_empty() {
+ // Same numbering as Capture Data Set when a row leaves the name blank or the file has
+ // no name column. The search starts after the last number given.
+ let (n, name) = (number + 1..)
+ .map(|n| (n, format!("Data Set {n}")))
+ .find(|(_, n)| !seen.contains(n) && !existing_names.contains(n))
+ .unwrap_or_else(|| (line, format!("Data Set {line}")));
+ number = n;
+ name
+ } else if raw_name.chars().count() > MAX_NAME {
+ return Err(bad(C, format!("line {line}: `name` is longer than {MAX_NAME} characters")));
+ } else {
+ raw_name.to_string()
+ };
+ if !seen.insert(name.clone()) {
+ return Err(bad(C, format!("line {line}: dataset `{name}` appears twice")));
+ }
+ let mut values = BTreeMap::new();
+ for &(col, ref var, kind) in &columns {
+ let cell = row.get(col).map(String::as_str).unwrap_or("");
+ match cell_to_value(kind, cell, var, C) {
+ Ok(Some(v)) => {
+ values.insert(var.clone(), v);
+ }
+ Ok(None) => {}
+ Err(_) => return Err(bad(C, format!("line {line}: value for `{var}` must match its variable kind"))),
+ }
+ }
+ parsed.push(DataSet { name, values });
+ }
+ let names: Vec<String> = parsed.iter().map(|d| d.name.clone()).collect();
+ let new_variables: Vec<String> = defined.iter().map(|v| v.name.clone()).collect();
+ let (imported, updated) = s.edit("Import Data Sets", |d, _| {
+ d.variables.variables.extend(defined);
+ let mut imported = 0usize;
+ let mut updated = 0usize;
+ for row in parsed {
+ if let Some(existing) = d.variables.datasets.iter_mut().find(|x| x.name == row.name) {
+ *existing = row;
+ updated += 1;
+ } else {
+ d.variables.datasets.push(row);
+ imported += 1;
+ }
+ }
+ Ok((imported, updated))
+ })?;
+ Ok(json!({"imported": imported, "updated": updated, "names": names, "defined": new_variables}))
+}
+
+/// Export every dataset as CSV (`name` then variables). With `path`, write the file; else return `{csv}`.
+pub(super) fn dataset_export(s: &mut Session, p: &Value) -> Result<Value> {
+ const C: &str = "dataset.export";
+ let st = s.doc()?;
+ if st.doc.variables.variables.is_empty() {
+ return Err(bad(C, "define a variable first"));
+ }
+ let csv = datasets_to_csv(&st.doc.variables);
+ if csv.len() > MAX_CSV_BYTES {
+ return Err(bad(C, format!("CSV would be larger than {MAX_CSV_BYTES} bytes")));
+ }
+ match str_param(p, "path") {
+ Some(path) => {
+ let path = absolute_path(path);
+ write_file(&path, csv.as_bytes())?;
+ Ok(json!({"path": path, "datasets": st.doc.variables.datasets.len()}))
+ }
+ None => Ok(json!({"csv": csv, "datasets": st.doc.variables.datasets.len()})),
+ }
+}
diff --git a/crates/engine/src/cmd/variablecmds/library.rs b/crates/engine/src/cmd/variablecmds/library.rs
new file mode 100644
index 000000000..83374a16d
--- /dev/null
+++ b/crates/engine/src/cmd/variablecmds/library.rs
@@ -0,0 +1,298 @@
+use super::*;
+
+/// Caps on a variable library (paths and MCP payloads are untrusted): its size, its XML nodes, how
+/// deep its elements nest, and the entities, variables and data sets it declares.
+const MAX_LIBRARY_BYTES: usize = 8 << 20;
+
+const MAX_LIBRARY_ENTITIES: usize = 64;
+
+const MAX_LIBRARY_NODES: u32 = 2_000_000;
+
+const MAX_LIBRARY_NESTING: usize = 256;
+
+const MAX_LIBRARY_VARIABLES: usize = 256;
+
+const MAX_LIBRARY_DATASETS: usize = 1_000;
+
+/// Bytes of a variable library from `path`, `xml` or `dataBase64`.
+fn library_bytes(p: &Value, cmd: &str) -> Result<Vec<u8>> {
+ let bytes = match (str_param(p, "path"), str_param(p, "xml"), str_param(p, "dataBase64")) {
+ // One byte past the cap is enough to refuse the file below.
+ (Some(path), ..) => read_file_head(&absolute_path(path), MAX_LIBRARY_BYTES + 1)?,
+ (None, Some(xml), _) => xml.as_bytes().to_vec(),
+ (None, None, Some(b64)) => vectorcraft_format::base64_decode(b64).ok_or_else(|| bad(cmd, "bad dataBase64"))?,
+ _ => return Err(bad(cmd, "give `path`, `xml` or `dataBase64`")),
+ };
+ if bytes.len() > MAX_LIBRARY_BYTES {
+ return Err(bad(cmd, format!("the variable library is larger than {MAX_LIBRARY_BYTES} bytes")));
+ }
+ Ok(bytes)
+}
+
+/// `xml` without the document type declaration of its prolog, with each reference to a general
+/// entity that the declaration defines replaced by the entity's value, its quotes and `>` escaped,
+/// outside CDATA sections, comments and processing instructions. The declaration may hold an
+/// external identifier and an internal subset of white space, comments, processing instructions and
+/// up to [`MAX_LIBRARY_ENTITIES`] entity declarations, each with a name of at most [`MAX_NAME`]
+/// bytes and a quoted value that holds no `<`, `&` or `%`. Anything else in it is refused, and so is
+/// a text that the replacements make longer than [`MAX_LIBRARY_BYTES`]. The XML parser then refuses
+/// any other document type declaration, so [`nesting`](super::place::docx::nesting) counts the
+/// elements the parser reads.
+fn without_doctype(xml: &str, cmd: &str) -> Result<String> {
+ let b = xml.as_bytes();
+ let past = |from: usize, end: &str| xml.get(from..).and_then(|rest| rest.find(end)).map(|at| from + at + end.len());
+ let spaces = |from: usize| from + b.get(from..).map_or(0, |rest| rest.iter().take_while(|c| c.is_ascii_whitespace()).count());
+ let at = |i: usize, s: &str| xml.get(i..).is_some_and(|rest| rest.starts_with(s));
+ let refused = || bad(cmd, "the variable library's document type declaration holds something other than entities of plain text");
+ // The prolog: an XML declaration, white space, comments and processing instructions.
+ let mut i = 0;
+ loop {
+ i = spaces(i);
+ if at(i, "<?") {
+ i = past(i + 2, "?>").ok_or_else(refused)?;
+ } else if at(i, "<!--") {
+ i = past(i + 4, "-->").ok_or_else(refused)?;
+ } else {
+ break;
+ }
+ }
+ if !at(i, "<!DOCTYPE") {
+ return Ok(xml.to_string());
+ }
+ let start = i;
+ // A quoted literal starting at `i` → the index past it and its text.
+ let literal = |i: usize| -> Option<(usize, &str)> {
+ let quote = *b.get(i).filter(|c| matches!(c, b'"' | b'\''))?;
+ let len = b.get(i + 1..)?.iter().position(|c| *c == quote)?;
+ Some((i + len + 2, xml.get(i + 1..i + 1 + len)?))
+ };
+ // `<!DOCTYPE` S Name (S ExternalID)? S? ('[' intSubset ']' S?)? '>'
+ i = spaces(i + "<!DOCTYPE".len());
+ i += b.get(i..).map_or(0, |rest| rest.iter().take_while(|c| !c.is_ascii_whitespace() && !matches!(c, b'[' | b'>')).count());
+ i = spaces(i);
+ for (id, literals) in [("SYSTEM", 1), ("PUBLIC", 2)] {
+ if at(i, id) {
+ i += id.len();
+ for _ in 0..literals {
+ i = literal(spaces(i)).ok_or_else(refused)?.0;
+ }
+ i = spaces(i);
+ }
+ }
+ let mut entities: Vec<(&str, String)> = Vec::new();
+ if at(i, "[") {
+ i += 1;
+ loop {
+ i = spaces(i);
+ if at(i, "]") {
+ i = spaces(i + 1);
+ break;
+ } else if at(i, "<!--") {
+ i = past(i + 4, "-->").ok_or_else(refused)?;
+ } else if at(i, "<?") {
+ i = past(i + 2, "?>").ok_or_else(refused)?;
+ } else if at(i, "<!ENTITY") {
+ i = spaces(i + "<!ENTITY".len());
+ let len =
+ b.get(i..).map_or(0, |rest| rest.iter().take_while(|c| !c.is_ascii_whitespace() && !matches!(c, b'"' | b'\'' | b'>')).count());
+ let name = xml.get(i..i + len).filter(|n| !n.is_empty() && n.len() <= MAX_NAME && !n.starts_with('%')).ok_or_else(refused)?;
+ let (end, value) = literal(spaces(i + len)).ok_or_else(refused)?;
+ if value.contains(['<', '&', '%']) {
+ return Err(refused());
+ }
+ i = spaces(end);
+ if !at(i, ">") {
+ return Err(refused());
+ }
+ i += 1;
+ if entities.len() == MAX_LIBRARY_ENTITIES {
+ return Err(bad(cmd, format!("the variable library declares more than {MAX_LIBRARY_ENTITIES} entities")));
+ }
+ entities.push((name, value.replace('"', "&quot;").replace('\'', "&apos;").replace('>', "&gt;")));
+ } else {
+ return Err(refused());
+ }
+ }
+ }
+ if !at(i, ">") {
+ return Err(refused());
+ }
+ let mut out = String::with_capacity(xml.len());
+ out.push_str(xml.get(..start).unwrap_or_default());
+ // The rest, copied with the references replaced outside CDATA sections, comments and
+ // processing instructions, where the parser reads none.
+ let mut i = i + 1;
+ while let Some(c) = xml.get(i..).and_then(|rest| rest.chars().next()) {
+ let verbatim = [("<![CDATA[", "]]>"), ("<!--", "-->"), ("<?", "?>")].into_iter().find(|(open, _)| at(i, open));
+ if let Some((open, close)) = verbatim {
+ let end = past(i + open.len(), close).unwrap_or(xml.len());
+ out.push_str(xml.get(i..end).unwrap_or_default());
+ i = end;
+ continue;
+ }
+ // Entity names are at most `MAX_NAME` bytes long, so the `;` is looked for that far.
+ let entity = if c == '&' {
+ let name = xml.get(i + 1..).and_then(|rest| rest.get(..rest.bytes().take(MAX_NAME + 1).position(|b| b == b';')?));
+ name.and_then(|name| entities.iter().find(|(n, _)| *n == name))
+ } else {
+ None
+ };
+ match entity {
+ Some((name, value)) => {
+ out.push_str(value);
+ i += name.len() + 2;
+ if out.len() > MAX_LIBRARY_BYTES {
+ return Err(bad(cmd, format!("the variable library's entities expand to more than {MAX_LIBRARY_BYTES} bytes")));
+ }
+ }
+ None => {
+ out.push(c);
+ i += c.len_utf8();
+ }
+ }
+ }
+ if out.len() > MAX_LIBRARY_BYTES {
+ return Err(bad(cmd, format!("the variable library's entities expand to more than {MAX_LIBRARY_BYTES} bytes")));
+ }
+ Ok(out)
+}
+
+/// A variable library as Load Variable Library reads it: the names of its text variables in the
+/// order it declares them, the name and `trait` of each variable of another kind, and its data
+/// sets with their text values.
+struct Library {
+ text: Vec<String>,
+ other: Vec<(String, String)>,
+ datasets: Vec<DataSet>,
+}
+
+/// Read a variable library: XML whose `variableSets` element holds `variableSet` elements, each
+/// with a `variables` element of `variable` elements (`varName`, `trait`) and a `sampleDataSets`
+/// element of `sampleDataSet` elements (`dataSetName`). Elements are matched by their local names.
+/// A variable whose `trait` is `textcontent` is a text variable. In a data set, an element named
+/// after a text variable gives its value: the text of its `p` elements, one paragraph each, and
+/// none when it has no `p` element. Values for variables of other kinds are left out. A variable
+/// or data set declared twice, a value for a variable the library doesn't declare, and a library
+/// past the caps are refused. Entities that its document type declaration defines are read as
+/// [`without_doctype`] says.
+fn parse_library(xml: &str, cmd: &str) -> Result<Library> {
+ let xml = without_doctype(xml, cmd)?;
+ // roxmltree parses nested elements recursively.
+ if crate::cmd::place::docx::nesting(&xml) > MAX_LIBRARY_NESTING {
+ return Err(bad(cmd, format!("the variable library nests elements more than {MAX_LIBRARY_NESTING} deep")));
+ }
+ let opts = roxmltree::ParsingOptions { nodes_limit: MAX_LIBRARY_NODES, ..roxmltree::ParsingOptions::default() };
+ let doc =
+ roxmltree::Document::parse_with_options(&xml, opts).map_err(|e| bad(cmd, format!("the variable library is not well-formed XML: {e}")))?;
+ let is = |n: &roxmltree::Node, name: &str| n.is_element() && n.tag_name().name() == name;
+ let Some(root) = doc.descendants().find(|n| is(n, "variableSets")) else {
+ return Err(bad(cmd, "the file holds no `variableSets` element"));
+ };
+ let sets: Vec<roxmltree::Node> = root.children().filter(|n| is(n, "variableSet")).collect();
+ let (mut text, mut other, mut declared) = (Vec::new(), Vec::new(), BTreeSet::new());
+ for var in sets.iter().flat_map(|s| s.children()).filter(|n| is(n, "variables")).flat_map(|v| v.children()).filter(|n| is(n, "variable")) {
+ let name = named(var.attribute("varName"), "varName", cmd)?;
+ if !declared.insert(name.clone()) {
+ return Err(bad(cmd, format!("variable `{name}` is declared twice")));
+ }
+ if declared.len() > MAX_LIBRARY_VARIABLES {
+ return Err(bad(cmd, format!("the variable library declares more than {MAX_LIBRARY_VARIABLES} variables")));
+ }
+ match var.attribute("trait") {
+ Some("textcontent") => text.push(name),
+ Some(t) => other.push((name, t.chars().take(MAX_NAME).collect())),
+ None => return Err(bad(cmd, format!("variable `{name}` has no `trait`"))),
+ }
+ }
+ if declared.is_empty() {
+ return Err(bad(cmd, "the variable library declares no variables"));
+ }
+ let text_names: BTreeSet<&str> = text.iter().map(String::as_str).collect();
+ let (mut datasets, mut seen) = (Vec::new(), BTreeSet::new());
+ for row in
+ sets.iter().flat_map(|s| s.children()).filter(|n| is(n, "sampleDataSets")).flat_map(|d| d.children()).filter(|n| is(n, "sampleDataSet"))
+ {
+ let name = named(row.attribute("dataSetName"), "dataSetName", cmd)?;
+ if !seen.insert(name.clone()) {
+ return Err(bad(cmd, format!("data set `{name}` is declared twice")));
+ }
+ if seen.len() > MAX_LIBRARY_DATASETS {
+ return Err(bad(cmd, format!("the variable library declares more than {MAX_LIBRARY_DATASETS} data sets")));
+ }
+ let (mut values, mut held) = (BTreeMap::new(), BTreeSet::new());
+ for el in row.children().filter(roxmltree::Node::is_element) {
+ let var = el.tag_name().name();
+ if !declared.contains(var) {
+ return Err(bad(cmd, format!("data set `{name}` holds a value for `{var}`, which the library doesn't declare")));
+ }
+ if !held.insert(var) {
+ return Err(bad(cmd, format!("data set `{name}` holds two values for `{var}`")));
+ }
+ if !text_names.contains(var) {
+ continue;
+ }
+ let paras: Vec<String> = el
+ .children()
+ .filter(|n| is(n, "p"))
+ .map(|p| p.descendants().filter(roxmltree::Node::is_text).filter_map(|t| t.text()).collect())
+ .collect();
+ if !paras.is_empty() {
+ values.insert(var.to_string(), DataValue::Text(paras.join("\n")));
+ }
+ }
+ datasets.push(DataSet { name, values });
+ }
+ Ok(Library { text, other, datasets })
+}
+
+/// Load Variable Library: replace the document's variables and data sets with a variable library's
+/// ([`parse_library`]) in one undo step. Every variable of the document is deleted with its bindings
+/// and values, and every data set is deleted. The library's text variables are then defined,
+/// unbound, and its data sets are added. The library's variables of other kinds are left out, named
+/// in `skipped` and in a warning.
+pub(super) fn load_library(s: &mut Session, p: &Value) -> Result<Value> {
+ const C: &str = "variable.loadLibrary";
+ let bytes = library_bytes(p, C)?;
+ let xml = std::str::from_utf8(bytes.strip_prefix(b"\xEF\xBB\xBF").unwrap_or(&bytes)).map_err(|_| bad(C, "the variable library is not UTF-8"))?;
+ let lib = parse_library(xml, C)?;
+ let vars = &s.doc()?.doc.variables;
+ let deleted: Vec<String> = vars.variables.iter().map(|v| v.name.clone()).collect();
+ let deleted_sets: Vec<String> = vars.datasets.iter().map(|d| d.name.clone()).collect();
+ // Lock Variables refuses New Variable and Delete Variable, so the load is refused when it would
+ // delete or define a variable.
+ if !deleted.is_empty() || !lib.text.is_empty() {
+ variables_unlocked(s).map_err(|why| EngineError::Disabled(C.into(), why))?;
+ }
+ let added: Vec<Variable> = lib.text.iter().map(|n| Variable { name: n.clone(), kind: VariableKind::Text }).collect();
+ let names: Vec<String> = lib.datasets.iter().map(|d| d.name.clone()).collect();
+ s.edit("Load Variable Library", |d, _| {
+ d.variables.prune(&deleted);
+ d.variables.variables.extend(added);
+ d.variables.datasets = lib.datasets;
+ d.variables.active_dataset = None;
+ Ok(())
+ })?;
+ // A highlight of a variable that is gone highlights nothing.
+ let st = s.doc_mut()?;
+ if st.variables_highlight.as_ref().is_some_and(|h| st.doc.variables.variable(h).is_none()) {
+ st.variables_highlight = None;
+ }
+ st.revision += 1;
+ let skipped: Vec<Value> = lib.other.iter().map(|(name, t)| json!({"name": name, "trait": t})).collect();
+ let mut out = json!({
+ "variables": lib.text, "deleted": deleted, "deletedDataSets": deleted_sets, "names": names, "skipped": skipped,
+ });
+ // Each `trait` once, in the order the library first uses it, with how many variables have it.
+ let mut kinds: Vec<(&str, usize)> = Vec::new();
+ for (_, t) in &lib.other {
+ match kinds.iter_mut().find(|(k, _)| k == t) {
+ Some((_, n)) => *n += 1,
+ None => kinds.push((t, 1)),
+ }
+ }
+ if !kinds.is_empty() {
+ let kinds = kinds.iter().map(|(k, n)| format!("{k} ({n})")).collect::<Vec<_>>().join(", ");
+ out["warning"] = json!(format!("variables that aren't text were left out: {kinds}"));
+ }
+ Ok(out)
+}