VectorCraft: An open-source, vector art tool written by scratch in pure Rust
refactor(engine): move cmd/variablecmds.rs items to cmd/variablecmds/{csv,library}.rs
aiongg4f7dd932681d
diff --git a/crates/engine/src/cmd/variablecmds.rs b/crates/engine/src/cmd/variablecmds.rsindex 0106e9895..be5fec64a 100644--- a/crates/engine/src/cmd/variablecmds.rs+++ b/crates/engine/src/cmd/variablecmds.rs@@ -21,24 +21,14 @@ use super::fileio::{absolute_path, read_file_head, write_file};use super::textedit::set_plain_text;use super::*;-/// The longest variable or dataset name, in characters (names arrive from MCP and files).-const MAX_NAME: usize = 255;-/// Cap on a graph variable's CSV (MCP and files are untrusted).-const MAX_GRAPH_CSV: usize = 256 * 1024;+mod csv;+mod library;-/// Caps on a datasets CSV (paths and MCP payloads are untrusted).-const MAX_CSV_BYTES: usize = 1_048_576;-const MAX_CSV_ROWS: usize = 1_000;-const MAX_CSV_COLS: usize = 256;+use csv::{MAX_GRAPH_CSV, dataset_export, dataset_import};+use library::load_library;-/// Caps on a variable library (paths and MCP payloads are untrusted): its size, its XML nodes, how-/// deep its elements nest, and the entities, variables and data sets it declares.-const MAX_LIBRARY_BYTES: usize = 8 << 20;-const MAX_LIBRARY_ENTITIES: usize = 64;-const MAX_LIBRARY_NODES: u32 = 2_000_000;-const MAX_LIBRARY_NESTING: usize = 256;-const MAX_LIBRARY_VARIABLES: usize = 256;-const MAX_LIBRARY_DATASETS: usize = 1_000;+/// The longest variable or dataset name, in characters (names arrive from MCP and files).+const MAX_NAME: usize = 255;/// Lock Variables refuses New Variable, Delete Variable, Rename Variable (Variable Options),/// Make … Dynamic, an import whose header would define a variable and a Load Variable Library@@ -545,346 +535,6 @@ fn dataset_select(s: &mut Session, p: &Value) -> Result<Value> {apply_dataset(s, &var_name(p, "dataset.select")?)}-/// Bytes of a datasets CSV from `path`, `data`, `dataBase64` or `csv`.-fn datasets_csv_bytes(p: &Value, cmd: &str) -> Result<Vec<u8>> {- let bytes = match (str_param(p, "path"), str_param(p, "data"), str_param(p, "dataBase64"), str_param(p, "csv")) {- // One byte past the cap is enough to refuse the file below.- (Some(path), ..) => read_file_head(&absolute_path(path), MAX_CSV_BYTES + 1)?,- (None, Some(text), ..) => text.as_bytes().to_vec(),- (None, None, Some(b64), _) => vectorcraft_format::base64_decode(b64).ok_or_else(|| bad(cmd, "bad dataBase64"))?,- (None, None, None, Some(csv)) => csv.as_bytes().to_vec(),- _ => return Err(bad(cmd, "give `path`, `data`, `dataBase64` or `csv`")),- };- if bytes.len() > MAX_CSV_BYTES {- return Err(bad(cmd, format!("CSV is larger than {MAX_CSV_BYTES} bytes")));- }- Ok(bytes)-}--/// A row of a datasets CSV: the file line it starts on, and its cells.-type CsvRow = (usize, Vec<String>);--/// Whether a datasets CSV separates its cells with tabs: its header row (the first line that-/// isn't blank), read with tabs between cells, has a tab outside quotes and no comma outside-/// quotes.-fn tab_separated(text: &str) -> bool {- // `start`: nothing but `\r` read yet in this cell. `blank`: no cell on this line holds- // anything but white space yet.- let (mut in_quotes, mut start, mut blank, mut tab) = (false, true, true, false);- let mut chars = text.chars().peekable();- while let Some(c) = chars.next() {- match c {- '"' if in_quotes => {- if chars.peek() == Some(&'"') {- chars.next();- blank = false;- } else {- in_quotes = false;- }- }- '"' if start => in_quotes = true,- c if in_quotes => blank &= c.is_whitespace(),- ',' => return false,- '\t' => {- tab = true;- start = true;- continue;- }- '\r' => continue,- '\n' if blank => {- tab = false;- start = true;- continue;- }- '\n' => return tab,- c => blank &= c.is_whitespace(),- }- start = false;- }- tab-}--/// Parse a datasets CSV into rows of cells (no header interpretation yet). Cells are separated-/// by commas, or by tabs when [`tab_separated`] says so. `"` quotes a cell (doubled inside),-/// which may then hold separators and line breaks, as [`csv_escape`] writes them. An unquoted-/// cell is trimmed. A quoted cell keeps what its quotes hold as written, spaces included, and-/// drops white space after the closing quote at the end of the cell. Blank lines are skipped.-fn parse_datasets_csv(text: &str, cmd: &str) -> Result<Vec<CsvRow>> {- let text = text.strip_prefix('\u{feff}').unwrap_or(text);- let sep = if tab_separated(text) { '\t' } else { ',' };- let (mut rows, mut row, mut cur) = (Vec::new(), Vec::<String>::new(), String::new());- // `closed`: where the closing quote of a quoted cell falls in `cur`. `start`: the line the- // current row starts on.- let (mut in_quotes, mut closed, mut line, mut start) = (false, None, 1usize, 1usize);- let mut chars = text.chars().peekable();- loop {- let c = chars.next();- match c {- Some('"') if in_quotes => {- if chars.peek() == Some(&'"') {- cur.push('"');- chars.next();- } else {- in_quotes = false;- closed = Some(cur.len());- }- }- Some('"') if cur.is_empty() => in_quotes = true,- Some(ch) if ch == sep && !in_quotes => row.push(end_csv_cell(&mut cur, &mut closed)),- Some('\r') if !in_quotes => {}- Some('\n') | None if !in_quotes => {- row.push(end_csv_cell(&mut cur, &mut closed));- if row.iter().any(|c| !c.trim().is_empty()) {- rows.push((start, std::mem::take(&mut row)));- // +1 for the header.- if rows.len() > MAX_CSV_ROWS + 1 {- return Err(bad(cmd, format!("more than {MAX_CSV_ROWS} data rows")));- }- }- row.clear();- if c.is_none() {- return Ok(rows);- }- line += 1;- start = line;- }- None => return Err(bad(cmd, format!("line {line}: unclosed quote"))),- Some(c) => {- line += usize::from(c == '\n');- cur.push(c);- }- }- if row.len() > MAX_CSV_COLS {- return Err(bad(cmd, format!("line {line}: more than {MAX_CSV_COLS} columns")));- }- }-}--/// Take the cell read so far out of `cur`. Without a closing quote (`closed` is `None`) the cell-/// is trimmed; with one, the text before it is kept as written and the text after it is-/// trimmed at the end.-fn end_csv_cell(cur: &mut String, closed: &mut Option<usize>) -> String {- let cell = std::mem::take(cur);- match closed.take() {- Some(at) => match cell.split_at_checked(at) {- Some((quoted, after)) => format!("{quoted}{}", after.trim_end()),- None => cell,- },- None => cell.trim().to_string(),- }-}--fn csv_escape(s: &str) -> String {- if s.contains([',', '"', '\n', '\r']) || s != s.trim() { format!("\"{}\"", s.replace('"', "\"\"")) } else { s.to_string() }-}--/// A visibility cell: true/false, 1/0, yes/no (ASCII, case-insensitive).-fn parse_visible_cell(s: &str, var: &str, cmd: &str) -> Result<bool> {- match s.trim().to_ascii_lowercase().as_str() {- "true" | "1" | "yes" => Ok(true),- "false" | "0" | "no" => Ok(false),- _ => Err(bad(cmd, format!("value for `{var}` must be true or false"))),- }-}--fn cell_to_value(kind: VariableKind, cell: &str, var: &str, cmd: &str) -> Result<Option<DataValue>> {- // A text cell is not trimmed here (the parser trims unquoted cells); image, graph and- // visibility cells are trimmed.- let t = if kind == VariableKind::Text { cell } else { cell.trim() };- if t.is_empty() {- return Ok(None);- }- Ok(Some(match kind {- VariableKind::Text => DataValue::Text(t.to_string()),- VariableKind::Image => DataValue::Image(t.to_string()),- VariableKind::Graph if t.len() <= MAX_GRAPH_CSV => DataValue::Graph(t.to_string()),- VariableKind::Graph => return Err(bad(cmd, format!("value for `{var}` is longer than {MAX_GRAPH_CSV} bytes"))),- VariableKind::Visibility => DataValue::Visible(parse_visible_cell(t, var, cmd)?),- }))-}--fn value_to_cell(v: &DataValue) -> String {- match v {- DataValue::Text(t) | DataValue::Image(t) | DataValue::Graph(t) => t.clone(),- DataValue::Visible(b) => if *b { "true" } else { "false" }.to_string(),- }-}--/// Build the CSV of every dataset (header `name` then variables in definition order).-fn datasets_to_csv(vars: &Variables) -> String {- let mut out = String::from("name");- for v in &vars.variables {- out.push(',');- out.push_str(&csv_escape(&v.name));- }- out.push('\n');- for ds in &vars.datasets {- out.push_str(&csv_escape(&ds.name));- for v in &vars.variables {- out.push(',');- if let Some(val) = ds.values.get(&v.name) {- out.push_str(&csv_escape(&value_to_cell(val)));- }- }- out.push('\n');- }- out-}--/// The variable that a datasets CSV header cell (trimmed, not empty) in 1-based `column` names,-/// and its kind. A cell that is the name of a variable in `kinds` or in `defined` (the variables-/// that earlier columns define) names that variable. Otherwise `#` before the name makes it a-/// visibility variable, `@` makes it an image variable, and a cell without a prefix names a text-/// variable. A prefixed name that is already a variable must be one of the prefix's kind; a new-/// name is added to `defined`. A `%` cell that names no variable (graph data files) is refused.-fn header_variable(- cell: &str,- column: usize,- kinds: &BTreeMap<String, VariableKind>,- defined: &mut Vec<Variable>,- cmd: &str,-) -> Result<(String, VariableKind)> {- let known = |name: &str, defined: &[Variable]| kinds.get(name).copied().or_else(|| defined.iter().find(|v| v.name == name).map(|v| v.kind));- if let Some(kind) = known(cell, defined) {- return Ok((cell.to_string(), kind));- }- let (name, kind) = if let Some(rest) = cell.strip_prefix('#') {- (rest.trim(), VariableKind::Visibility)- } else if let Some(rest) = cell.strip_prefix('@') {- (rest.trim(), VariableKind::Image)- } else if cell.starts_with('%') {- return Err(bad(cmd, format!("column {column}: `{cell}` is a `%` column of graph data files, which are not read")));- } else {- (cell, VariableKind::Text)- };- if name.is_empty() {- return Err(bad(cmd, format!("column {column}: `{cell}` names no variable")));- }- match known(name, defined) {- Some(k) if k == kind => Ok((name.to_string(), kind)),- Some(k) => Err(bad(cmd, format!("column {column}: `{cell}` names `{name}`, which is a {} variable", k.label()))),- None if name.chars().count() > MAX_NAME => Err(bad(cmd, format!("column {column}: the variable name is longer than {MAX_NAME} characters"))),- None => {- defined.push(Variable { name: name.to_string(), kind });- Ok((name.to_string(), kind))- }- }-}--/// Import datasets from CSV. A first header cell `name` makes the first column the dataset-/// names; the other headers name variables, as [`header_variable`] reads them, and the-/// variables the document doesn't define yet are defined, unbound. Without the `name` column,-/// every header names a variable and the rows take the first free names "Data Set 1", "Data-/// Set 2" and so on. Existing rows of the same name are replaced; new names are appended. One-/// undo step.-fn dataset_import(s: &mut Session, p: &Value) -> Result<Value> {- const C: &str = "dataset.import";- let bytes = datasets_csv_bytes(p, C)?;- let text = std::str::from_utf8(&bytes).map_err(|_| bad(C, "CSV is not UTF-8"))?;- let rows = parse_datasets_csv(text, C)?;- let Some((_, header)) = rows.first() else {- return Err(bad(C, "CSV is empty"));- };- let name_column = header.first().is_some_and(|h| h.trim().eq_ignore_ascii_case("name"));- let (kinds, existing_names) = {- let st = s.doc()?;- let kinds: BTreeMap<String, VariableKind> = st.doc.variables.variables.iter().map(|v| (v.name.clone(), v.kind)).collect();- let existing: BTreeSet<String> = st.doc.variables.datasets.iter().map(|d| d.name.clone()).collect();- (kinds, existing)- };- let mut columns: Vec<(usize, String, VariableKind)> = Vec::new();- let mut defined: Vec<Variable> = Vec::new();- for (i, h) in header.iter().enumerate().skip(usize::from(name_column)) {- let cell = h.trim();- if cell.is_empty() {- return Err(bad(C, format!("empty header in column {}", i + 1)));- }- let (name, kind) = header_variable(cell, i + 1, &kinds, &mut defined, C)?;- columns.push((i, name, kind));- }- // Lock Variables refuses New Variable, so the import is refused when a header would define one.- if !defined.is_empty() {- variables_unlocked(s).map_err(|why| EngineError::Disabled(C.into(), why))?;- }- if rows.len() == 1 {- return Err(bad(C, "CSV has a header but no data rows"));- }- let mut parsed: Vec<DataSet> = Vec::new();- // The names of the rows read so far, and the last number given to a row without a name.- let (mut seen, mut number) = (BTreeSet::new(), 0usize);- for &(line, ref row) in rows.iter().skip(1) {- let raw_name = if name_column { row.first().map_or("", |c| c.trim()) } else { "" };- let name = if raw_name.is_empty() {- // Same numbering as Capture Data Set when a row leaves the name blank or the file has- // no name column. The search starts after the last number given.- let (n, name) = (number + 1..)- .map(|n| (n, format!("Data Set {n}")))- .find(|(_, n)| !seen.contains(n) && !existing_names.contains(n))- .unwrap_or_else(|| (line, format!("Data Set {line}")));- number = n;- name- } else if raw_name.chars().count() > MAX_NAME {- return Err(bad(C, format!("line {line}: `name` is longer than {MAX_NAME} characters")));- } else {- raw_name.to_string()- };- if !seen.insert(name.clone()) {- return Err(bad(C, format!("line {line}: dataset `{name}` appears twice")));- }- let mut values = BTreeMap::new();- for &(col, ref var, kind) in &columns {- let cell = row.get(col).map(String::as_str).unwrap_or("");- match cell_to_value(kind, cell, var, C) {- Ok(Some(v)) => {- values.insert(var.clone(), v);- }- Ok(None) => {}- Err(_) => return Err(bad(C, format!("line {line}: value for `{var}` must match its variable kind"))),- }- }- parsed.push(DataSet { name, values });- }- let names: Vec<String> = parsed.iter().map(|d| d.name.clone()).collect();- let new_variables: Vec<String> = defined.iter().map(|v| v.name.clone()).collect();- let (imported, updated) = s.edit("Import Data Sets", |d, _| {- d.variables.variables.extend(defined);- let mut imported = 0usize;- let mut updated = 0usize;- for row in parsed {- if let Some(existing) = d.variables.datasets.iter_mut().find(|x| x.name == row.name) {- *existing = row;- updated += 1;- } else {- d.variables.datasets.push(row);- imported += 1;- }- }- Ok((imported, updated))- })?;- Ok(json!({"imported": imported, "updated": updated, "names": names, "defined": new_variables}))-}--/// Export every dataset as CSV (`name` then variables). With `path`, write the file; else return `{csv}`.-fn dataset_export(s: &mut Session, p: &Value) -> Result<Value> {- const C: &str = "dataset.export";- let st = s.doc()?;- if st.doc.variables.variables.is_empty() {- return Err(bad(C, "define a variable first"));- }- let csv = datasets_to_csv(&st.doc.variables);- if csv.len() > MAX_CSV_BYTES {- return Err(bad(C, format!("CSV would be larger than {MAX_CSV_BYTES} bytes")));- }- match str_param(p, "path") {- Some(path) => {- let path = absolute_path(path);- write_file(&path, csv.as_bytes())?;- Ok(json!({"path": path, "datasets": st.doc.variables.datasets.len()}))- }- None => Ok(json!({"csv": csv, "datasets": st.doc.variables.datasets.len()})),- }-}-fn dataset_step(s: &mut Session, cmd: &str, forward: bool) -> Result<Value> {let st = s.doc()?;let n = st.doc.variables.datasets.len();@@ -902,289 +552,6 @@ fn dataset_step(s: &mut Session, cmd: &str, forward: bool) -> Result<Value> {apply_dataset(s, &name)}-/// Bytes of a variable library from `path`, `xml` or `dataBase64`.-fn library_bytes(p: &Value, cmd: &str) -> Result<Vec<u8>> {- let bytes = match (str_param(p, "path"), str_param(p, "xml"), str_param(p, "dataBase64")) {- // One byte past the cap is enough to refuse the file below.- (Some(path), ..) => read_file_head(&absolute_path(path), MAX_LIBRARY_BYTES + 1)?,- (None, Some(xml), _) => xml.as_bytes().to_vec(),- (None, None, Some(b64)) => vectorcraft_format::base64_decode(b64).ok_or_else(|| bad(cmd, "bad dataBase64"))?,- _ => return Err(bad(cmd, "give `path`, `xml` or `dataBase64`")),- };- if bytes.len() > MAX_LIBRARY_BYTES {- return Err(bad(cmd, format!("the variable library is larger than {MAX_LIBRARY_BYTES} bytes")));- }- Ok(bytes)-}--/// `xml` without the document type declaration of its prolog, with each reference to a general-/// entity that the declaration defines replaced by the entity's value, its quotes and `>` escaped,-/// outside CDATA sections, comments and processing instructions. The declaration may hold an-/// external identifier and an internal subset of white space, comments, processing instructions and-/// up to [`MAX_LIBRARY_ENTITIES`] entity declarations, each with a name of at most [`MAX_NAME`]-/// bytes and a quoted value that holds no `<`, `&` or `%`. Anything else in it is refused, and so is-/// a text that the replacements make longer than [`MAX_LIBRARY_BYTES`]. The XML parser then refuses-/// any other document type declaration, so [`nesting`](super::place::docx::nesting) counts the-/// elements the parser reads.-fn without_doctype(xml: &str, cmd: &str) -> Result<String> {- let b = xml.as_bytes();- let past = |from: usize, end: &str| xml.get(from..).and_then(|rest| rest.find(end)).map(|at| from + at + end.len());- let spaces = |from: usize| from + b.get(from..).map_or(0, |rest| rest.iter().take_while(|c| c.is_ascii_whitespace()).count());- let at = |i: usize, s: &str| xml.get(i..).is_some_and(|rest| rest.starts_with(s));- let refused = || bad(cmd, "the variable library's document type declaration holds something other than entities of plain text");- // The prolog: an XML declaration, white space, comments and processing instructions.- let mut i = 0;- loop {- i = spaces(i);- if at(i, "<?") {- i = past(i + 2, "?>").ok_or_else(refused)?;- } else if at(i, "<!--") {- i = past(i + 4, "-->").ok_or_else(refused)?;- } else {- break;- }- }- if !at(i, "<!DOCTYPE") {- return Ok(xml.to_string());- }- let start = i;- // A quoted literal starting at `i` → the index past it and its text.- let literal = |i: usize| -> Option<(usize, &str)> {- let quote = *b.get(i).filter(|c| matches!(c, b'"' | b'\''))?;- let len = b.get(i + 1..)?.iter().position(|c| *c == quote)?;- Some((i + len + 2, xml.get(i + 1..i + 1 + len)?))- };- // `<!DOCTYPE` S Name (S ExternalID)? S? ('[' intSubset ']' S?)? '>'- i = spaces(i + "<!DOCTYPE".len());- i += b.get(i..).map_or(0, |rest| rest.iter().take_while(|c| !c.is_ascii_whitespace() && !matches!(c, b'[' | b'>')).count());- i = spaces(i);- for (id, literals) in [("SYSTEM", 1), ("PUBLIC", 2)] {- if at(i, id) {- i += id.len();- for _ in 0..literals {- i = literal(spaces(i)).ok_or_else(refused)?.0;- }- i = spaces(i);- }- }- let mut entities: Vec<(&str, String)> = Vec::new();- if at(i, "[") {- i += 1;- loop {- i = spaces(i);- if at(i, "]") {- i = spaces(i + 1);- break;- } else if at(i, "<!--") {- i = past(i + 4, "-->").ok_or_else(refused)?;- } else if at(i, "<?") {- i = past(i + 2, "?>").ok_or_else(refused)?;- } else if at(i, "<!ENTITY") {- i = spaces(i + "<!ENTITY".len());- let len =- b.get(i..).map_or(0, |rest| rest.iter().take_while(|c| !c.is_ascii_whitespace() && !matches!(c, b'"' | b'\'' | b'>')).count());- let name = xml.get(i..i + len).filter(|n| !n.is_empty() && n.len() <= MAX_NAME && !n.starts_with('%')).ok_or_else(refused)?;- let (end, value) = literal(spaces(i + len)).ok_or_else(refused)?;- if value.contains(['<', '&', '%']) {- return Err(refused());- }- i = spaces(end);- if !at(i, ">") {- return Err(refused());- }- i += 1;- if entities.len() == MAX_LIBRARY_ENTITIES {- return Err(bad(cmd, format!("the variable library declares more than {MAX_LIBRARY_ENTITIES} entities")));- }- entities.push((name, value.replace('"', """).replace('\'', "'").replace('>', ">")));- } else {- return Err(refused());- }- }- }- if !at(i, ">") {- return Err(refused());- }- let mut out = String::with_capacity(xml.len());- out.push_str(xml.get(..start).unwrap_or_default());- // The rest, copied with the references replaced outside CDATA sections, comments and- // processing instructions, where the parser reads none.- let mut i = i + 1;- while let Some(c) = xml.get(i..).and_then(|rest| rest.chars().next()) {- let verbatim = [("<![CDATA[", "]]>"), ("<!--", "-->"), ("<?", "?>")].into_iter().find(|(open, _)| at(i, open));- if let Some((open, close)) = verbatim {- let end = past(i + open.len(), close).unwrap_or(xml.len());- out.push_str(xml.get(i..end).unwrap_or_default());- i = end;- continue;- }- // Entity names are at most `MAX_NAME` bytes long, so the `;` is looked for that far.- let entity = if c == '&' {- let name = xml.get(i + 1..).and_then(|rest| rest.get(..rest.bytes().take(MAX_NAME + 1).position(|b| b == b';')?));- name.and_then(|name| entities.iter().find(|(n, _)| *n == name))- } else {- None- };- match entity {- Some((name, value)) => {- out.push_str(value);- i += name.len() + 2;- if out.len() > MAX_LIBRARY_BYTES {- return Err(bad(cmd, format!("the variable library's entities expand to more than {MAX_LIBRARY_BYTES} bytes")));- }- }- None => {- out.push(c);- i += c.len_utf8();- }- }- }- if out.len() > MAX_LIBRARY_BYTES {- return Err(bad(cmd, format!("the variable library's entities expand to more than {MAX_LIBRARY_BYTES} bytes")));- }- Ok(out)-}--/// A variable library as Load Variable Library reads it: the names of its text variables in the-/// order it declares them, the name and `trait` of each variable of another kind, and its data-/// sets with their text values.-struct Library {- text: Vec<String>,- other: Vec<(String, String)>,- datasets: Vec<DataSet>,-}--/// Read a variable library: XML whose `variableSets` element holds `variableSet` elements, each-/// with a `variables` element of `variable` elements (`varName`, `trait`) and a `sampleDataSets`-/// element of `sampleDataSet` elements (`dataSetName`). Elements are matched by their local names.-/// A variable whose `trait` is `textcontent` is a text variable. In a data set, an element named-/// after a text variable gives its value: the text of its `p` elements, one paragraph each, and-/// none when it has no `p` element. Values for variables of other kinds are left out. A variable-/// or data set declared twice, a value for a variable the library doesn't declare, and a library-/// past the caps are refused. Entities that its document type declaration defines are read as-/// [`without_doctype`] says.-fn parse_library(xml: &str, cmd: &str) -> Result<Library> {- let xml = without_doctype(xml, cmd)?;- // roxmltree parses nested elements recursively.- if super::place::docx::nesting(&xml) > MAX_LIBRARY_NESTING {- return Err(bad(cmd, format!("the variable library nests elements more than {MAX_LIBRARY_NESTING} deep")));- }- let opts = roxmltree::ParsingOptions { nodes_limit: MAX_LIBRARY_NODES, ..roxmltree::ParsingOptions::default() };- let doc =- roxmltree::Document::parse_with_options(&xml, opts).map_err(|e| bad(cmd, format!("the variable library is not well-formed XML: {e}")))?;- let is = |n: &roxmltree::Node, name: &str| n.is_element() && n.tag_name().name() == name;- let Some(root) = doc.descendants().find(|n| is(n, "variableSets")) else {- return Err(bad(cmd, "the file holds no `variableSets` element"));- };- let sets: Vec<roxmltree::Node> = root.children().filter(|n| is(n, "variableSet")).collect();- let (mut text, mut other, mut declared) = (Vec::new(), Vec::new(), BTreeSet::new());- for var in sets.iter().flat_map(|s| s.children()).filter(|n| is(n, "variables")).flat_map(|v| v.children()).filter(|n| is(n, "variable")) {- let name = named(var.attribute("varName"), "varName", cmd)?;- if !declared.insert(name.clone()) {- return Err(bad(cmd, format!("variable `{name}` is declared twice")));- }- if declared.len() > MAX_LIBRARY_VARIABLES {- return Err(bad(cmd, format!("the variable library declares more than {MAX_LIBRARY_VARIABLES} variables")));- }- match var.attribute("trait") {- Some("textcontent") => text.push(name),- Some(t) => other.push((name, t.chars().take(MAX_NAME).collect())),- None => return Err(bad(cmd, format!("variable `{name}` has no `trait`"))),- }- }- if declared.is_empty() {- return Err(bad(cmd, "the variable library declares no variables"));- }- let text_names: BTreeSet<&str> = text.iter().map(String::as_str).collect();- let (mut datasets, mut seen) = (Vec::new(), BTreeSet::new());- for row in- sets.iter().flat_map(|s| s.children()).filter(|n| is(n, "sampleDataSets")).flat_map(|d| d.children()).filter(|n| is(n, "sampleDataSet"))- {- let name = named(row.attribute("dataSetName"), "dataSetName", cmd)?;- if !seen.insert(name.clone()) {- return Err(bad(cmd, format!("data set `{name}` is declared twice")));- }- if seen.len() > MAX_LIBRARY_DATASETS {- return Err(bad(cmd, format!("the variable library declares more than {MAX_LIBRARY_DATASETS} data sets")));- }- let (mut values, mut held) = (BTreeMap::new(), BTreeSet::new());- for el in row.children().filter(roxmltree::Node::is_element) {- let var = el.tag_name().name();- if !declared.contains(var) {- return Err(bad(cmd, format!("data set `{name}` holds a value for `{var}`, which the library doesn't declare")));- }- if !held.insert(var) {- return Err(bad(cmd, format!("data set `{name}` holds two values for `{var}`")));- }- if !text_names.contains(var) {- continue;- }- let paras: Vec<String> = el- .children()- .filter(|n| is(n, "p"))- .map(|p| p.descendants().filter(roxmltree::Node::is_text).filter_map(|t| t.text()).collect())- .collect();- if !paras.is_empty() {- values.insert(var.to_string(), DataValue::Text(paras.join("\n")));- }- }- datasets.push(DataSet { name, values });- }- Ok(Library { text, other, datasets })-}--/// Load Variable Library: replace the document's variables and data sets with a variable library's-/// ([`parse_library`]) in one undo step. Every variable of the document is deleted with its bindings-/// and values, and every data set is deleted. The library's text variables are then defined,-/// unbound, and its data sets are added. The library's variables of other kinds are left out, named-/// in `skipped` and in a warning.-fn load_library(s: &mut Session, p: &Value) -> Result<Value> {- const C: &str = "variable.loadLibrary";- let bytes = library_bytes(p, C)?;- let xml = std::str::from_utf8(bytes.strip_prefix(b"\xEF\xBB\xBF").unwrap_or(&bytes)).map_err(|_| bad(C, "the variable library is not UTF-8"))?;- let lib = parse_library(xml, C)?;- let vars = &s.doc()?.doc.variables;- let deleted: Vec<String> = vars.variables.iter().map(|v| v.name.clone()).collect();- let deleted_sets: Vec<String> = vars.datasets.iter().map(|d| d.name.clone()).collect();- // Lock Variables refuses New Variable and Delete Variable, so the load is refused when it would- // delete or define a variable.- if !deleted.is_empty() || !lib.text.is_empty() {- variables_unlocked(s).map_err(|why| EngineError::Disabled(C.into(), why))?;- }- let added: Vec<Variable> = lib.text.iter().map(|n| Variable { name: n.clone(), kind: VariableKind::Text }).collect();- let names: Vec<String> = lib.datasets.iter().map(|d| d.name.clone()).collect();- s.edit("Load Variable Library", |d, _| {- d.variables.prune(&deleted);- d.variables.variables.extend(added);- d.variables.datasets = lib.datasets;- d.variables.active_dataset = None;- Ok(())- })?;- // A highlight of a variable that is gone highlights nothing.- let st = s.doc_mut()?;- if st.variables_highlight.as_ref().is_some_and(|h| st.doc.variables.variable(h).is_none()) {- st.variables_highlight = None;- }- st.revision += 1;- let skipped: Vec<Value> = lib.other.iter().map(|(name, t)| json!({"name": name, "trait": t})).collect();- let mut out = json!({- "variables": lib.text, "deleted": deleted, "deletedDataSets": deleted_sets, "names": names, "skipped": skipped,- });- // Each `trait` once, in the order the library first uses it, with how many variables have it.- let mut kinds: Vec<(&str, usize)> = Vec::new();- for (_, t) in &lib.other {- match kinds.iter_mut().find(|(k, _)| k == t) {- Some((_, n)) => *n += 1,- None => kinds.push((t, 1)),- }- }- if !kinds.is_empty() {- let kinds = kinds.iter().map(|(k, n)| format!("{k} ({n})")).collect::<Vec<_>>().join(", ");- out["warning"] = json!(format!("variables that aren't text were left out: {kinds}"));- }- Ok(out)-}-pub fn specs() -> Vec<CommandSpec> {vec![cmd!(diff --git a/crates/engine/src/cmd/variablecmds/csv.rs b/crates/engine/src/cmd/variablecmds/csv.rsnew file mode 100644index 000000000..6cb10284b--- /dev/null+++ b/crates/engine/src/cmd/variablecmds/csv.rs@@ -0,0 +1,351 @@+use super::*;++/// Cap on a graph variable's CSV (MCP and files are untrusted).+pub(super) const MAX_GRAPH_CSV: usize = 256 * 1024;++/// Caps on a datasets CSV (paths and MCP payloads are untrusted).+const MAX_CSV_BYTES: usize = 1_048_576;++const MAX_CSV_ROWS: usize = 1_000;++const MAX_CSV_COLS: usize = 256;++/// Bytes of a datasets CSV from `path`, `data`, `dataBase64` or `csv`.+fn datasets_csv_bytes(p: &Value, cmd: &str) -> Result<Vec<u8>> {+ let bytes = match (str_param(p, "path"), str_param(p, "data"), str_param(p, "dataBase64"), str_param(p, "csv")) {+ // One byte past the cap is enough to refuse the file below.+ (Some(path), ..) => read_file_head(&absolute_path(path), MAX_CSV_BYTES + 1)?,+ (None, Some(text), ..) => text.as_bytes().to_vec(),+ (None, None, Some(b64), _) => vectorcraft_format::base64_decode(b64).ok_or_else(|| bad(cmd, "bad dataBase64"))?,+ (None, None, None, Some(csv)) => csv.as_bytes().to_vec(),+ _ => return Err(bad(cmd, "give `path`, `data`, `dataBase64` or `csv`")),+ };+ if bytes.len() > MAX_CSV_BYTES {+ return Err(bad(cmd, format!("CSV is larger than {MAX_CSV_BYTES} bytes")));+ }+ Ok(bytes)+}++/// A row of a datasets CSV: the file line it starts on, and its cells.+type CsvRow = (usize, Vec<String>);++/// Whether a datasets CSV separates its cells with tabs: its header row (the first line that+/// isn't blank), read with tabs between cells, has a tab outside quotes and no comma outside+/// quotes.+fn tab_separated(text: &str) -> bool {+ // `start`: nothing but `\r` read yet in this cell. `blank`: no cell on this line holds+ // anything but white space yet.+ let (mut in_quotes, mut start, mut blank, mut tab) = (false, true, true, false);+ let mut chars = text.chars().peekable();+ while let Some(c) = chars.next() {+ match c {+ '"' if in_quotes => {+ if chars.peek() == Some(&'"') {+ chars.next();+ blank = false;+ } else {+ in_quotes = false;+ }+ }+ '"' if start => in_quotes = true,+ c if in_quotes => blank &= c.is_whitespace(),+ ',' => return false,+ '\t' => {+ tab = true;+ start = true;+ continue;+ }+ '\r' => continue,+ '\n' if blank => {+ tab = false;+ start = true;+ continue;+ }+ '\n' => return tab,+ c => blank &= c.is_whitespace(),+ }+ start = false;+ }+ tab+}++/// Parse a datasets CSV into rows of cells (no header interpretation yet). Cells are separated+/// by commas, or by tabs when [`tab_separated`] says so. `"` quotes a cell (doubled inside),+/// which may then hold separators and line breaks, as [`csv_escape`] writes them. An unquoted+/// cell is trimmed. A quoted cell keeps what its quotes hold as written, spaces included, and+/// drops white space after the closing quote at the end of the cell. Blank lines are skipped.+fn parse_datasets_csv(text: &str, cmd: &str) -> Result<Vec<CsvRow>> {+ let text = text.strip_prefix('\u{feff}').unwrap_or(text);+ let sep = if tab_separated(text) { '\t' } else { ',' };+ let (mut rows, mut row, mut cur) = (Vec::new(), Vec::<String>::new(), String::new());+ // `closed`: where the closing quote of a quoted cell falls in `cur`. `start`: the line the+ // current row starts on.+ let (mut in_quotes, mut closed, mut line, mut start) = (false, None, 1usize, 1usize);+ let mut chars = text.chars().peekable();+ loop {+ let c = chars.next();+ match c {+ Some('"') if in_quotes => {+ if chars.peek() == Some(&'"') {+ cur.push('"');+ chars.next();+ } else {+ in_quotes = false;+ closed = Some(cur.len());+ }+ }+ Some('"') if cur.is_empty() => in_quotes = true,+ Some(ch) if ch == sep && !in_quotes => row.push(end_csv_cell(&mut cur, &mut closed)),+ Some('\r') if !in_quotes => {}+ Some('\n') | None if !in_quotes => {+ row.push(end_csv_cell(&mut cur, &mut closed));+ if row.iter().any(|c| !c.trim().is_empty()) {+ rows.push((start, std::mem::take(&mut row)));+ // +1 for the header.+ if rows.len() > MAX_CSV_ROWS + 1 {+ return Err(bad(cmd, format!("more than {MAX_CSV_ROWS} data rows")));+ }+ }+ row.clear();+ if c.is_none() {+ return Ok(rows);+ }+ line += 1;+ start = line;+ }+ None => return Err(bad(cmd, format!("line {line}: unclosed quote"))),+ Some(c) => {+ line += usize::from(c == '\n');+ cur.push(c);+ }+ }+ if row.len() > MAX_CSV_COLS {+ return Err(bad(cmd, format!("line {line}: more than {MAX_CSV_COLS} columns")));+ }+ }+}++/// Take the cell read so far out of `cur`. Without a closing quote (`closed` is `None`) the cell+/// is trimmed; with one, the text before it is kept as written and the text after it is+/// trimmed at the end.+fn end_csv_cell(cur: &mut String, closed: &mut Option<usize>) -> String {+ let cell = std::mem::take(cur);+ match closed.take() {+ Some(at) => match cell.split_at_checked(at) {+ Some((quoted, after)) => format!("{quoted}{}", after.trim_end()),+ None => cell,+ },+ None => cell.trim().to_string(),+ }+}++fn csv_escape(s: &str) -> String {+ if s.contains([',', '"', '\n', '\r']) || s != s.trim() { format!("\"{}\"", s.replace('"', "\"\"")) } else { s.to_string() }+}++/// A visibility cell: true/false, 1/0, yes/no (ASCII, case-insensitive).+fn parse_visible_cell(s: &str, var: &str, cmd: &str) -> Result<bool> {+ match s.trim().to_ascii_lowercase().as_str() {+ "true" | "1" | "yes" => Ok(true),+ "false" | "0" | "no" => Ok(false),+ _ => Err(bad(cmd, format!("value for `{var}` must be true or false"))),+ }+}++fn cell_to_value(kind: VariableKind, cell: &str, var: &str, cmd: &str) -> Result<Option<DataValue>> {+ // A text cell is not trimmed here (the parser trims unquoted cells); image, graph and+ // visibility cells are trimmed.+ let t = if kind == VariableKind::Text { cell } else { cell.trim() };+ if t.is_empty() {+ return Ok(None);+ }+ Ok(Some(match kind {+ VariableKind::Text => DataValue::Text(t.to_string()),+ VariableKind::Image => DataValue::Image(t.to_string()),+ VariableKind::Graph if t.len() <= MAX_GRAPH_CSV => DataValue::Graph(t.to_string()),+ VariableKind::Graph => return Err(bad(cmd, format!("value for `{var}` is longer than {MAX_GRAPH_CSV} bytes"))),+ VariableKind::Visibility => DataValue::Visible(parse_visible_cell(t, var, cmd)?),+ }))+}++fn value_to_cell(v: &DataValue) -> String {+ match v {+ DataValue::Text(t) | DataValue::Image(t) | DataValue::Graph(t) => t.clone(),+ DataValue::Visible(b) => if *b { "true" } else { "false" }.to_string(),+ }+}++/// Build the CSV of every dataset (header `name` then variables in definition order).+fn datasets_to_csv(vars: &Variables) -> String {+ let mut out = String::from("name");+ for v in &vars.variables {+ out.push(',');+ out.push_str(&csv_escape(&v.name));+ }+ out.push('\n');+ for ds in &vars.datasets {+ out.push_str(&csv_escape(&ds.name));+ for v in &vars.variables {+ out.push(',');+ if let Some(val) = ds.values.get(&v.name) {+ out.push_str(&csv_escape(&value_to_cell(val)));+ }+ }+ out.push('\n');+ }+ out+}++/// The variable that a datasets CSV header cell (trimmed, not empty) in 1-based `column` names,+/// and its kind. A cell that is the name of a variable in `kinds` or in `defined` (the variables+/// that earlier columns define) names that variable. Otherwise `#` before the name makes it a+/// visibility variable, `@` makes it an image variable, and a cell without a prefix names a text+/// variable. A prefixed name that is already a variable must be one of the prefix's kind; a new+/// name is added to `defined`. A `%` cell that names no variable (graph data files) is refused.+fn header_variable(+ cell: &str,+ column: usize,+ kinds: &BTreeMap<String, VariableKind>,+ defined: &mut Vec<Variable>,+ cmd: &str,+) -> Result<(String, VariableKind)> {+ let known = |name: &str, defined: &[Variable]| kinds.get(name).copied().or_else(|| defined.iter().find(|v| v.name == name).map(|v| v.kind));+ if let Some(kind) = known(cell, defined) {+ return Ok((cell.to_string(), kind));+ }+ let (name, kind) = if let Some(rest) = cell.strip_prefix('#') {+ (rest.trim(), VariableKind::Visibility)+ } else if let Some(rest) = cell.strip_prefix('@') {+ (rest.trim(), VariableKind::Image)+ } else if cell.starts_with('%') {+ return Err(bad(cmd, format!("column {column}: `{cell}` is a `%` column of graph data files, which are not read")));+ } else {+ (cell, VariableKind::Text)+ };+ if name.is_empty() {+ return Err(bad(cmd, format!("column {column}: `{cell}` names no variable")));+ }+ match known(name, defined) {+ Some(k) if k == kind => Ok((name.to_string(), kind)),+ Some(k) => Err(bad(cmd, format!("column {column}: `{cell}` names `{name}`, which is a {} variable", k.label()))),+ None if name.chars().count() > MAX_NAME => Err(bad(cmd, format!("column {column}: the variable name is longer than {MAX_NAME} characters"))),+ None => {+ defined.push(Variable { name: name.to_string(), kind });+ Ok((name.to_string(), kind))+ }+ }+}++/// Import datasets from CSV. A first header cell `name` makes the first column the dataset+/// names; the other headers name variables, as [`header_variable`] reads them, and the+/// variables the document doesn't define yet are defined, unbound. Without the `name` column,+/// every header names a variable and the rows take the first free names "Data Set 1", "Data+/// Set 2" and so on. Existing rows of the same name are replaced; new names are appended. One+/// undo step.+pub(super) fn dataset_import(s: &mut Session, p: &Value) -> Result<Value> {+ const C: &str = "dataset.import";+ let bytes = datasets_csv_bytes(p, C)?;+ let text = std::str::from_utf8(&bytes).map_err(|_| bad(C, "CSV is not UTF-8"))?;+ let rows = parse_datasets_csv(text, C)?;+ let Some((_, header)) = rows.first() else {+ return Err(bad(C, "CSV is empty"));+ };+ let name_column = header.first().is_some_and(|h| h.trim().eq_ignore_ascii_case("name"));+ let (kinds, existing_names) = {+ let st = s.doc()?;+ let kinds: BTreeMap<String, VariableKind> = st.doc.variables.variables.iter().map(|v| (v.name.clone(), v.kind)).collect();+ let existing: BTreeSet<String> = st.doc.variables.datasets.iter().map(|d| d.name.clone()).collect();+ (kinds, existing)+ };+ let mut columns: Vec<(usize, String, VariableKind)> = Vec::new();+ let mut defined: Vec<Variable> = Vec::new();+ for (i, h) in header.iter().enumerate().skip(usize::from(name_column)) {+ let cell = h.trim();+ if cell.is_empty() {+ return Err(bad(C, format!("empty header in column {}", i + 1)));+ }+ let (name, kind) = header_variable(cell, i + 1, &kinds, &mut defined, C)?;+ columns.push((i, name, kind));+ }+ // Lock Variables refuses New Variable, so the import is refused when a header would define one.+ if !defined.is_empty() {+ variables_unlocked(s).map_err(|why| EngineError::Disabled(C.into(), why))?;+ }+ if rows.len() == 1 {+ return Err(bad(C, "CSV has a header but no data rows"));+ }+ let mut parsed: Vec<DataSet> = Vec::new();+ // The names of the rows read so far, and the last number given to a row without a name.+ let (mut seen, mut number) = (BTreeSet::new(), 0usize);+ for &(line, ref row) in rows.iter().skip(1) {+ let raw_name = if name_column { row.first().map_or("", |c| c.trim()) } else { "" };+ let name = if raw_name.is_empty() {+ // Same numbering as Capture Data Set when a row leaves the name blank or the file has+ // no name column. The search starts after the last number given.+ let (n, name) = (number + 1..)+ .map(|n| (n, format!("Data Set {n}")))+ .find(|(_, n)| !seen.contains(n) && !existing_names.contains(n))+ .unwrap_or_else(|| (line, format!("Data Set {line}")));+ number = n;+ name+ } else if raw_name.chars().count() > MAX_NAME {+ return Err(bad(C, format!("line {line}: `name` is longer than {MAX_NAME} characters")));+ } else {+ raw_name.to_string()+ };+ if !seen.insert(name.clone()) {+ return Err(bad(C, format!("line {line}: dataset `{name}` appears twice")));+ }+ let mut values = BTreeMap::new();+ for &(col, ref var, kind) in &columns {+ let cell = row.get(col).map(String::as_str).unwrap_or("");+ match cell_to_value(kind, cell, var, C) {+ Ok(Some(v)) => {+ values.insert(var.clone(), v);+ }+ Ok(None) => {}+ Err(_) => return Err(bad(C, format!("line {line}: value for `{var}` must match its variable kind"))),+ }+ }+ parsed.push(DataSet { name, values });+ }+ let names: Vec<String> = parsed.iter().map(|d| d.name.clone()).collect();+ let new_variables: Vec<String> = defined.iter().map(|v| v.name.clone()).collect();+ let (imported, updated) = s.edit("Import Data Sets", |d, _| {+ d.variables.variables.extend(defined);+ let mut imported = 0usize;+ let mut updated = 0usize;+ for row in parsed {+ if let Some(existing) = d.variables.datasets.iter_mut().find(|x| x.name == row.name) {+ *existing = row;+ updated += 1;+ } else {+ d.variables.datasets.push(row);+ imported += 1;+ }+ }+ Ok((imported, updated))+ })?;+ Ok(json!({"imported": imported, "updated": updated, "names": names, "defined": new_variables}))+}++/// Export every dataset as CSV (`name` then variables). With `path`, write the file; else return `{csv}`.+pub(super) fn dataset_export(s: &mut Session, p: &Value) -> Result<Value> {+ const C: &str = "dataset.export";+ let st = s.doc()?;+ if st.doc.variables.variables.is_empty() {+ return Err(bad(C, "define a variable first"));+ }+ let csv = datasets_to_csv(&st.doc.variables);+ if csv.len() > MAX_CSV_BYTES {+ return Err(bad(C, format!("CSV would be larger than {MAX_CSV_BYTES} bytes")));+ }+ match str_param(p, "path") {+ Some(path) => {+ let path = absolute_path(path);+ write_file(&path, csv.as_bytes())?;+ Ok(json!({"path": path, "datasets": st.doc.variables.datasets.len()}))+ }+ None => Ok(json!({"csv": csv, "datasets": st.doc.variables.datasets.len()})),+ }+}diff --git a/crates/engine/src/cmd/variablecmds/library.rs b/crates/engine/src/cmd/variablecmds/library.rsnew file mode 100644index 000000000..83374a16d--- /dev/null+++ b/crates/engine/src/cmd/variablecmds/library.rs@@ -0,0 +1,298 @@+use super::*;++/// Caps on a variable library (paths and MCP payloads are untrusted): its size, its XML nodes, how+/// deep its elements nest, and the entities, variables and data sets it declares.+const MAX_LIBRARY_BYTES: usize = 8 << 20;++const MAX_LIBRARY_ENTITIES: usize = 64;++const MAX_LIBRARY_NODES: u32 = 2_000_000;++const MAX_LIBRARY_NESTING: usize = 256;++const MAX_LIBRARY_VARIABLES: usize = 256;++const MAX_LIBRARY_DATASETS: usize = 1_000;++/// Bytes of a variable library from `path`, `xml` or `dataBase64`.+fn library_bytes(p: &Value, cmd: &str) -> Result<Vec<u8>> {+ let bytes = match (str_param(p, "path"), str_param(p, "xml"), str_param(p, "dataBase64")) {+ // One byte past the cap is enough to refuse the file below.+ (Some(path), ..) => read_file_head(&absolute_path(path), MAX_LIBRARY_BYTES + 1)?,+ (None, Some(xml), _) => xml.as_bytes().to_vec(),+ (None, None, Some(b64)) => vectorcraft_format::base64_decode(b64).ok_or_else(|| bad(cmd, "bad dataBase64"))?,+ _ => return Err(bad(cmd, "give `path`, `xml` or `dataBase64`")),+ };+ if bytes.len() > MAX_LIBRARY_BYTES {+ return Err(bad(cmd, format!("the variable library is larger than {MAX_LIBRARY_BYTES} bytes")));+ }+ Ok(bytes)+}++/// `xml` without the document type declaration of its prolog, with each reference to a general+/// entity that the declaration defines replaced by the entity's value, its quotes and `>` escaped,+/// outside CDATA sections, comments and processing instructions. The declaration may hold an+/// external identifier and an internal subset of white space, comments, processing instructions and+/// up to [`MAX_LIBRARY_ENTITIES`] entity declarations, each with a name of at most [`MAX_NAME`]+/// bytes and a quoted value that holds no `<`, `&` or `%`. Anything else in it is refused, and so is+/// a text that the replacements make longer than [`MAX_LIBRARY_BYTES`]. The XML parser then refuses+/// any other document type declaration, so [`nesting`](super::place::docx::nesting) counts the+/// elements the parser reads.+fn without_doctype(xml: &str, cmd: &str) -> Result<String> {+ let b = xml.as_bytes();+ let past = |from: usize, end: &str| xml.get(from..).and_then(|rest| rest.find(end)).map(|at| from + at + end.len());+ let spaces = |from: usize| from + b.get(from..).map_or(0, |rest| rest.iter().take_while(|c| c.is_ascii_whitespace()).count());+ let at = |i: usize, s: &str| xml.get(i..).is_some_and(|rest| rest.starts_with(s));+ let refused = || bad(cmd, "the variable library's document type declaration holds something other than entities of plain text");+ // The prolog: an XML declaration, white space, comments and processing instructions.+ let mut i = 0;+ loop {+ i = spaces(i);+ if at(i, "<?") {+ i = past(i + 2, "?>").ok_or_else(refused)?;+ } else if at(i, "<!--") {+ i = past(i + 4, "-->").ok_or_else(refused)?;+ } else {+ break;+ }+ }+ if !at(i, "<!DOCTYPE") {+ return Ok(xml.to_string());+ }+ let start = i;+ // A quoted literal starting at `i` → the index past it and its text.+ let literal = |i: usize| -> Option<(usize, &str)> {+ let quote = *b.get(i).filter(|c| matches!(c, b'"' | b'\''))?;+ let len = b.get(i + 1..)?.iter().position(|c| *c == quote)?;+ Some((i + len + 2, xml.get(i + 1..i + 1 + len)?))+ };+ // `<!DOCTYPE` S Name (S ExternalID)? S? ('[' intSubset ']' S?)? '>'+ i = spaces(i + "<!DOCTYPE".len());+ i += b.get(i..).map_or(0, |rest| rest.iter().take_while(|c| !c.is_ascii_whitespace() && !matches!(c, b'[' | b'>')).count());+ i = spaces(i);+ for (id, literals) in [("SYSTEM", 1), ("PUBLIC", 2)] {+ if at(i, id) {+ i += id.len();+ for _ in 0..literals {+ i = literal(spaces(i)).ok_or_else(refused)?.0;+ }+ i = spaces(i);+ }+ }+ let mut entities: Vec<(&str, String)> = Vec::new();+ if at(i, "[") {+ i += 1;+ loop {+ i = spaces(i);+ if at(i, "]") {+ i = spaces(i + 1);+ break;+ } else if at(i, "<!--") {+ i = past(i + 4, "-->").ok_or_else(refused)?;+ } else if at(i, "<?") {+ i = past(i + 2, "?>").ok_or_else(refused)?;+ } else if at(i, "<!ENTITY") {+ i = spaces(i + "<!ENTITY".len());+ let len =+ b.get(i..).map_or(0, |rest| rest.iter().take_while(|c| !c.is_ascii_whitespace() && !matches!(c, b'"' | b'\'' | b'>')).count());+ let name = xml.get(i..i + len).filter(|n| !n.is_empty() && n.len() <= MAX_NAME && !n.starts_with('%')).ok_or_else(refused)?;+ let (end, value) = literal(spaces(i + len)).ok_or_else(refused)?;+ if value.contains(['<', '&', '%']) {+ return Err(refused());+ }+ i = spaces(end);+ if !at(i, ">") {+ return Err(refused());+ }+ i += 1;+ if entities.len() == MAX_LIBRARY_ENTITIES {+ return Err(bad(cmd, format!("the variable library declares more than {MAX_LIBRARY_ENTITIES} entities")));+ }+ entities.push((name, value.replace('"', """).replace('\'', "'").replace('>', ">")));+ } else {+ return Err(refused());+ }+ }+ }+ if !at(i, ">") {+ return Err(refused());+ }+ let mut out = String::with_capacity(xml.len());+ out.push_str(xml.get(..start).unwrap_or_default());+ // The rest, copied with the references replaced outside CDATA sections, comments and+ // processing instructions, where the parser reads none.+ let mut i = i + 1;+ while let Some(c) = xml.get(i..).and_then(|rest| rest.chars().next()) {+ let verbatim = [("<![CDATA[", "]]>"), ("<!--", "-->"), ("<?", "?>")].into_iter().find(|(open, _)| at(i, open));+ if let Some((open, close)) = verbatim {+ let end = past(i + open.len(), close).unwrap_or(xml.len());+ out.push_str(xml.get(i..end).unwrap_or_default());+ i = end;+ continue;+ }+ // Entity names are at most `MAX_NAME` bytes long, so the `;` is looked for that far.+ let entity = if c == '&' {+ let name = xml.get(i + 1..).and_then(|rest| rest.get(..rest.bytes().take(MAX_NAME + 1).position(|b| b == b';')?));+ name.and_then(|name| entities.iter().find(|(n, _)| *n == name))+ } else {+ None+ };+ match entity {+ Some((name, value)) => {+ out.push_str(value);+ i += name.len() + 2;+ if out.len() > MAX_LIBRARY_BYTES {+ return Err(bad(cmd, format!("the variable library's entities expand to more than {MAX_LIBRARY_BYTES} bytes")));+ }+ }+ None => {+ out.push(c);+ i += c.len_utf8();+ }+ }+ }+ if out.len() > MAX_LIBRARY_BYTES {+ return Err(bad(cmd, format!("the variable library's entities expand to more than {MAX_LIBRARY_BYTES} bytes")));+ }+ Ok(out)+}++/// A variable library as Load Variable Library reads it: the names of its text variables in the+/// order it declares them, the name and `trait` of each variable of another kind, and its data+/// sets with their text values.+struct Library {+ text: Vec<String>,+ other: Vec<(String, String)>,+ datasets: Vec<DataSet>,+}++/// Read a variable library: XML whose `variableSets` element holds `variableSet` elements, each+/// with a `variables` element of `variable` elements (`varName`, `trait`) and a `sampleDataSets`+/// element of `sampleDataSet` elements (`dataSetName`). Elements are matched by their local names.+/// A variable whose `trait` is `textcontent` is a text variable. In a data set, an element named+/// after a text variable gives its value: the text of its `p` elements, one paragraph each, and+/// none when it has no `p` element. Values for variables of other kinds are left out. A variable+/// or data set declared twice, a value for a variable the library doesn't declare, and a library+/// past the caps are refused. Entities that its document type declaration defines are read as+/// [`without_doctype`] says.+fn parse_library(xml: &str, cmd: &str) -> Result<Library> {+ let xml = without_doctype(xml, cmd)?;+ // roxmltree parses nested elements recursively.+ if crate::cmd::place::docx::nesting(&xml) > MAX_LIBRARY_NESTING {+ return Err(bad(cmd, format!("the variable library nests elements more than {MAX_LIBRARY_NESTING} deep")));+ }+ let opts = roxmltree::ParsingOptions { nodes_limit: MAX_LIBRARY_NODES, ..roxmltree::ParsingOptions::default() };+ let doc =+ roxmltree::Document::parse_with_options(&xml, opts).map_err(|e| bad(cmd, format!("the variable library is not well-formed XML: {e}")))?;+ let is = |n: &roxmltree::Node, name: &str| n.is_element() && n.tag_name().name() == name;+ let Some(root) = doc.descendants().find(|n| is(n, "variableSets")) else {+ return Err(bad(cmd, "the file holds no `variableSets` element"));+ };+ let sets: Vec<roxmltree::Node> = root.children().filter(|n| is(n, "variableSet")).collect();+ let (mut text, mut other, mut declared) = (Vec::new(), Vec::new(), BTreeSet::new());+ for var in sets.iter().flat_map(|s| s.children()).filter(|n| is(n, "variables")).flat_map(|v| v.children()).filter(|n| is(n, "variable")) {+ let name = named(var.attribute("varName"), "varName", cmd)?;+ if !declared.insert(name.clone()) {+ return Err(bad(cmd, format!("variable `{name}` is declared twice")));+ }+ if declared.len() > MAX_LIBRARY_VARIABLES {+ return Err(bad(cmd, format!("the variable library declares more than {MAX_LIBRARY_VARIABLES} variables")));+ }+ match var.attribute("trait") {+ Some("textcontent") => text.push(name),+ Some(t) => other.push((name, t.chars().take(MAX_NAME).collect())),+ None => return Err(bad(cmd, format!("variable `{name}` has no `trait`"))),+ }+ }+ if declared.is_empty() {+ return Err(bad(cmd, "the variable library declares no variables"));+ }+ let text_names: BTreeSet<&str> = text.iter().map(String::as_str).collect();+ let (mut datasets, mut seen) = (Vec::new(), BTreeSet::new());+ for row in+ sets.iter().flat_map(|s| s.children()).filter(|n| is(n, "sampleDataSets")).flat_map(|d| d.children()).filter(|n| is(n, "sampleDataSet"))+ {+ let name = named(row.attribute("dataSetName"), "dataSetName", cmd)?;+ if !seen.insert(name.clone()) {+ return Err(bad(cmd, format!("data set `{name}` is declared twice")));+ }+ if seen.len() > MAX_LIBRARY_DATASETS {+ return Err(bad(cmd, format!("the variable library declares more than {MAX_LIBRARY_DATASETS} data sets")));+ }+ let (mut values, mut held) = (BTreeMap::new(), BTreeSet::new());+ for el in row.children().filter(roxmltree::Node::is_element) {+ let var = el.tag_name().name();+ if !declared.contains(var) {+ return Err(bad(cmd, format!("data set `{name}` holds a value for `{var}`, which the library doesn't declare")));+ }+ if !held.insert(var) {+ return Err(bad(cmd, format!("data set `{name}` holds two values for `{var}`")));+ }+ if !text_names.contains(var) {+ continue;+ }+ let paras: Vec<String> = el+ .children()+ .filter(|n| is(n, "p"))+ .map(|p| p.descendants().filter(roxmltree::Node::is_text).filter_map(|t| t.text()).collect())+ .collect();+ if !paras.is_empty() {+ values.insert(var.to_string(), DataValue::Text(paras.join("\n")));+ }+ }+ datasets.push(DataSet { name, values });+ }+ Ok(Library { text, other, datasets })+}++/// Load Variable Library: replace the document's variables and data sets with a variable library's+/// ([`parse_library`]) in one undo step. Every variable of the document is deleted with its bindings+/// and values, and every data set is deleted. The library's text variables are then defined,+/// unbound, and its data sets are added. The library's variables of other kinds are left out, named+/// in `skipped` and in a warning.+pub(super) fn load_library(s: &mut Session, p: &Value) -> Result<Value> {+ const C: &str = "variable.loadLibrary";+ let bytes = library_bytes(p, C)?;+ let xml = std::str::from_utf8(bytes.strip_prefix(b"\xEF\xBB\xBF").unwrap_or(&bytes)).map_err(|_| bad(C, "the variable library is not UTF-8"))?;+ let lib = parse_library(xml, C)?;+ let vars = &s.doc()?.doc.variables;+ let deleted: Vec<String> = vars.variables.iter().map(|v| v.name.clone()).collect();+ let deleted_sets: Vec<String> = vars.datasets.iter().map(|d| d.name.clone()).collect();+ // Lock Variables refuses New Variable and Delete Variable, so the load is refused when it would+ // delete or define a variable.+ if !deleted.is_empty() || !lib.text.is_empty() {+ variables_unlocked(s).map_err(|why| EngineError::Disabled(C.into(), why))?;+ }+ let added: Vec<Variable> = lib.text.iter().map(|n| Variable { name: n.clone(), kind: VariableKind::Text }).collect();+ let names: Vec<String> = lib.datasets.iter().map(|d| d.name.clone()).collect();+ s.edit("Load Variable Library", |d, _| {+ d.variables.prune(&deleted);+ d.variables.variables.extend(added);+ d.variables.datasets = lib.datasets;+ d.variables.active_dataset = None;+ Ok(())+ })?;+ // A highlight of a variable that is gone highlights nothing.+ let st = s.doc_mut()?;+ if st.variables_highlight.as_ref().is_some_and(|h| st.doc.variables.variable(h).is_none()) {+ st.variables_highlight = None;+ }+ st.revision += 1;+ let skipped: Vec<Value> = lib.other.iter().map(|(name, t)| json!({"name": name, "trait": t})).collect();+ let mut out = json!({+ "variables": lib.text, "deleted": deleted, "deletedDataSets": deleted_sets, "names": names, "skipped": skipped,+ });+ // Each `trait` once, in the order the library first uses it, with how many variables have it.+ let mut kinds: Vec<(&str, usize)> = Vec::new();+ for (_, t) in &lib.other {+ match kinds.iter_mut().find(|(k, _)| k == t) {+ Some((_, n)) => *n += 1,+ None => kinds.push((t, 1)),+ }+ }+ if !kinds.is_empty() {+ let kinds = kinds.iter().map(|(k, n)| format!("{k} ({n})")).collect::<Vec<_>>().join(", ");+ out["warning"] = json!(format!("variables that aren't text were left out: {kinds}"));+ }+ Ok(out)+}