Skip to content

ProCraft for Android: An Android native painting app written in Rust

Rust

feat(import): Clip Studio sub tools as brushes, read with our own SQLite reader

Josh4628d9794013
diff --git a/apps/procraft-android/android/app/src/main/java/io/github/artcraftandroid/procraft/canvas/CanvasViewModel.kt b/apps/procraft-android/android/app/src/main/java/io/github/artcraftandroid/procraft/canvas/CanvasViewModel.kt
index 773e5a1..d03ab4e 100644
--- a/apps/procraft-android/android/app/src/main/java/io/github/artcraftandroid/procraft/canvas/CanvasViewModel.kt
+++ b/apps/procraft-android/android/app/src/main/java/io/github/artcraftandroid/procraft/canvas/CanvasViewModel.kt
@@ -5863,8 +5863,8 @@ class CanvasViewModel(app: Application) : AndroidViewModel(app) {
/** Brush and brush set files, as other painting apps share them. */
const val BRUSH_EXTENSION = "brush"
const val BRUSH_SET_EXTENSION = "brushset"
- /** Other apps' brush files: Photoshop's, GIMP's brushes and pipes, Krita's presets and bundles. */
- val OTHER_BRUSH_EXTENSIONS = listOf("abr", "gbr", "gih", "kpp", "bundle")
+ /** Other apps' brush files: Photoshop's, GIMP's brushes and pipes, Krita's presets and bundles, Clip Studio's sub tools. */
+ val OTHER_BRUSH_EXTENSIONS = listOf("abr", "gbr", "gih", "kpp", "bundle", "sut")
val PACKAGE_LABEL = PACKAGE_EXTENSION.replaceFirstChar(Char::titlecase)
/** What the gallery's Share offers, in order: still pictures, then animations. */
diff --git a/apps/procraft-android/src/brush_files.rs b/apps/procraft-android/src/brush_files.rs
index 663ed6e..51e5277 100644
--- a/apps/procraft-android/src/brush_files.rs
+++ b/apps/procraft-android/src/brush_files.rs
@@ -108,7 +108,7 @@ impl CanvasSession {
}
/// Adds the brushes in another app's file (`bytes`): a Photoshop brush file's sampled tips, a
- /// GIMP brush or pipe, or a Krita preset or bundle. They go in a set of their own named `name`
+ /// GIMP brush or pipe, a Krita preset or bundle, or a Clip Studio sub tool. They go in a set of their own named `name`
/// (a bundle's own name first; else "Imported") at the top of the library, each with the
/// file's shape and spacing. Returns the set, or none if the file holds none.
pub fn import_other_brushes(&self, bytes: Vec<u8>, name: String) -> Option<u32> {
@@ -213,11 +213,39 @@ fn other_brushes(bytes: &[u8], name: String) -> Option<(String, Vec<Brush>)> {
let name = if title.trim().is_empty() { name } else { title };
return Some((name, presets.iter().map(krita_brush).collect()));
}
+ if let Some(brushes) = procraft_engine::csp::read(bytes) {
+ return Some((name, brushes.iter().map(csp_brush).collect()));
+ }
// a preset alone has no images of its own: one it names comes round
let preset = krita::read_preset(bytes, &|_| None)?;
Some((name, vec![krita_brush(&preset)]))
}
+/// A brush of a Clip Studio sub tool: its tip (an image, else round with its edge and roundness),
+/// spacing and angle.
+fn csp_brush(c: &procraft_engine::csp::CspBrush) -> Brush {
+ // its tips paint white, or else dark
+ let tip = c
+ .tip_png
+ .as_deref()
+ .and_then(|png| png_tip(png, true).filter(|t| t.pixels.iter().any(|&v| v > 0)).or_else(|| png_tip(png, false)));
+ let mut b = match tip {
+ Some(t) => tip_brush(&procraft_engine::abr::AbrTip { name: c.name.clone(), ..t }),
+ None => Brush { name: c.name.clone(), ..Brush::round_pen() },
+ };
+ if let Some(h) = c.hardness {
+ b.hardness = h.clamp(0.0, 1.0);
+ }
+ if let Some(r) = c.roundness {
+ b.shape.roundness = r.clamp(0.01, 1.0);
+ }
+ if let Some(spacing) = c.spacing {
+ b.spacing = spacing.clamp(0.01, 10.0);
+ }
+ b.shape.rotation = c.angle;
+ b
+}
+
/// A brush of a sampled tip: its shape, with its spacing.
fn tip_brush(t: &procraft_engine::abr::AbrTip) -> Brush {
let mut b = Brush { name: t.name.clone(), ..Brush::round_pen() };
@@ -236,7 +264,7 @@ fn krita_brush(k: &procraft_engine::krita::KritaBrush) -> Brush {
let image = match &k.tip {
KritaTip::Round { .. } => None,
KritaTip::Image(t) => Some(fitted(t)),
- KritaTip::Png(png) => png_tip(png).map(|t| fitted(&t)),
+ KritaTip::Png(png) => png_tip(png, false).map(|t| fitted(&t)),
};
let mut b = match image {
Some(pixels) => {
@@ -264,8 +292,8 @@ fn krita_brush(k: &procraft_engine::krita::KritaBrush) -> Brush {
b
}
-/// A PNG tip as a grey one: dark, opaque pixels paint.
-fn png_tip(png: &[u8]) -> Option<procraft_engine::abr::AbrTip> {
+/// A PNG tip as a grey one: its opaque pixels paint, the light ones if `white`, else the dark.
+fn png_tip(png: &[u8], white: bool) -> Option<procraft_engine::abr::AbrTip> {
let (w, h, channels, buf) = decode_png(png)?;
let pixels: Vec<u8> = buf
.chunks_exact(channels.max(1))
@@ -278,7 +306,8 @@ fn png_tip(png: &[u8]) -> Option<procraft_engine::abr::AbrTip> {
3 => ((get(0) * 299 + get(1) * 587 + get(2) * 114) / 1000, 255),
_ => ((get(0) * 299 + get(1) * 587 + get(2) * 114) / 1000, get(3)),
};
- ((255 - grey.min(255)) * alpha / 255) as u8
+ let ink = if white { grey.min(255) } else { 255 - grey.min(255) };
+ (ink * alpha / 255) as u8
})
.collect();
(pixels.len() == w * h).then(|| procraft_engine::abr::AbrTip {
diff --git a/apps/procraft-android/src/tests.rs b/apps/procraft-android/src/tests.rs
index 2c49c96..7111fd8 100644
--- a/apps/procraft-android/src/tests.rs
+++ b/apps/procraft-android/src/tests.rs
@@ -2826,7 +2826,8 @@ fn a_photoshop_brush_files_tips_become_a_set() {
}
/// Real brush files, when a folder of them is given (`BRUSH_CORPUS=folder`, never committed):
-/// every `.brush`, `.brushset` and `.abr` in it and below imports, each brush draws in its
+/// every brush file in it and below (ours, and Photoshop's, GIMP's, Krita's and Clip Studio's)
+/// imports, each brush draws in its
/// preview, and a brush written out, read back and written out again comes out the same.
#[test]
fn a_corpus_of_real_brush_files_imports_draws_and_round_trips() {
@@ -2841,7 +2842,10 @@ fn a_corpus_of_real_brush_files_imports_draws_and_round_trips() {
let path = entry.path();
if path.is_dir() {
folders.push(path);
- } else if matches!(extension(&path).as_str(), "brush" | "brushset" | "abr") {
+ } else if matches!(
+ extension(&path).as_str(),
+ "brush" | "brushset" | "abr" | "gbr" | "gih" | "kpp" | "bundle" | "sut"
+ ) {
files.push(path);
}
}
@@ -2856,8 +2860,11 @@ fn a_corpus_of_real_brush_files_imports_draws_and_round_trips() {
let Some(s) = session(64, 64) else { return };
let bytes = std::fs::read(path).unwrap();
let kind = extension(path);
- let set =
- if kind == "abr" { s.import_other_brushes(bytes, "Corpus".to_owned()) } else { s.import_brushes(bytes) };
+ let set = if kind == "brush" || kind == "brushset" {
+ s.import_brushes(bytes)
+ } else {
+ s.import_other_brushes(bytes, "Corpus".to_owned())
+ };
let Some(set) = set else {
failures.push(format!("{}: not imported", path.display()));
continue;
diff --git a/crates/engine/src/csp.rs b/crates/engine/src/csp.rs
new file mode 100644
index 0000000..99b4855
--- /dev/null
+++ b/crates/engine/src/csp.rs
@@ -0,0 +1,216 @@
+//! Reading Clip Studio sub tools (`.sut`): a SQLite database whose `Node` table names the brushes,
+//! each pointing (`NodeVariantID`) at its settings in `Variant`, and whose `MaterialFile` table
+//! holds the tip images (PNGs, often inside a tar) that the settings name by key.
+
+use crate::sqlite::{self, Table, Value};
+
+/// A brush as a sub tool has it: its settings where it has them.
+#[derive(Clone, Debug, PartialEq)]
+pub struct CspBrush {
+ pub name: String,
+ /// The edge, 0 soft to 1 hard.
+ pub hardness: Option<f32>,
+ /// Space between stamps as a share of the size.
+ pub spacing: Option<f32>,
+ /// How round the tip is, 1 a circle.
+ pub roundness: Option<f32>,
+ /// The tip's angle, radians.
+ pub angle: f32,
+ /// The tip's image (a PNG), if it isn't round.
+ pub tip_png: Option<Vec<u8>>,
+}
+
+/// The most brushes read from one file, and PNGs looked at in a material.
+const MOST_BRUSHES: usize = 256;
+const MOST_PNGS: usize = 32;
+const PNG: [u8; 8] = [0x89, b'P', b'N', b'G', 0x0D, 0x0A, 0x1A, 0x0A];
+
+/// The brushes in a sub tool file, in its order; none if it isn't one.
+pub fn read(bytes: &[u8]) -> Option<Vec<CspBrush>> {
+ let db = sqlite::open(bytes)?;
+ let nodes = db.table("Node")?;
+ let variants = db.table("Variant");
+ let materials = db.table("MaterialFile");
+ let named = |row: &Vec<Value>| {
+ nodes.get(row, "NodeName").and_then(Value::as_text).map(str::trim).filter(|n| !n.is_empty()).map(str::to_owned)
+ };
+ // the nodes with settings of their own (a folder of sub tools has none)
+ let mut out = Vec::new();
+ for row in &nodes.rows {
+ let Some(name) = named(row) else { continue };
+ let id = nodes.get(row, "NodeVariantID").and_then(Value::as_int);
+ let variant = variants.as_ref().and_then(|v| {
+ id.and_then(|id| v.rows.iter().find(|r| v.get(r, "VariantID").and_then(Value::as_int) == Some(id)))
+ .map(|r| (v, r))
+ });
+ if variant.is_some() {
+ out.push(brush(&name, variant, materials.as_ref()));
+ }
+ if out.len() >= MOST_BRUSHES {
+ break;
+ }
+ }
+ // none: the first named node, round
+ if out.is_empty() {
+ out.extend(nodes.rows.iter().find_map(named).map(|name| brush(&name, None, materials.as_ref())));
+ }
+ (!out.is_empty()).then_some(out)
+}
+
+fn brush(name: &str, variant: Option<(&Table, &Vec<Value>)>, materials: Option<&Table>) -> CspBrush {
+ let number =
+ |key: &str| variant.and_then(|(t, r)| t.get(r, key)).and_then(Value::as_real).filter(|v| v.is_finite());
+ let pattern =
+ variant.and_then(|(t, r)| t.get(r, "BrushPatternImageArray")).and_then(Value::as_blob).and_then(pattern_key);
+ CspBrush {
+ name: name.to_owned(),
+ hardness: number("BrushHardness").filter(|v| (0.0..=100.0).contains(v)).map(|v| (v / 100.0) as f32),
+ spacing: number("BrushInterval").filter(|v| *v > 0.0).map(|v| (v / 100.0) as f32),
+ roundness: number("BrushThickness").filter(|v| (1.0..=100.0).contains(v)).map(|v| (v / 100.0) as f32),
+ angle: number("BrushRotation").map_or(0.0, |v| v.to_radians() as f32),
+ tip_png: pattern.and_then(|key| materials.and_then(|m| material_png(m, &key))),
+ }
+}
+
+/// The key a pattern list names its first image by: a UTF-16 string with `:data:` in it.
+fn pattern_key(blob: &[u8]) -> Option<String> {
+ let mut current = String::new();
+ let mut found = Vec::new();
+ for pair in blob.as_chunks::<2>().0 {
+ match char::from_u32(u32::from(u16::from_le_bytes(*pair))).filter(|c| c.is_ascii_graphic()) {
+ Some(c) => current.push(c),
+ None => found.push(std::mem::take(&mut current)),
+ }
+ }
+ found.push(current);
+ found.into_iter().find(|s| s.contains(":data:") && s.len() >= 8)
+}
+
+/// The largest PNG in the material `key` names (a row with it in a text column), else in one that
+/// holds a brush pattern, else in any.
+fn material_png(table: &Table, key: &str) -> Option<Vec<u8>> {
+ let named: Vec<&Vec<Value>> =
+ table.rows.iter().filter(|r| r.iter().any(|v| v.as_text().is_some_and(|t| t.contains(key)))).collect();
+ let rows = if !named.is_empty() {
+ named
+ } else {
+ let patterns: Vec<&Vec<Value>> = table
+ .rows
+ .iter()
+ .filter(|r| r.iter().any(|v| v.as_blob().is_some_and(|b| find(b, 0, b"BrushPattern").is_some())))
+ .collect();
+ if patterns.is_empty() { table.rows.iter().collect() } else { patterns }
+ };
+ rows.into_iter()
+ .flat_map(|r| r.iter().filter_map(Value::as_blob).flat_map(pngs))
+ .max_by_key(|(area, _)| *area)
+ .map(|(_, png)| png.to_vec())
+}
+
+/// The PNGs inside `bytes` (each from its signature through its end chunk), with their areas.
+fn pngs(bytes: &[u8]) -> Vec<(u64, &[u8])> {
+ let mut out = Vec::new();
+ let mut from = 0;
+ while out.len() < MOST_PNGS {
+ let Some(start) = find(bytes, from, &PNG) else { break };
+ let Some(end) = find(bytes, start + 8, b"IEND").map(|e| e + 8) else { break };
+ let size = |at: usize| bytes.get(at..at + 4).and_then(|b| b.try_into().ok()).map(u32::from_be_bytes);
+ // the header chunk's width and height
+ if let (Some(w), Some(h), Some(png)) = (size(start + 16), size(start + 20), bytes.get(start..end)) {
+ out.push((u64::from(w) * u64::from(h), png));
+ }
+ from = end;
+ }
+ out
+}
+
+fn find(hay: &[u8], from: usize, needle: &[u8]) -> Option<usize> {
+ hay.get(from..)?.windows(needle.len()).position(|w| w == needle).map(|p| p + from)
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use crate::sqlite::tests::database;
+
+ fn utf16(s: &str) -> Vec<u8> {
+ s.encode_utf16().flat_map(u16::to_le_bytes).collect()
+ }
+
+ /// A PNG-shaped run: signature, a header chunk of `w` × `h`, an end chunk.
+ fn png(w: u32, h: u32) -> Vec<u8> {
+ let mut p = PNG.to_vec();
+ p.extend_from_slice(&13u32.to_be_bytes());
+ p.extend_from_slice(b"IHDR");
+ p.extend_from_slice(&w.to_be_bytes());
+ p.extend_from_slice(&h.to_be_bytes());
+ p.extend_from_slice(&[8, 0, 0, 0, 0, 0, 0, 0, 0]);
+ p.extend_from_slice(&[0, 0, 0, 0]);
+ p.extend_from_slice(b"IEND");
+ p.extend_from_slice(&[0; 4]);
+ p
+ }
+
+ fn sample() -> Vec<u8> {
+ let mut pattern = vec![0u8; 6];
+ pattern.extend(utf16("0:data:tip-1"));
+ pattern.extend([0, 0]);
+ let mut tar = b"tip.png\0padding".to_vec();
+ tar.extend(png(4, 4));
+ tar.extend(b"more");
+ tar.extend(png(64, 32));
+ database(&[
+ (
+ "Node",
+ "CREATE TABLE Node(_PW_ID INTEGER PRIMARY KEY, NodeName TEXT, NodeVariantID INTEGER)",
+ vec![
+ vec![Value::Null, Value::Text("Tools".into()), Value::Int(0)],
+ vec![Value::Null, Value::Text("Dry Pencil".into()), Value::Int(7)],
+ ],
+ ),
+ (
+ "Variant",
+ "CREATE TABLE Variant(_PW_ID INTEGER PRIMARY KEY, VariantID INTEGER, BrushHardness REAL, BrushInterval REAL, BrushThickness REAL, BrushRotation REAL, BrushPatternImageArray BLOB)",
+ vec![vec![
+ Value::Null,
+ Value::Int(7),
+ Value::Real(80.0),
+ Value::Real(25.0),
+ Value::Real(50.0),
+ Value::Real(90.0),
+ Value::Blob(pattern),
+ ]],
+ ),
+ (
+ "MaterialFile",
+ "CREATE TABLE MaterialFile(_PW_ID INTEGER PRIMARY KEY, CatalogPath TEXT, FileData BLOB)",
+ vec![
+ vec![Value::Null, Value::Text("other".into()), Value::Blob(png(500, 500))],
+ vec![Value::Null, Value::Text("x/0:data:tip-1/y".into()), Value::Blob(tar)],
+ ],
+ ),
+ ])
+ }
+
+ #[test]
+ fn a_sub_tool_reads_its_settings_and_named_tip() {
+ let brushes = read(&sample()).unwrap();
+ // the folder (no settings) isn't a brush
+ assert_eq!(brushes.len(), 1);
+ let b = &brushes[0];
+ assert_eq!(b.name, "Dry Pencil");
+ assert_eq!((b.hardness, b.spacing, b.roundness), (Some(0.8), Some(0.25), Some(0.5)));
+ assert!((b.angle - std::f32::consts::FRAC_PI_2).abs() < 1e-6);
+ // the material its pattern names, the larger of its two pictures
+ let tip = b.tip_png.as_ref().unwrap();
+ assert_eq!(&tip[16..24], &[0, 0, 0, 64, 0, 0, 0, 32]);
+ }
+
+ #[test]
+ fn other_files_and_damaged_ones_are_refused_not_a_crash() {
+ assert!(read(b"nothing").is_none());
+ crate::damaged::survives("Clip Studio sub tool", &sample(), |b| {
+ let _ = read(b);
+ });
+ }
+}
diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs
index 4222231..a13cd46 100644
--- a/crates/engine/src/lib.rs
+++ b/crates/engine/src/lib.rs
@@ -11,6 +11,7 @@ pub mod brushfile;
pub mod cms;
pub mod cmyk;
pub mod crop;
+pub mod csp;
pub mod curve;
#[cfg(test)]
mod damaged;
@@ -38,6 +39,7 @@ pub mod profile;
pub mod psd;
pub mod quickshape;
pub mod selection;
+pub mod sqlite;
pub mod stroke;
pub mod studio;
pub mod textures;
diff --git a/crates/engine/src/sqlite.rs b/crates/engine/src/sqlite.rs
new file mode 100644
index 0000000..da33771
--- /dev/null
+++ b/crates/engine/src/sqlite.rs
@@ -0,0 +1,488 @@
+//! Reading SQLite 3 databases, as the file format is documented: the schema table on the first
+//! page, each table's b-tree of pages (with payloads spilling onto overflow pages), and records.
+//! Enough to list a table's rows; damage ends a table where it is.
+
+/// A value in a row.
+#[derive(Clone, Debug, PartialEq)]
+pub enum Value {
+ Null,
+ Int(i64),
+ Real(f64),
+ Text(String),
+ Blob(Vec<u8>),
+}
+
+impl Value {
+ pub fn as_int(&self) -> Option<i64> {
+ match self {
+ Value::Int(v) => Some(*v),
+ Value::Real(v) if v.is_finite() => Some(*v as i64),
+ _ => None,
+ }
+ }
+
+ pub fn as_real(&self) -> Option<f64> {
+ match self {
+ Value::Int(v) => Some(*v as f64),
+ Value::Real(v) if v.is_finite() => Some(*v),
+ _ => None,
+ }
+ }
+
+ pub fn as_text(&self) -> Option<&str> {
+ match self {
+ Value::Text(t) => Some(t),
+ _ => None,
+ }
+ }
+
+ pub fn as_blob(&self) -> Option<&[u8]> {
+ match self {
+ Value::Blob(b) => Some(b),
+ _ => None,
+ }
+ }
+}
+
+/// A table's columns (as its schema names them) and rows.
+#[derive(Clone, Debug, Default, PartialEq)]
+pub struct Table {
+ pub columns: Vec<String>,
+ pub rows: Vec<Vec<Value>>,
+}
+
+impl Table {
+ /// The value of column `name` (any case) in `row`.
+ pub fn get<'a>(&self, row: &'a [Value], name: &str) -> Option<&'a Value> {
+ let at = self.columns.iter().position(|c| c.eq_ignore_ascii_case(name))?;
+ row.get(at)
+ }
+}
+
+/// An open database.
+pub struct Database<'a> {
+ bytes: &'a [u8],
+ page_size: usize,
+ usable: usize,
+ utf16: Option<bool>,
+}
+
+/// The most rows read from a table, and the most bytes a payload may have.
+const MOST_ROWS: usize = 100_000;
+const MOST_PAYLOAD: usize = 64 << 20;
+
+/// The database in `bytes`; none if it isn't one.
+pub fn open(bytes: &[u8]) -> Option<Database<'_>> {
+ if bytes.get(..16)? != b"SQLite format 3\0" {
+ return None;
+ }
+ let size = u16::from_be_bytes([*bytes.get(16)?, *bytes.get(17)?]);
+ let page_size = if size == 1 { 65536 } else { usize::from(size) };
+ if page_size < 512 || !page_size.is_power_of_two() {
+ return None;
+ }
+ let reserved = usize::from(*bytes.get(20)?);
+ let usable = page_size.checked_sub(reserved).filter(|u| *u >= 480)?;
+ // text: UTF-8, or UTF-16 little- or big-endian
+ let utf16 = match u32::from_be_bytes(bytes.get(56..60)?.try_into().ok()?) {
+ 2 => Some(false),
+ 3 => Some(true),
+ _ => None,
+ };
+ Some(Database { bytes, page_size, usable, utf16 })
+}
+
+impl Database<'_> {
+ /// The table named `name` (any case); none if there is none.
+ pub fn table(&self, name: &str) -> Option<Table> {
+ let schema = self.rows(1);
+ let entry = schema.iter().find(|(_, r)| {
+ r.first().and_then(Value::as_text) == Some("table")
+ && r.get(1).and_then(Value::as_text).is_some_and(|n| n.eq_ignore_ascii_case(name))
+ })?;
+ let root = u32::try_from(entry.1.get(3)?.as_int()?).ok()?;
+ let sql = entry.1.get(4).and_then(Value::as_text).unwrap_or_default();
+ let (columns, rowid) = columns(sql);
+ let rows = self
+ .rows(root)
+ .into_iter()
+ .map(|(id, mut row)| {
+ // a column that is the row's id is kept as nothing in the record
+ if let Some(slot) = rowid.and_then(|at| row.get_mut(at)).filter(|v| **v == Value::Null) {
+ *slot = Value::Int(id);
+ }
+ row.resize(columns.len().max(row.len()), Value::Null);
+ row
+ })
+ .collect();
+ Some(Table { columns, rows })
+ }
+
+ fn page(&self, n: u32) -> Option<&[u8]> {
+ let start = (n as usize).checked_sub(1)?.checked_mul(self.page_size)?;
+ self.bytes.get(start..start.checked_add(self.page_size)?)
+ }
+
+ /// The rows of the table whose b-tree starts at page `root`: each its id and values.
+ fn rows(&self, root: u32) -> Vec<(i64, Vec<Value>)> {
+ let mut out = Vec::new();
+ let mut seen = vec![false; self.bytes.len() / self.page_size + 1];
+ let mut pending = vec![root];
+ while let Some(n) = pending.pop() {
+ if out.len() >= MOST_ROWS {
+ break;
+ }
+ match seen.get_mut(n as usize) {
+ Some(s) if !*s => *s = true,
+ _ => continue,
+ }
+ let Some(page) = self.page(n) else { continue };
+ let h = if n == 1 { 100 } else { 0 };
+ let (Some(&kind), Some(count)) = (page.get(h), be16(page, h + 3)) else { continue };
+ match kind {
+ // interior: the children left to right (pushed so the leftmost comes off first)
+ 0x05 => {
+ let mut children: Vec<u32> = (0..count)
+ .filter_map(|i| {
+ be16(page, h + 12 + 2 * usize::from(i)).and_then(|at| be32(page, usize::from(at)))
+ })
+ .collect();
+ children.extend(be32(page, h + 8));
+ pending.extend(children.into_iter().rev());
+ }
+ 0x0D => {
+ for i in 0..count {
+ let Some(at) = be16(page, h + 8 + 2 * usize::from(i)) else { break };
+ if let Some(row) = self.cell(page, usize::from(at)) {
+ out.push(row);
+ }
+ }
+ }
+ _ => {}
+ }
+ }
+ out
+ }
+
+ /// A leaf cell: its row id and record, the payload gathered from overflow pages.
+ fn cell(&self, page: &[u8], at: usize) -> Option<(i64, Vec<Value>)> {
+ let (len, a) = varint(page, at)?;
+ let (id, b) = varint(page, at + a)?;
+ let len = usize::try_from(len).ok().filter(|l| *l <= MOST_PAYLOAD)?;
+ let start = at + a + b;
+ // how much of the payload is on this page (the rest on overflow pages)
+ let u = self.usable;
+ let most_here = u - 35;
+ let local = if len <= most_here {
+ len
+ } else {
+ let least = (u - 12) * 32 / 255 - 23;
+ let k = least + (len - least) % (u - 4);
+ if k <= most_here { k } else { least }
+ };
+ let mut payload = page.get(start..start + local)?.to_vec();
+ let mut next = if local < len { be32(page, start + local)? } else { 0 };
+ let mut hops = 0;
+ while payload.len() < len && next != 0 && hops < self.bytes.len() / self.page_size {
+ let over = self.page(next)?;
+ let take = (len - payload.len()).min(u - 4);
+ payload.extend_from_slice(over.get(4..4 + take)?);
+ next = be32(over, 0)?;
+ hops += 1;
+ }
+ if payload.len() < len {
+ return None;
+ }
+ Some((id, self.record(&payload)?))
+ }
+
+ /// A record's values.
+ fn record(&self, payload: &[u8]) -> Option<Vec<Value>> {
+ let (header, mut at) = varint(payload, 0)?;
+ let header = usize::try_from(header).ok()?;
+ let mut body = header;
+ let mut values = Vec::new();
+ while at < header {
+ let (kind, n) = varint(payload, at)?;
+ at += n;
+ let (value, used) = match kind {
+ 0 => (Value::Null, 0),
+ 1..=6 => {
+ let size = match kind {
+ 5 => 6,
+ 6 => 8,
+ k => k as usize,
+ };
+ let bytes = payload.get(body..body + size)?;
+ // big-endian, signed
+ let mut v: i64 = if bytes.first().is_some_and(|b| b & 0x80 != 0) { -1 } else { 0 };
+ for &b in bytes {
+ v = (v << 8) | i64::from(b);
+ }
+ (Value::Int(v), size)
+ }
+ 7 => (Value::Real(f64::from_be_bytes(payload.get(body..body + 8)?.try_into().ok()?)), 8),
+ 8 => (Value::Int(0), 0),
+ 9 => (Value::Int(1), 0),
+ k if k >= 12 && k % 2 == 0 => {
+ let size = usize::try_from((k - 12) / 2).ok()?;
+ (Value::Blob(payload.get(body..body.checked_add(size)?)?.to_vec()), size)
+ }
+ k if k >= 13 => {
+ let size = usize::try_from((k - 13) / 2).ok()?;
+ (Value::Text(self.text(payload.get(body..body.checked_add(size)?)?)), size)
+ }
+ _ => return None,
+ };
+ body += used;
+ values.push(value);
+ }
+ Some(values)
+ }
+
+ fn text(&self, bytes: &[u8]) -> String {
+ match self.utf16 {
+ None => String::from_utf8_lossy(bytes).into_owned(),
+ Some(big) => {
+ let units: Vec<u16> = bytes
+ .as_chunks::<2>()
+ .0
+ .iter()
+ .map(|c| if big { u16::from_be_bytes(*c) } else { u16::from_le_bytes(*c) })
+ .collect();
+ String::from_utf16_lossy(&units)
+ }
+ }
+ }
+}
+
+/// A table's column names from its `CREATE TABLE` statement, and which one (if any) is the row's
+/// id (an `INTEGER PRIMARY KEY`).
+fn columns(sql: &str) -> (Vec<String>, Option<usize>) {
+ let (Some(open), Some(close)) = (sql.find('('), sql.rfind(')')) else { return (Vec::new(), None) };
+ let inside = sql.get(open + 1..close).unwrap_or_default();
+ let mut parts = Vec::new();
+ let (mut depth, mut from) = (0i32, 0usize);
+ for (i, c) in inside.char_indices() {
+ match c {
+ '(' => depth += 1,
+ ')' => depth -= 1,
+ ',' if depth == 0 => {
+ parts.push(inside.get(from..i).unwrap_or_default());
+ from = i + 1;
+ }
+ _ => {}
+ }
+ }
+ parts.push(inside.get(from..).unwrap_or_default());
+ let mut names = Vec::new();
+ let mut rowid = None;
+ for part in parts {
+ let part = part.trim();
+ let first = part.split_whitespace().next().unwrap_or_default();
+ let upper = part.to_ascii_uppercase();
+ // table constraints, not columns (their keyword may run into a parenthesis)
+ let word: String = part
+ .chars()
+ .take_while(|c| c.is_ascii_alphanumeric() || *c == '_')
+ .collect::<String>()
+ .to_ascii_uppercase();
+ if ["PRIMARY", "UNIQUE", "CHECK", "FOREIGN", "CONSTRAINT"].contains(&word.as_str()) {
+ continue;
+ }
+ let name = first.trim_matches(|c| c == '"' || c == '`' || c == '[' || c == ']' || c == '\'');
+ if upper.contains("INTEGER") && upper.contains("PRIMARY KEY") {
+ rowid = Some(names.len());
+ }
+ names.push(name.to_owned());
+ }
+ (names, rowid)
+}
+
+fn be16(b: &[u8], at: usize) -> Option<u16> {
+ Some(u16::from_be_bytes([*b.get(at)?, *b.get(at + 1)?]))
+}
+
+fn be32(b: &[u8], at: usize) -> Option<u32> {
+ Some(u32::from_be_bytes(b.get(at..at + 4)?.try_into().ok()?))
+}
+
+/// A variable-length number (up to 9 bytes, seven bits each but the ninth's eight) and its length.
+fn varint(b: &[u8], at: usize) -> Option<(i64, usize)> {
+ let mut v: u64 = 0;
+ for i in 0..9 {
+ let byte = *b.get(at + i)?;
+ if i == 8 {
+ return Some((((v << 8) | u64::from(byte)) as i64, 9));
+ }
+ v = (v << 7) | u64::from(byte & 0x7F);
+ if byte & 0x80 == 0 {
+ return Some((v as i64, i + 1));
+ }
+ }
+ None
+}
+
+#[cfg(test)]
+pub(crate) mod tests {
+ use super::*;
+
+ /// A one-page-per-table database of 4096-byte pages: `tables` each a name, its `CREATE TABLE`
+ /// statement and its rows (values as serial type and bytes), all on one leaf page; a value
+ /// too big for its page spills onto overflow pages.
+ pub(crate) fn database(tables: &[(&str, &str, Vec<Vec<Value>>)]) -> Vec<u8> {
+ const PAGE: usize = 4096;
+ fn put_varint(out: &mut Vec<u8>, mut v: u64) {
+ let mut bytes = vec![(v & 0x7F) as u8];
+ v >>= 7;
+ while v > 0 {
+ bytes.push((v & 0x7F) as u8 | 0x80);
+ v >>= 7;
+ }
+ out.extend(bytes.iter().rev());
+ }
+ fn record(values: &[Value]) -> Vec<u8> {
+ let (mut types, mut body) = (Vec::new(), Vec::new());
+ for v in values {
+ match v {
+ Value::Null => put_varint(&mut types, 0),
+ Value::Int(i) => {
+ put_varint(&mut types, 6);
+ body.extend_from_slice(&i.to_be_bytes());
+ }
+ Value::Real(r) => {
+ put_varint(&mut types, 7);
+ body.extend_from_slice(&r.to_be_bytes());
+ }
+ Value::Text(t) => {
+ put_varint(&mut types, 13 + 2 * t.len() as u64);
+ body.extend_from_slice(t.as_bytes());
+ }
+ Value::Blob(b) => {
+ put_varint(&mut types, 12 + 2 * b.len() as u64);
+ body.extend_from_slice(b);
+ }
+ }
+ }
+ let mut out = Vec::new();
+ put_varint(&mut out, types.len() as u64 + 1);
+ out.extend(types);
+ out.extend(body);
+ out
+ }
+ // page 1 the schema; then each table's leaf; overflow pages after
+ let mut pages: Vec<Vec<u8>> = vec![Vec::new(); 1 + tables.len()];
+ let leaf = |pages: &mut Vec<Vec<u8>>, n: usize, rows: Vec<Vec<Value>>| {
+ let h = if n == 0 { 100 } else { 0 };
+ let mut page = vec![0u8; PAGE];
+ let mut cells: Vec<Vec<u8>> = Vec::new();
+ for (id, row) in rows.iter().enumerate() {
+ let payload = record(row);
+ let mut cell = Vec::new();
+ put_varint(&mut cell, payload.len() as u64);
+ put_varint(&mut cell, id as u64 + 1);
+ let u = PAGE;
+ let (most, least) = (u - 35, (u - 12) * 32 / 255 - 23);
+ if payload.len() <= most {
+ cell.extend(&payload);
+ } else {
+ let k = least + (payload.len() - least) % (u - 4);
+ let local = if k <= most { k } else { least };
+ cell.extend(&payload[..local]);
+ let mut rest = &payload[local..];
+ let first = pages.len() + 1;
+ cell.extend_from_slice(&(first as u32).to_be_bytes());
+ while !rest.is_empty() {
+ let take = rest.len().min(u - 4);
+ let mut over = vec![0u8; PAGE];
+ let next = if rest.len() > take { pages.len() + 2 } else { 0 };
+ over[..4].copy_from_slice(&(next as u32).to_be_bytes());
+ over[4..4 + take].copy_from_slice(&rest[..take]);
+ pages.push(over);
+ rest = &rest[take..];
+ }
+ }
+ cells.push(cell);
+ }
+ page[h] = 0x0D;
+ page[h + 3..h + 5].copy_from_slice(&(cells.len() as u16).to_be_bytes());
+ let mut end = PAGE;
+ for (i, cell) in cells.iter().enumerate() {
+ end -= cell.len();
+ page[end..end + cell.len()].copy_from_slice(cell);
+ page[h + 8 + 2 * i..h + 10 + 2 * i].copy_from_slice(&(end as u16).to_be_bytes());
+ }
+ page[h + 5..h + 7].copy_from_slice(&(end as u16).to_be_bytes());
+ pages[n] = page;
+ };
+ let schema: Vec<Vec<Value>> = tables
+ .iter()
+ .enumerate()
+ .map(|(i, (name, sql, _))| {
+ vec![
+ Value::Text("table".into()),
+ Value::Text((*name).into()),
+ Value::Text((*name).into()),
+ Value::Int(i as i64 + 2),
+ Value::Text((*sql).into()),
+ ]
+ })
+ .collect();
+ for (i, (_, _, rows)) in tables.iter().enumerate() {
+ leaf(&mut pages, i + 1, rows.clone());
+ }
+ leaf(&mut pages, 0, schema);
+ let mut out: Vec<u8> = pages.concat();
+ out[..16].copy_from_slice(b"SQLite format 3\0");
+ out[16..18].copy_from_slice(&(PAGE as u16).to_be_bytes());
+ out[56..60].copy_from_slice(&1u32.to_be_bytes());
+ out
+ }
+
+ #[test]
+ fn tables_read_with_their_columns_ids_and_long_values() {
+ let long: Vec<u8> = (0..20_000).map(|i| (i % 251) as u8).collect();
+ let db = database(&[
+ (
+ "Node",
+ "CREATE TABLE Node (_PW_ID INTEGER PRIMARY KEY AUTOINCREMENT, NodeName TEXT, NodeVariantID INTEGER)",
+ vec![
+ vec![Value::Null, Value::Text("Pencil".into()), Value::Int(3)],
+ vec![Value::Null, Value::Text("Ink".into()), Value::Int(4)],
+ ],
+ ),
+ (
+ "Material",
+ "CREATE TABLE \"Material\" (id INTEGER, \"FileData\" BLOB, Size REAL, UNIQUE(id))",
+ vec![vec![Value::Int(1), Value::Blob(long.clone()), Value::Real(2.5)]],
+ ),
+ ]);
+ let d = open(&db).unwrap();
+ let node = d.table("node").unwrap();
+ assert_eq!(node.columns, ["_PW_ID", "NodeName", "NodeVariantID"]);
+ assert_eq!(node.rows.len(), 2);
+ assert_eq!(node.get(&node.rows[1], "NodeName").and_then(Value::as_text), Some("Ink"));
+ // the id column holds the row's id
+ assert_eq!(node.get(&node.rows[1], "_PW_ID").and_then(Value::as_int), Some(2));
+ let m = d.table("Material").unwrap();
+ assert_eq!(m.columns, ["id", "FileData", "Size"]);
+ assert_eq!(m.get(&m.rows[0], "FileData").and_then(Value::as_blob), Some(&long[..]));
+ assert_eq!(m.get(&m.rows[0], "Size").and_then(Value::as_real), Some(2.5));
+ assert!(d.table("Nothing").is_none());
+ }
+
+ #[test]
+ fn other_files_and_damaged_ones_are_refused_not_a_crash() {
+ assert!(open(b"not a database").is_none());
+ let db = database(&[(
+ "T",
+ "CREATE TABLE T (a TEXT, b BLOB)",
+ vec![vec![Value::Text("x".into()), Value::Blob(vec![1; 5000])]],
+ )]);
+ crate::damaged::survives("SQLite", &db, |b| {
+ if let Some(d) = open(b) {
+ let _ = d.table("T");
+ }
+ });
+ }
+}