ProCraft for Android: An Android native painting app written in Rust
Rust
feat(import): Clip Studio sub tools as brushes, read with our own SQLite reader
Josh4628d9794013
diff --git a/apps/procraft-android/android/app/src/main/java/io/github/artcraftandroid/procraft/canvas/CanvasViewModel.kt b/apps/procraft-android/android/app/src/main/java/io/github/artcraftandroid/procraft/canvas/CanvasViewModel.ktindex 773e5a1..d03ab4e 100644--- a/apps/procraft-android/android/app/src/main/java/io/github/artcraftandroid/procraft/canvas/CanvasViewModel.kt+++ b/apps/procraft-android/android/app/src/main/java/io/github/artcraftandroid/procraft/canvas/CanvasViewModel.kt@@ -5863,8 +5863,8 @@ class CanvasViewModel(app: Application) : AndroidViewModel(app) {/** Brush and brush set files, as other painting apps share them. */const val BRUSH_EXTENSION = "brush"const val BRUSH_SET_EXTENSION = "brushset"- /** Other apps' brush files: Photoshop's, GIMP's brushes and pipes, Krita's presets and bundles. */- val OTHER_BRUSH_EXTENSIONS = listOf("abr", "gbr", "gih", "kpp", "bundle")+ /** Other apps' brush files: Photoshop's, GIMP's brushes and pipes, Krita's presets and bundles, Clip Studio's sub tools. */+ val OTHER_BRUSH_EXTENSIONS = listOf("abr", "gbr", "gih", "kpp", "bundle", "sut")val PACKAGE_LABEL = PACKAGE_EXTENSION.replaceFirstChar(Char::titlecase)/** What the gallery's Share offers, in order: still pictures, then animations. */diff --git a/apps/procraft-android/src/brush_files.rs b/apps/procraft-android/src/brush_files.rsindex 663ed6e..51e5277 100644--- a/apps/procraft-android/src/brush_files.rs+++ b/apps/procraft-android/src/brush_files.rs@@ -108,7 +108,7 @@ impl CanvasSession {}/// Adds the brushes in another app's file (`bytes`): a Photoshop brush file's sampled tips, a- /// GIMP brush or pipe, or a Krita preset or bundle. They go in a set of their own named `name`+ /// GIMP brush or pipe, a Krita preset or bundle, or a Clip Studio sub tool. They go in a set of their own named `name`/// (a bundle's own name first; else "Imported") at the top of the library, each with the/// file's shape and spacing. Returns the set, or none if the file holds none.pub fn import_other_brushes(&self, bytes: Vec<u8>, name: String) -> Option<u32> {@@ -213,11 +213,39 @@ fn other_brushes(bytes: &[u8], name: String) -> Option<(String, Vec<Brush>)> {let name = if title.trim().is_empty() { name } else { title };return Some((name, presets.iter().map(krita_brush).collect()));}+ if let Some(brushes) = procraft_engine::csp::read(bytes) {+ return Some((name, brushes.iter().map(csp_brush).collect()));+ }// a preset alone has no images of its own: one it names comes roundlet preset = krita::read_preset(bytes, &|_| None)?;Some((name, vec![krita_brush(&preset)]))}+/// A brush of a Clip Studio sub tool: its tip (an image, else round with its edge and roundness),+/// spacing and angle.+fn csp_brush(c: &procraft_engine::csp::CspBrush) -> Brush {+ // its tips paint white, or else dark+ let tip = c+ .tip_png+ .as_deref()+ .and_then(|png| png_tip(png, true).filter(|t| t.pixels.iter().any(|&v| v > 0)).or_else(|| png_tip(png, false)));+ let mut b = match tip {+ Some(t) => tip_brush(&procraft_engine::abr::AbrTip { name: c.name.clone(), ..t }),+ None => Brush { name: c.name.clone(), ..Brush::round_pen() },+ };+ if let Some(h) = c.hardness {+ b.hardness = h.clamp(0.0, 1.0);+ }+ if let Some(r) = c.roundness {+ b.shape.roundness = r.clamp(0.01, 1.0);+ }+ if let Some(spacing) = c.spacing {+ b.spacing = spacing.clamp(0.01, 10.0);+ }+ b.shape.rotation = c.angle;+ b+}+/// A brush of a sampled tip: its shape, with its spacing.fn tip_brush(t: &procraft_engine::abr::AbrTip) -> Brush {let mut b = Brush { name: t.name.clone(), ..Brush::round_pen() };@@ -236,7 +264,7 @@ fn krita_brush(k: &procraft_engine::krita::KritaBrush) -> Brush {let image = match &k.tip {KritaTip::Round { .. } => None,KritaTip::Image(t) => Some(fitted(t)),- KritaTip::Png(png) => png_tip(png).map(|t| fitted(&t)),+ KritaTip::Png(png) => png_tip(png, false).map(|t| fitted(&t)),};let mut b = match image {Some(pixels) => {@@ -264,8 +292,8 @@ fn krita_brush(k: &procraft_engine::krita::KritaBrush) -> Brush {b}-/// A PNG tip as a grey one: dark, opaque pixels paint.-fn png_tip(png: &[u8]) -> Option<procraft_engine::abr::AbrTip> {+/// A PNG tip as a grey one: its opaque pixels paint, the light ones if `white`, else the dark.+fn png_tip(png: &[u8], white: bool) -> Option<procraft_engine::abr::AbrTip> {let (w, h, channels, buf) = decode_png(png)?;let pixels: Vec<u8> = buf.chunks_exact(channels.max(1))@@ -278,7 +306,8 @@ fn png_tip(png: &[u8]) -> Option<procraft_engine::abr::AbrTip> {3 => ((get(0) * 299 + get(1) * 587 + get(2) * 114) / 1000, 255),_ => ((get(0) * 299 + get(1) * 587 + get(2) * 114) / 1000, get(3)),};- ((255 - grey.min(255)) * alpha / 255) as u8+ let ink = if white { grey.min(255) } else { 255 - grey.min(255) };+ (ink * alpha / 255) as u8}).collect();(pixels.len() == w * h).then(|| procraft_engine::abr::AbrTip {diff --git a/apps/procraft-android/src/tests.rs b/apps/procraft-android/src/tests.rsindex 2c49c96..7111fd8 100644--- a/apps/procraft-android/src/tests.rs+++ b/apps/procraft-android/src/tests.rs@@ -2826,7 +2826,8 @@ fn a_photoshop_brush_files_tips_become_a_set() {}/// Real brush files, when a folder of them is given (`BRUSH_CORPUS=folder`, never committed):-/// every `.brush`, `.brushset` and `.abr` in it and below imports, each brush draws in its+/// every brush file in it and below (ours, and Photoshop's, GIMP's, Krita's and Clip Studio's)+/// imports, each brush draws in its/// preview, and a brush written out, read back and written out again comes out the same.#[test]fn a_corpus_of_real_brush_files_imports_draws_and_round_trips() {@@ -2841,7 +2842,10 @@ fn a_corpus_of_real_brush_files_imports_draws_and_round_trips() {let path = entry.path();if path.is_dir() {folders.push(path);- } else if matches!(extension(&path).as_str(), "brush" | "brushset" | "abr") {+ } else if matches!(+ extension(&path).as_str(),+ "brush" | "brushset" | "abr" | "gbr" | "gih" | "kpp" | "bundle" | "sut"+ ) {files.push(path);}}@@ -2856,8 +2860,11 @@ fn a_corpus_of_real_brush_files_imports_draws_and_round_trips() {let Some(s) = session(64, 64) else { return };let bytes = std::fs::read(path).unwrap();let kind = extension(path);- let set =- if kind == "abr" { s.import_other_brushes(bytes, "Corpus".to_owned()) } else { s.import_brushes(bytes) };+ let set = if kind == "brush" || kind == "brushset" {+ s.import_brushes(bytes)+ } else {+ s.import_other_brushes(bytes, "Corpus".to_owned())+ };let Some(set) = set else {failures.push(format!("{}: not imported", path.display()));continue;diff --git a/crates/engine/src/csp.rs b/crates/engine/src/csp.rsnew file mode 100644index 0000000..99b4855--- /dev/null+++ b/crates/engine/src/csp.rs@@ -0,0 +1,216 @@+//! Reading Clip Studio sub tools (`.sut`): a SQLite database whose `Node` table names the brushes,+//! each pointing (`NodeVariantID`) at its settings in `Variant`, and whose `MaterialFile` table+//! holds the tip images (PNGs, often inside a tar) that the settings name by key.++use crate::sqlite::{self, Table, Value};++/// A brush as a sub tool has it: its settings where it has them.+#[derive(Clone, Debug, PartialEq)]+pub struct CspBrush {+ pub name: String,+ /// The edge, 0 soft to 1 hard.+ pub hardness: Option<f32>,+ /// Space between stamps as a share of the size.+ pub spacing: Option<f32>,+ /// How round the tip is, 1 a circle.+ pub roundness: Option<f32>,+ /// The tip's angle, radians.+ pub angle: f32,+ /// The tip's image (a PNG), if it isn't round.+ pub tip_png: Option<Vec<u8>>,+}++/// The most brushes read from one file, and PNGs looked at in a material.+const MOST_BRUSHES: usize = 256;+const MOST_PNGS: usize = 32;+const PNG: [u8; 8] = [0x89, b'P', b'N', b'G', 0x0D, 0x0A, 0x1A, 0x0A];++/// The brushes in a sub tool file, in its order; none if it isn't one.+pub fn read(bytes: &[u8]) -> Option<Vec<CspBrush>> {+ let db = sqlite::open(bytes)?;+ let nodes = db.table("Node")?;+ let variants = db.table("Variant");+ let materials = db.table("MaterialFile");+ let named = |row: &Vec<Value>| {+ nodes.get(row, "NodeName").and_then(Value::as_text).map(str::trim).filter(|n| !n.is_empty()).map(str::to_owned)+ };+ // the nodes with settings of their own (a folder of sub tools has none)+ let mut out = Vec::new();+ for row in &nodes.rows {+ let Some(name) = named(row) else { continue };+ let id = nodes.get(row, "NodeVariantID").and_then(Value::as_int);+ let variant = variants.as_ref().and_then(|v| {+ id.and_then(|id| v.rows.iter().find(|r| v.get(r, "VariantID").and_then(Value::as_int) == Some(id)))+ .map(|r| (v, r))+ });+ if variant.is_some() {+ out.push(brush(&name, variant, materials.as_ref()));+ }+ if out.len() >= MOST_BRUSHES {+ break;+ }+ }+ // none: the first named node, round+ if out.is_empty() {+ out.extend(nodes.rows.iter().find_map(named).map(|name| brush(&name, None, materials.as_ref())));+ }+ (!out.is_empty()).then_some(out)+}++fn brush(name: &str, variant: Option<(&Table, &Vec<Value>)>, materials: Option<&Table>) -> CspBrush {+ let number =+ |key: &str| variant.and_then(|(t, r)| t.get(r, key)).and_then(Value::as_real).filter(|v| v.is_finite());+ let pattern =+ variant.and_then(|(t, r)| t.get(r, "BrushPatternImageArray")).and_then(Value::as_blob).and_then(pattern_key);+ CspBrush {+ name: name.to_owned(),+ hardness: number("BrushHardness").filter(|v| (0.0..=100.0).contains(v)).map(|v| (v / 100.0) as f32),+ spacing: number("BrushInterval").filter(|v| *v > 0.0).map(|v| (v / 100.0) as f32),+ roundness: number("BrushThickness").filter(|v| (1.0..=100.0).contains(v)).map(|v| (v / 100.0) as f32),+ angle: number("BrushRotation").map_or(0.0, |v| v.to_radians() as f32),+ tip_png: pattern.and_then(|key| materials.and_then(|m| material_png(m, &key))),+ }+}++/// The key a pattern list names its first image by: a UTF-16 string with `:data:` in it.+fn pattern_key(blob: &[u8]) -> Option<String> {+ let mut current = String::new();+ let mut found = Vec::new();+ for pair in blob.as_chunks::<2>().0 {+ match char::from_u32(u32::from(u16::from_le_bytes(*pair))).filter(|c| c.is_ascii_graphic()) {+ Some(c) => current.push(c),+ None => found.push(std::mem::take(&mut current)),+ }+ }+ found.push(current);+ found.into_iter().find(|s| s.contains(":data:") && s.len() >= 8)+}++/// The largest PNG in the material `key` names (a row with it in a text column), else in one that+/// holds a brush pattern, else in any.+fn material_png(table: &Table, key: &str) -> Option<Vec<u8>> {+ let named: Vec<&Vec<Value>> =+ table.rows.iter().filter(|r| r.iter().any(|v| v.as_text().is_some_and(|t| t.contains(key)))).collect();+ let rows = if !named.is_empty() {+ named+ } else {+ let patterns: Vec<&Vec<Value>> = table+ .rows+ .iter()+ .filter(|r| r.iter().any(|v| v.as_blob().is_some_and(|b| find(b, 0, b"BrushPattern").is_some())))+ .collect();+ if patterns.is_empty() { table.rows.iter().collect() } else { patterns }+ };+ rows.into_iter()+ .flat_map(|r| r.iter().filter_map(Value::as_blob).flat_map(pngs))+ .max_by_key(|(area, _)| *area)+ .map(|(_, png)| png.to_vec())+}++/// The PNGs inside `bytes` (each from its signature through its end chunk), with their areas.+fn pngs(bytes: &[u8]) -> Vec<(u64, &[u8])> {+ let mut out = Vec::new();+ let mut from = 0;+ while out.len() < MOST_PNGS {+ let Some(start) = find(bytes, from, &PNG) else { break };+ let Some(end) = find(bytes, start + 8, b"IEND").map(|e| e + 8) else { break };+ let size = |at: usize| bytes.get(at..at + 4).and_then(|b| b.try_into().ok()).map(u32::from_be_bytes);+ // the header chunk's width and height+ if let (Some(w), Some(h), Some(png)) = (size(start + 16), size(start + 20), bytes.get(start..end)) {+ out.push((u64::from(w) * u64::from(h), png));+ }+ from = end;+ }+ out+}++fn find(hay: &[u8], from: usize, needle: &[u8]) -> Option<usize> {+ hay.get(from..)?.windows(needle.len()).position(|w| w == needle).map(|p| p + from)+}++#[cfg(test)]+mod tests {+ use super::*;+ use crate::sqlite::tests::database;++ fn utf16(s: &str) -> Vec<u8> {+ s.encode_utf16().flat_map(u16::to_le_bytes).collect()+ }++ /// A PNG-shaped run: signature, a header chunk of `w` × `h`, an end chunk.+ fn png(w: u32, h: u32) -> Vec<u8> {+ let mut p = PNG.to_vec();+ p.extend_from_slice(&13u32.to_be_bytes());+ p.extend_from_slice(b"IHDR");+ p.extend_from_slice(&w.to_be_bytes());+ p.extend_from_slice(&h.to_be_bytes());+ p.extend_from_slice(&[8, 0, 0, 0, 0, 0, 0, 0, 0]);+ p.extend_from_slice(&[0, 0, 0, 0]);+ p.extend_from_slice(b"IEND");+ p.extend_from_slice(&[0; 4]);+ p+ }++ fn sample() -> Vec<u8> {+ let mut pattern = vec![0u8; 6];+ pattern.extend(utf16("0:data:tip-1"));+ pattern.extend([0, 0]);+ let mut tar = b"tip.png\0padding".to_vec();+ tar.extend(png(4, 4));+ tar.extend(b"more");+ tar.extend(png(64, 32));+ database(&[+ (+ "Node",+ "CREATE TABLE Node(_PW_ID INTEGER PRIMARY KEY, NodeName TEXT, NodeVariantID INTEGER)",+ vec![+ vec![Value::Null, Value::Text("Tools".into()), Value::Int(0)],+ vec![Value::Null, Value::Text("Dry Pencil".into()), Value::Int(7)],+ ],+ ),+ (+ "Variant",+ "CREATE TABLE Variant(_PW_ID INTEGER PRIMARY KEY, VariantID INTEGER, BrushHardness REAL, BrushInterval REAL, BrushThickness REAL, BrushRotation REAL, BrushPatternImageArray BLOB)",+ vec![vec![+ Value::Null,+ Value::Int(7),+ Value::Real(80.0),+ Value::Real(25.0),+ Value::Real(50.0),+ Value::Real(90.0),+ Value::Blob(pattern),+ ]],+ ),+ (+ "MaterialFile",+ "CREATE TABLE MaterialFile(_PW_ID INTEGER PRIMARY KEY, CatalogPath TEXT, FileData BLOB)",+ vec![+ vec![Value::Null, Value::Text("other".into()), Value::Blob(png(500, 500))],+ vec![Value::Null, Value::Text("x/0:data:tip-1/y".into()), Value::Blob(tar)],+ ],+ ),+ ])+ }++ #[test]+ fn a_sub_tool_reads_its_settings_and_named_tip() {+ let brushes = read(&sample()).unwrap();+ // the folder (no settings) isn't a brush+ assert_eq!(brushes.len(), 1);+ let b = &brushes[0];+ assert_eq!(b.name, "Dry Pencil");+ assert_eq!((b.hardness, b.spacing, b.roundness), (Some(0.8), Some(0.25), Some(0.5)));+ assert!((b.angle - std::f32::consts::FRAC_PI_2).abs() < 1e-6);+ // the material its pattern names, the larger of its two pictures+ let tip = b.tip_png.as_ref().unwrap();+ assert_eq!(&tip[16..24], &[0, 0, 0, 64, 0, 0, 0, 32]);+ }++ #[test]+ fn other_files_and_damaged_ones_are_refused_not_a_crash() {+ assert!(read(b"nothing").is_none());+ crate::damaged::survives("Clip Studio sub tool", &sample(), |b| {+ let _ = read(b);+ });+ }+}diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rsindex 4222231..a13cd46 100644--- a/crates/engine/src/lib.rs+++ b/crates/engine/src/lib.rs@@ -11,6 +11,7 @@ pub mod brushfile;pub mod cms;pub mod cmyk;pub mod crop;+pub mod csp;pub mod curve;#[cfg(test)]mod damaged;@@ -38,6 +39,7 @@ pub mod profile;pub mod psd;pub mod quickshape;pub mod selection;+pub mod sqlite;pub mod stroke;pub mod studio;pub mod textures;diff --git a/crates/engine/src/sqlite.rs b/crates/engine/src/sqlite.rsnew file mode 100644index 0000000..da33771--- /dev/null+++ b/crates/engine/src/sqlite.rs@@ -0,0 +1,488 @@+//! Reading SQLite 3 databases, as the file format is documented: the schema table on the first+//! page, each table's b-tree of pages (with payloads spilling onto overflow pages), and records.+//! Enough to list a table's rows; damage ends a table where it is.++/// A value in a row.+#[derive(Clone, Debug, PartialEq)]+pub enum Value {+ Null,+ Int(i64),+ Real(f64),+ Text(String),+ Blob(Vec<u8>),+}++impl Value {+ pub fn as_int(&self) -> Option<i64> {+ match self {+ Value::Int(v) => Some(*v),+ Value::Real(v) if v.is_finite() => Some(*v as i64),+ _ => None,+ }+ }++ pub fn as_real(&self) -> Option<f64> {+ match self {+ Value::Int(v) => Some(*v as f64),+ Value::Real(v) if v.is_finite() => Some(*v),+ _ => None,+ }+ }++ pub fn as_text(&self) -> Option<&str> {+ match self {+ Value::Text(t) => Some(t),+ _ => None,+ }+ }++ pub fn as_blob(&self) -> Option<&[u8]> {+ match self {+ Value::Blob(b) => Some(b),+ _ => None,+ }+ }+}++/// A table's columns (as its schema names them) and rows.+#[derive(Clone, Debug, Default, PartialEq)]+pub struct Table {+ pub columns: Vec<String>,+ pub rows: Vec<Vec<Value>>,+}++impl Table {+ /// The value of column `name` (any case) in `row`.+ pub fn get<'a>(&self, row: &'a [Value], name: &str) -> Option<&'a Value> {+ let at = self.columns.iter().position(|c| c.eq_ignore_ascii_case(name))?;+ row.get(at)+ }+}++/// An open database.+pub struct Database<'a> {+ bytes: &'a [u8],+ page_size: usize,+ usable: usize,+ utf16: Option<bool>,+}++/// The most rows read from a table, and the most bytes a payload may have.+const MOST_ROWS: usize = 100_000;+const MOST_PAYLOAD: usize = 64 << 20;++/// The database in `bytes`; none if it isn't one.+pub fn open(bytes: &[u8]) -> Option<Database<'_>> {+ if bytes.get(..16)? != b"SQLite format 3\0" {+ return None;+ }+ let size = u16::from_be_bytes([*bytes.get(16)?, *bytes.get(17)?]);+ let page_size = if size == 1 { 65536 } else { usize::from(size) };+ if page_size < 512 || !page_size.is_power_of_two() {+ return None;+ }+ let reserved = usize::from(*bytes.get(20)?);+ let usable = page_size.checked_sub(reserved).filter(|u| *u >= 480)?;+ // text: UTF-8, or UTF-16 little- or big-endian+ let utf16 = match u32::from_be_bytes(bytes.get(56..60)?.try_into().ok()?) {+ 2 => Some(false),+ 3 => Some(true),+ _ => None,+ };+ Some(Database { bytes, page_size, usable, utf16 })+}++impl Database<'_> {+ /// The table named `name` (any case); none if there is none.+ pub fn table(&self, name: &str) -> Option<Table> {+ let schema = self.rows(1);+ let entry = schema.iter().find(|(_, r)| {+ r.first().and_then(Value::as_text) == Some("table")+ && r.get(1).and_then(Value::as_text).is_some_and(|n| n.eq_ignore_ascii_case(name))+ })?;+ let root = u32::try_from(entry.1.get(3)?.as_int()?).ok()?;+ let sql = entry.1.get(4).and_then(Value::as_text).unwrap_or_default();+ let (columns, rowid) = columns(sql);+ let rows = self+ .rows(root)+ .into_iter()+ .map(|(id, mut row)| {+ // a column that is the row's id is kept as nothing in the record+ if let Some(slot) = rowid.and_then(|at| row.get_mut(at)).filter(|v| **v == Value::Null) {+ *slot = Value::Int(id);+ }+ row.resize(columns.len().max(row.len()), Value::Null);+ row+ })+ .collect();+ Some(Table { columns, rows })+ }++ fn page(&self, n: u32) -> Option<&[u8]> {+ let start = (n as usize).checked_sub(1)?.checked_mul(self.page_size)?;+ self.bytes.get(start..start.checked_add(self.page_size)?)+ }++ /// The rows of the table whose b-tree starts at page `root`: each its id and values.+ fn rows(&self, root: u32) -> Vec<(i64, Vec<Value>)> {+ let mut out = Vec::new();+ let mut seen = vec![false; self.bytes.len() / self.page_size + 1];+ let mut pending = vec![root];+ while let Some(n) = pending.pop() {+ if out.len() >= MOST_ROWS {+ break;+ }+ match seen.get_mut(n as usize) {+ Some(s) if !*s => *s = true,+ _ => continue,+ }+ let Some(page) = self.page(n) else { continue };+ let h = if n == 1 { 100 } else { 0 };+ let (Some(&kind), Some(count)) = (page.get(h), be16(page, h + 3)) else { continue };+ match kind {+ // interior: the children left to right (pushed so the leftmost comes off first)+ 0x05 => {+ let mut children: Vec<u32> = (0..count)+ .filter_map(|i| {+ be16(page, h + 12 + 2 * usize::from(i)).and_then(|at| be32(page, usize::from(at)))+ })+ .collect();+ children.extend(be32(page, h + 8));+ pending.extend(children.into_iter().rev());+ }+ 0x0D => {+ for i in 0..count {+ let Some(at) = be16(page, h + 8 + 2 * usize::from(i)) else { break };+ if let Some(row) = self.cell(page, usize::from(at)) {+ out.push(row);+ }+ }+ }+ _ => {}+ }+ }+ out+ }++ /// A leaf cell: its row id and record, the payload gathered from overflow pages.+ fn cell(&self, page: &[u8], at: usize) -> Option<(i64, Vec<Value>)> {+ let (len, a) = varint(page, at)?;+ let (id, b) = varint(page, at + a)?;+ let len = usize::try_from(len).ok().filter(|l| *l <= MOST_PAYLOAD)?;+ let start = at + a + b;+ // how much of the payload is on this page (the rest on overflow pages)+ let u = self.usable;+ let most_here = u - 35;+ let local = if len <= most_here {+ len+ } else {+ let least = (u - 12) * 32 / 255 - 23;+ let k = least + (len - least) % (u - 4);+ if k <= most_here { k } else { least }+ };+ let mut payload = page.get(start..start + local)?.to_vec();+ let mut next = if local < len { be32(page, start + local)? } else { 0 };+ let mut hops = 0;+ while payload.len() < len && next != 0 && hops < self.bytes.len() / self.page_size {+ let over = self.page(next)?;+ let take = (len - payload.len()).min(u - 4);+ payload.extend_from_slice(over.get(4..4 + take)?);+ next = be32(over, 0)?;+ hops += 1;+ }+ if payload.len() < len {+ return None;+ }+ Some((id, self.record(&payload)?))+ }++ /// A record's values.+ fn record(&self, payload: &[u8]) -> Option<Vec<Value>> {+ let (header, mut at) = varint(payload, 0)?;+ let header = usize::try_from(header).ok()?;+ let mut body = header;+ let mut values = Vec::new();+ while at < header {+ let (kind, n) = varint(payload, at)?;+ at += n;+ let (value, used) = match kind {+ 0 => (Value::Null, 0),+ 1..=6 => {+ let size = match kind {+ 5 => 6,+ 6 => 8,+ k => k as usize,+ };+ let bytes = payload.get(body..body + size)?;+ // big-endian, signed+ let mut v: i64 = if bytes.first().is_some_and(|b| b & 0x80 != 0) { -1 } else { 0 };+ for &b in bytes {+ v = (v << 8) | i64::from(b);+ }+ (Value::Int(v), size)+ }+ 7 => (Value::Real(f64::from_be_bytes(payload.get(body..body + 8)?.try_into().ok()?)), 8),+ 8 => (Value::Int(0), 0),+ 9 => (Value::Int(1), 0),+ k if k >= 12 && k % 2 == 0 => {+ let size = usize::try_from((k - 12) / 2).ok()?;+ (Value::Blob(payload.get(body..body.checked_add(size)?)?.to_vec()), size)+ }+ k if k >= 13 => {+ let size = usize::try_from((k - 13) / 2).ok()?;+ (Value::Text(self.text(payload.get(body..body.checked_add(size)?)?)), size)+ }+ _ => return None,+ };+ body += used;+ values.push(value);+ }+ Some(values)+ }++ fn text(&self, bytes: &[u8]) -> String {+ match self.utf16 {+ None => String::from_utf8_lossy(bytes).into_owned(),+ Some(big) => {+ let units: Vec<u16> = bytes+ .as_chunks::<2>()+ .0+ .iter()+ .map(|c| if big { u16::from_be_bytes(*c) } else { u16::from_le_bytes(*c) })+ .collect();+ String::from_utf16_lossy(&units)+ }+ }+ }+}++/// A table's column names from its `CREATE TABLE` statement, and which one (if any) is the row's+/// id (an `INTEGER PRIMARY KEY`).+fn columns(sql: &str) -> (Vec<String>, Option<usize>) {+ let (Some(open), Some(close)) = (sql.find('('), sql.rfind(')')) else { return (Vec::new(), None) };+ let inside = sql.get(open + 1..close).unwrap_or_default();+ let mut parts = Vec::new();+ let (mut depth, mut from) = (0i32, 0usize);+ for (i, c) in inside.char_indices() {+ match c {+ '(' => depth += 1,+ ')' => depth -= 1,+ ',' if depth == 0 => {+ parts.push(inside.get(from..i).unwrap_or_default());+ from = i + 1;+ }+ _ => {}+ }+ }+ parts.push(inside.get(from..).unwrap_or_default());+ let mut names = Vec::new();+ let mut rowid = None;+ for part in parts {+ let part = part.trim();+ let first = part.split_whitespace().next().unwrap_or_default();+ let upper = part.to_ascii_uppercase();+ // table constraints, not columns (their keyword may run into a parenthesis)+ let word: String = part+ .chars()+ .take_while(|c| c.is_ascii_alphanumeric() || *c == '_')+ .collect::<String>()+ .to_ascii_uppercase();+ if ["PRIMARY", "UNIQUE", "CHECK", "FOREIGN", "CONSTRAINT"].contains(&word.as_str()) {+ continue;+ }+ let name = first.trim_matches(|c| c == '"' || c == '`' || c == '[' || c == ']' || c == '\'');+ if upper.contains("INTEGER") && upper.contains("PRIMARY KEY") {+ rowid = Some(names.len());+ }+ names.push(name.to_owned());+ }+ (names, rowid)+}++fn be16(b: &[u8], at: usize) -> Option<u16> {+ Some(u16::from_be_bytes([*b.get(at)?, *b.get(at + 1)?]))+}++fn be32(b: &[u8], at: usize) -> Option<u32> {+ Some(u32::from_be_bytes(b.get(at..at + 4)?.try_into().ok()?))+}++/// A variable-length number (up to 9 bytes, seven bits each but the ninth's eight) and its length.+fn varint(b: &[u8], at: usize) -> Option<(i64, usize)> {+ let mut v: u64 = 0;+ for i in 0..9 {+ let byte = *b.get(at + i)?;+ if i == 8 {+ return Some((((v << 8) | u64::from(byte)) as i64, 9));+ }+ v = (v << 7) | u64::from(byte & 0x7F);+ if byte & 0x80 == 0 {+ return Some((v as i64, i + 1));+ }+ }+ None+}++#[cfg(test)]+pub(crate) mod tests {+ use super::*;++ /// A one-page-per-table database of 4096-byte pages: `tables` each a name, its `CREATE TABLE`+ /// statement and its rows (values as serial type and bytes), all on one leaf page; a value+ /// too big for its page spills onto overflow pages.+ pub(crate) fn database(tables: &[(&str, &str, Vec<Vec<Value>>)]) -> Vec<u8> {+ const PAGE: usize = 4096;+ fn put_varint(out: &mut Vec<u8>, mut v: u64) {+ let mut bytes = vec![(v & 0x7F) as u8];+ v >>= 7;+ while v > 0 {+ bytes.push((v & 0x7F) as u8 | 0x80);+ v >>= 7;+ }+ out.extend(bytes.iter().rev());+ }+ fn record(values: &[Value]) -> Vec<u8> {+ let (mut types, mut body) = (Vec::new(), Vec::new());+ for v in values {+ match v {+ Value::Null => put_varint(&mut types, 0),+ Value::Int(i) => {+ put_varint(&mut types, 6);+ body.extend_from_slice(&i.to_be_bytes());+ }+ Value::Real(r) => {+ put_varint(&mut types, 7);+ body.extend_from_slice(&r.to_be_bytes());+ }+ Value::Text(t) => {+ put_varint(&mut types, 13 + 2 * t.len() as u64);+ body.extend_from_slice(t.as_bytes());+ }+ Value::Blob(b) => {+ put_varint(&mut types, 12 + 2 * b.len() as u64);+ body.extend_from_slice(b);+ }+ }+ }+ let mut out = Vec::new();+ put_varint(&mut out, types.len() as u64 + 1);+ out.extend(types);+ out.extend(body);+ out+ }+ // page 1 the schema; then each table's leaf; overflow pages after+ let mut pages: Vec<Vec<u8>> = vec![Vec::new(); 1 + tables.len()];+ let leaf = |pages: &mut Vec<Vec<u8>>, n: usize, rows: Vec<Vec<Value>>| {+ let h = if n == 0 { 100 } else { 0 };+ let mut page = vec![0u8; PAGE];+ let mut cells: Vec<Vec<u8>> = Vec::new();+ for (id, row) in rows.iter().enumerate() {+ let payload = record(row);+ let mut cell = Vec::new();+ put_varint(&mut cell, payload.len() as u64);+ put_varint(&mut cell, id as u64 + 1);+ let u = PAGE;+ let (most, least) = (u - 35, (u - 12) * 32 / 255 - 23);+ if payload.len() <= most {+ cell.extend(&payload);+ } else {+ let k = least + (payload.len() - least) % (u - 4);+ let local = if k <= most { k } else { least };+ cell.extend(&payload[..local]);+ let mut rest = &payload[local..];+ let first = pages.len() + 1;+ cell.extend_from_slice(&(first as u32).to_be_bytes());+ while !rest.is_empty() {+ let take = rest.len().min(u - 4);+ let mut over = vec![0u8; PAGE];+ let next = if rest.len() > take { pages.len() + 2 } else { 0 };+ over[..4].copy_from_slice(&(next as u32).to_be_bytes());+ over[4..4 + take].copy_from_slice(&rest[..take]);+ pages.push(over);+ rest = &rest[take..];+ }+ }+ cells.push(cell);+ }+ page[h] = 0x0D;+ page[h + 3..h + 5].copy_from_slice(&(cells.len() as u16).to_be_bytes());+ let mut end = PAGE;+ for (i, cell) in cells.iter().enumerate() {+ end -= cell.len();+ page[end..end + cell.len()].copy_from_slice(cell);+ page[h + 8 + 2 * i..h + 10 + 2 * i].copy_from_slice(&(end as u16).to_be_bytes());+ }+ page[h + 5..h + 7].copy_from_slice(&(end as u16).to_be_bytes());+ pages[n] = page;+ };+ let schema: Vec<Vec<Value>> = tables+ .iter()+ .enumerate()+ .map(|(i, (name, sql, _))| {+ vec![+ Value::Text("table".into()),+ Value::Text((*name).into()),+ Value::Text((*name).into()),+ Value::Int(i as i64 + 2),+ Value::Text((*sql).into()),+ ]+ })+ .collect();+ for (i, (_, _, rows)) in tables.iter().enumerate() {+ leaf(&mut pages, i + 1, rows.clone());+ }+ leaf(&mut pages, 0, schema);+ let mut out: Vec<u8> = pages.concat();+ out[..16].copy_from_slice(b"SQLite format 3\0");+ out[16..18].copy_from_slice(&(PAGE as u16).to_be_bytes());+ out[56..60].copy_from_slice(&1u32.to_be_bytes());+ out+ }++ #[test]+ fn tables_read_with_their_columns_ids_and_long_values() {+ let long: Vec<u8> = (0..20_000).map(|i| (i % 251) as u8).collect();+ let db = database(&[+ (+ "Node",+ "CREATE TABLE Node (_PW_ID INTEGER PRIMARY KEY AUTOINCREMENT, NodeName TEXT, NodeVariantID INTEGER)",+ vec![+ vec![Value::Null, Value::Text("Pencil".into()), Value::Int(3)],+ vec![Value::Null, Value::Text("Ink".into()), Value::Int(4)],+ ],+ ),+ (+ "Material",+ "CREATE TABLE \"Material\" (id INTEGER, \"FileData\" BLOB, Size REAL, UNIQUE(id))",+ vec![vec![Value::Int(1), Value::Blob(long.clone()), Value::Real(2.5)]],+ ),+ ]);+ let d = open(&db).unwrap();+ let node = d.table("node").unwrap();+ assert_eq!(node.columns, ["_PW_ID", "NodeName", "NodeVariantID"]);+ assert_eq!(node.rows.len(), 2);+ assert_eq!(node.get(&node.rows[1], "NodeName").and_then(Value::as_text), Some("Ink"));+ // the id column holds the row's id+ assert_eq!(node.get(&node.rows[1], "_PW_ID").and_then(Value::as_int), Some(2));+ let m = d.table("Material").unwrap();+ assert_eq!(m.columns, ["id", "FileData", "Size"]);+ assert_eq!(m.get(&m.rows[0], "FileData").and_then(Value::as_blob), Some(&long[..]));+ assert_eq!(m.get(&m.rows[0], "Size").and_then(Value::as_real), Some(2.5));+ assert!(d.table("Nothing").is_none());+ }++ #[test]+ fn other_files_and_damaged_ones_are_refused_not_a_crash() {+ assert!(open(b"not a database").is_none());+ let db = database(&[(+ "T",+ "CREATE TABLE T (a TEXT, b BLOB)",+ vec![vec![Value::Text("x".into()), Value::Blob(vec![1; 5000])]],+ )]);+ crate::damaged::survives("SQLite", &db, |b| {+ if let Some(d) = open(b) {+ let _ = d.table("T");+ }+ });+ }+}