web search done right
use nom in zimba parser (#236)
Mikkel Denkerd2fa5f4061bd
diff --git a/Cargo.lock b/Cargo.lockindex add0a98e..89617838 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -6516,6 +6516,7 @@ version = "0.1.0"dependencies = ["lzma","memmap2",+ "nom","thiserror","zstd 0.13.2",]diff --git a/crates/zimba/Cargo.toml b/crates/zimba/Cargo.tomlindex 24856f7d..ccd9f156 100644--- a/crates/zimba/Cargo.toml+++ b/crates/zimba/Cargo.toml@@ -6,7 +6,8 @@ version = "0.1.0"license = "MIT"[dependencies]-lzma = {workspace = true}-memmap2 = {workspace = true}-thiserror = {workspace = true}-zstd = {workspace = true}+lzma = { workspace = true }+memmap2 = { workspace = true }+thiserror = { workspace = true }+zstd = { workspace = true }+nom = { workspace = true }diff --git a/crates/zimba/src/lib.rs b/crates/zimba/src/lib.rsindex 809bf314..a79ce2df 100644--- a/crates/zimba/src/lib.rs+++ b/crates/zimba/src/lib.rs@@ -1,5 +1,5 @@// Stract is an open source web search engine.-// Copyright (C) 2023 Stract ApS+// Copyright (C) 2024 Stract ApS//// This program is free software: you can redistribute it and/or modify// it under the terms of the GNU Affero General Public License as@@ -21,9 +21,15 @@ pub mod wiki;pub use wiki::{Article, ArticleIterator, Image, ImageIterator};+use nom::{+ bytes::complete::{take, take_while},+ combinator::map,+ IResult,+};+use std::{fs::File,- io::{BufReader, Read},+ io::{self, BufReader, Read},path::Path,};@@ -48,16 +54,62 @@ pub enum Error {Lzma(#[from] lzma::Error),}-fn read_zero_terminated(bytes: &[u8]) -> Result<String, Error> {- let mut string = String::new();+fn read_zero_terminated(bytes: &[u8]) -> IResult<&[u8], String> {+ let (remaining, string) = map(take_while(|b| b != 0), |bytes: &[u8]| {+ String::from_utf8_lossy(bytes).into_owned()+ })(bytes)?;++ let (remaining, zero) = take(1usize)(remaining)?;+ if zero != [0] {+ return Err(nom::Err::Error(nom::error::Error::new(+ remaining,+ nom::error::ErrorKind::Tag,+ )));+ }++ Ok((remaining, string))+}++trait NomParseNumber: Sized {+ fn nom_parse_le(bytes: &[u8]) -> IResult<&[u8], Self>;+}++macro_rules! read_u {+ ($type:ty) => {+ impl NomParseNumber for $type {+ fn nom_parse_le(bytes: &[u8]) -> IResult<&[u8], Self> {+ let (remaining, bytes) = take(std::mem::size_of::<$type>())(bytes)?;++ if bytes.len() < std::mem::size_of::<$type>() {+ return Err(nom::Err::Error(nom::error::Error::new(+ remaining,+ nom::error::ErrorKind::Eof,+ )));+ }- let mut i = 0;- while i < bytes.len() && bytes[i] != 0 {- string.push(bytes[i] as char);- i += 1;+ Ok((remaining, Self::from_le_bytes(bytes.try_into().unwrap())))+ }+ }+ };+}++read_u!(u16);+read_u!(u32);+read_u!(u64);+read_u!(u128);++impl NomParseNumber for char {+ fn nom_parse_le(bytes: &[u8]) -> IResult<&[u8], Self> {+ let (remaining, byte) = take(1usize)(bytes)?;+ Ok((remaining, byte[0] as char))}+}- Ok(string)+impl NomParseNumber for u8 {+ fn nom_parse_le(bytes: &[u8]) -> IResult<&[u8], Self> {+ let (remaining, byte) = take(1usize)(bytes)?;+ Ok((remaining, byte[0]))+ }}#[derive(Debug)]@@ -66,7 +118,7 @@ struct Header {magic: u32,major_version: u16,minor_version: u16,- uuid: [u8; 16],+ uuid: u128,entry_count: u32,cluster_count: u32,url_ptr_pos: u64,@@ -84,46 +136,58 @@ impl Header {return Err(Error::UnexpectedEndOfBytes);}- let header = Header {- magic: u32::from_le_bytes([bytes[0], bytes[1], bytes[2], bytes[3]]),- major_version: u16::from_le_bytes([bytes[4], bytes[5]]),- minor_version: u16::from_le_bytes([bytes[6], bytes[7]]),- uuid: [- bytes[8], bytes[9], bytes[10], bytes[11], bytes[12], bytes[13], bytes[14],- bytes[15], bytes[16], bytes[17], bytes[18], bytes[19], bytes[20], bytes[21],- bytes[22], bytes[23],- ],- entry_count: u32::from_le_bytes([bytes[24], bytes[25], bytes[26], bytes[27]]),- cluster_count: u32::from_le_bytes([bytes[28], bytes[29], bytes[30], bytes[31]]),- url_ptr_pos: u64::from_le_bytes([- bytes[32], bytes[33], bytes[34], bytes[35], bytes[36], bytes[37], bytes[38],- bytes[39],- ]),- title_ptr_pos: u64::from_le_bytes([- bytes[40], bytes[41], bytes[42], bytes[43], bytes[44], bytes[45], bytes[46],- bytes[47],- ]),- cluster_ptr_pos: u64::from_le_bytes([- bytes[48], bytes[49], bytes[50], bytes[51], bytes[52], bytes[53], bytes[54],- bytes[55],- ]),- mime_list_pos: u64::from_le_bytes([- bytes[56], bytes[57], bytes[58], bytes[59], bytes[60], bytes[61], bytes[62],- bytes[63],- ]),- main_page: u32::from_le_bytes([bytes[64], bytes[65], bytes[66], bytes[67]]),- layout_page: u32::from_le_bytes([bytes[68], bytes[69], bytes[70], bytes[71]]),- checksum_pos: u64::from_le_bytes([- bytes[72], bytes[73], bytes[74], bytes[75], bytes[76], bytes[77], bytes[78],- bytes[79],- ]),- };+ let (remaining, magic) =+ u32::nom_parse_le(bytes).map_err(|_| Error::UnexpectedEndOfBytes)?;- if header.magic != 72_173_914 {+ if magic != 72_173_914 {return Err(Error::InvalidMagicNumber);}- Ok(header)+ let (remaining, major_version) =+ u16::nom_parse_le(remaining).map_err(|_| Error::UnexpectedEndOfBytes)?;+ let (remaining, minor_version) =+ u16::nom_parse_le(remaining).map_err(|_| Error::UnexpectedEndOfBytes)?;++ let (remaining, uuid) =+ u128::nom_parse_le(remaining).map_err(|_| Error::UnexpectedEndOfBytes)?;++ let (remaining, entry_count) =+ u32::nom_parse_le(remaining).map_err(|_| Error::UnexpectedEndOfBytes)?;+ let (remaining, cluster_count) =+ u32::nom_parse_le(remaining).map_err(|_| Error::UnexpectedEndOfBytes)?;++ let (remaining, url_ptr_pos) =+ u64::nom_parse_le(remaining).map_err(|_| Error::UnexpectedEndOfBytes)?;+ let (remaining, title_ptr_pos) =+ u64::nom_parse_le(remaining).map_err(|_| Error::UnexpectedEndOfBytes)?;+ let (remaining, cluster_ptr_pos) =+ u64::nom_parse_le(remaining).map_err(|_| Error::UnexpectedEndOfBytes)?;+ let (remaining, mime_list_pos) =+ u64::nom_parse_le(remaining).map_err(|_| Error::UnexpectedEndOfBytes)?;++ let (remaining, main_page) =+ u32::nom_parse_le(remaining).map_err(|_| Error::UnexpectedEndOfBytes)?;+ let (remaining, layout_page) =+ u32::nom_parse_le(remaining).map_err(|_| Error::UnexpectedEndOfBytes)?;++ let (_remaining, checksum_pos) =+ u64::nom_parse_le(remaining).map_err(|_| Error::UnexpectedEndOfBytes)?;++ Ok(Header {+ magic,+ major_version,+ minor_version,+ uuid,+ entry_count,+ cluster_count,+ url_ptr_pos,+ title_ptr_pos,+ cluster_ptr_pos,+ mime_list_pos,+ main_page,+ layout_page,+ checksum_pos,+ })}}@@ -133,16 +197,18 @@ impl MimeTypes {fn from_bytes(bytes: &[u8]) -> Result<Self, Error> {let mut mime_types = Vec::new();- let mut i = 0;- while i < bytes.len() {- let mime_type = read_zero_terminated(&bytes[i..])?;+ let mut bytes = bytes;+ while !bytes.is_empty() {+ let (remaining, mime_type) =+ read_zero_terminated(bytes).map_err(|_| Error::UnexpectedEndOfBytes)?;if mime_type.is_empty() {break;}- i += mime_type.len() + 1;mime_types.push(mime_type);++ bytes = remaining;}Ok(Self(mime_types))@@ -175,21 +241,15 @@ impl UrlPointerList {fn from_bytes(bytes: &[u8], num_urls: u32) -> Result<Self, Error> {let mut url_pointers = Vec::new();- let mut i = 0;+ let mut bytes = bytes;for _ in 0..num_urls {- let url_pointer = UrlPointer(u64::from_le_bytes([- bytes[i],- bytes[i + 1],- bytes[i + 2],- bytes[i + 3],- bytes[i + 4],- bytes[i + 5],- bytes[i + 6],- bytes[i + 7],- ]));+ let (remaining, url_pointer) =+ u64::nom_parse_le(bytes).map_err(|_| Error::UnexpectedEndOfBytes)?;+ let url_pointer = UrlPointer(url_pointer);url_pointers.push(url_pointer);- i += 8;++ bytes = remaining;}Ok(Self(url_pointers))@@ -215,17 +275,16 @@ impl TitlePointerList {fn from_bytes(bytes: &[u8], num_titles: u32) -> Result<Self, Error> {let mut title_pointers = Vec::new();- let mut i = 0;+ let mut bytes = bytes;for _ in 0..num_titles {- let title_pointer = TitlePointer(u32::from_le_bytes([- bytes[i],- bytes[i + 1],- bytes[i + 2],- bytes[i + 3],- ]));+ let (remaining, title_pointer) =+ u32::nom_parse_le(bytes).map_err(|_| Error::UnexpectedEndOfBytes)?;++ let title_pointer = TitlePointer(title_pointer);title_pointers.push(title_pointer);- i += 4;++ bytes = remaining;}Ok(Self(title_pointers))@@ -250,22 +309,16 @@ impl ClusterPointerList {fn from_bytes(bytes: &[u8], num_clusters: u32) -> Result<Self, Error> {let mut cluster_pointers = Vec::new();- let mut i = 0;-+ let mut bytes = bytes;for _ in 0..num_clusters {- let cluster_pointer = ClusterPointer(u64::from_le_bytes([- bytes[i],- bytes[i + 1],- bytes[i + 2],- bytes[i + 3],- bytes[i + 4],- bytes[i + 5],- bytes[i + 6],- bytes[i + 7],- ]));+ let (remaining, cluster_pointer) =+ u64::nom_parse_le(bytes).map_err(|_| Error::UnexpectedEndOfBytes)?;++ let cluster_pointer = ClusterPointer(cluster_pointer);cluster_pointers.push(cluster_pointer);- i += 8;++ bytes = remaining;}Ok(Self(cluster_pointers))@@ -295,17 +348,54 @@ pub enum DirEntry {},}+fn u8_reader_parser(bytes: &mut impl Iterator<Item = Result<u8, io::Error>>) -> Result<u8, Error> {+ Ok(bytes.next().ok_or(Error::UnexpectedEndOfBytes)??)+}++fn u32_reader_parser(+ bytes: &mut impl Iterator<Item = Result<u8, io::Error>>,+) -> Result<u32, Error> {+ Ok(u32::from_le_bytes([+ bytes.next().ok_or(Error::UnexpectedEndOfBytes)??,+ bytes.next().ok_or(Error::UnexpectedEndOfBytes)??,+ bytes.next().ok_or(Error::UnexpectedEndOfBytes)??,+ bytes.next().ok_or(Error::UnexpectedEndOfBytes)??,+ ]))+}++fn u64_reader_parser(+ bytes: &mut impl Iterator<Item = Result<u8, io::Error>>,+) -> Result<u64, Error> {+ Ok(u64::from_le_bytes([+ bytes.next().ok_or(Error::UnexpectedEndOfBytes)??,+ bytes.next().ok_or(Error::UnexpectedEndOfBytes)??,+ bytes.next().ok_or(Error::UnexpectedEndOfBytes)??,+ bytes.next().ok_or(Error::UnexpectedEndOfBytes)??,+ bytes.next().ok_or(Error::UnexpectedEndOfBytes)??,+ bytes.next().ok_or(Error::UnexpectedEndOfBytes)??,+ bytes.next().ok_or(Error::UnexpectedEndOfBytes)??,+ bytes.next().ok_or(Error::UnexpectedEndOfBytes)??,+ ]))+}+impl DirEntry {fn from_bytes(bytes: &[u8]) -> Result<Self, Error> {- let mime_type = u16::from_le_bytes([bytes[0], bytes[1]]);- let parameter_len = bytes[2];- let namespace = bytes[3] as char;- let revision = u32::from_le_bytes([bytes[4], bytes[5], bytes[6], bytes[7]]);+ let (remaining, mime_type) =+ u16::nom_parse_le(bytes).map_err(|_| Error::UnexpectedEndOfBytes)?;+ let (remaining, parameter_len) =+ u8::nom_parse_le(remaining).map_err(|_| Error::UnexpectedEndOfBytes)?;+ let (remaining, namespace) =+ char::nom_parse_le(remaining).map_err(|_| Error::UnexpectedEndOfBytes)?;+ let (remaining, revision) =+ u32::nom_parse_le(remaining).map_err(|_| Error::UnexpectedEndOfBytes)?;if mime_type == 0xffff {- let redirect_index = u32::from_le_bytes([bytes[8], bytes[9], bytes[10], bytes[11]]);- let url = read_zero_terminated(&bytes[12..])?;- let title = read_zero_terminated(&bytes[12 + url.len() + 1..])?;+ let (remaining, redirect_index) =+ u32::nom_parse_le(remaining).map_err(|_| Error::UnexpectedEndOfBytes)?;+ let (remaining, url) =+ read_zero_terminated(remaining).map_err(|_| Error::UnexpectedEndOfBytes)?;+ let (_, title) =+ read_zero_terminated(remaining).map_err(|_| Error::UnexpectedEndOfBytes)?;return Ok(Self::Redirect {mime_type,parameter_len,@@ -317,15 +407,23 @@ impl DirEntry {});}- let url = read_zero_terminated(&bytes[16..])?;- let title = read_zero_terminated(&bytes[16 + url.len() + 1..])?;+ let (remaining, cluster_number) =+ u32::nom_parse_le(remaining).map_err(|_| Error::UnexpectedEndOfBytes)?;+ let (remaining, blob_number) =+ u32::nom_parse_le(remaining).map_err(|_| Error::UnexpectedEndOfBytes)?;++ let (remaining, url) =+ read_zero_terminated(remaining).map_err(|_| Error::UnexpectedEndOfBytes)?;+ let (_, title) =+ read_zero_terminated(remaining).map_err(|_| Error::UnexpectedEndOfBytes)?;+Ok(Self::Content {mime_type,parameter_len,namespace,revision,- cluster_number: u32::from_le_bytes([bytes[8], bytes[9], bytes[10], bytes[11]]),- blob_number: u32::from_le_bytes([bytes[12], bytes[13], bytes[14], bytes[15]]),+ cluster_number,+ blob_number,url,title,})@@ -412,26 +510,12 @@ impl Cluster {match size {OffsetSize::U32 => {blob_offsets.push(ClusterOffset {- offset: u64::from(u32::from_le_bytes([- reader.next().ok_or(Error::UnexpectedEndOfBytes)??,- reader.next().ok_or(Error::UnexpectedEndOfBytes)??,- reader.next().ok_or(Error::UnexpectedEndOfBytes)??,- reader.next().ok_or(Error::UnexpectedEndOfBytes)??,- ])),+ offset: u64::from(u32_reader_parser(&mut reader)?),});}OffsetSize::U64 => {blob_offsets.push(ClusterOffset {- offset: u64::from_le_bytes([- reader.next().ok_or(Error::UnexpectedEndOfBytes)??,- reader.next().ok_or(Error::UnexpectedEndOfBytes)??,- reader.next().ok_or(Error::UnexpectedEndOfBytes)??,- reader.next().ok_or(Error::UnexpectedEndOfBytes)??,- reader.next().ok_or(Error::UnexpectedEndOfBytes)??,- reader.next().ok_or(Error::UnexpectedEndOfBytes)??,- reader.next().ok_or(Error::UnexpectedEndOfBytes)??,- reader.next().ok_or(Error::UnexpectedEndOfBytes)??,- ]),+ offset: u64_reader_parser(&mut reader)?,});}}@@ -445,26 +529,12 @@ impl Cluster {match size {OffsetSize::U32 => {blob_offsets.push(ClusterOffset {- offset: u64::from(u32::from_le_bytes([- reader.next().ok_or(Error::UnexpectedEndOfBytes)??,- reader.next().ok_or(Error::UnexpectedEndOfBytes)??,- reader.next().ok_or(Error::UnexpectedEndOfBytes)??,- reader.next().ok_or(Error::UnexpectedEndOfBytes)??,- ])),+ offset: u64::from(u32_reader_parser(&mut reader)?),});}OffsetSize::U64 => {blob_offsets.push(ClusterOffset {- offset: u64::from_le_bytes([- reader.next().ok_or(Error::UnexpectedEndOfBytes)??,- reader.next().ok_or(Error::UnexpectedEndOfBytes)??,- reader.next().ok_or(Error::UnexpectedEndOfBytes)??,- reader.next().ok_or(Error::UnexpectedEndOfBytes)??,- reader.next().ok_or(Error::UnexpectedEndOfBytes)??,- reader.next().ok_or(Error::UnexpectedEndOfBytes)??,- reader.next().ok_or(Error::UnexpectedEndOfBytes)??,- reader.next().ok_or(Error::UnexpectedEndOfBytes)??,- ]),+ offset: u64_reader_parser(&mut reader)?,});}}@@ -481,7 +551,7 @@ impl Cluster {let mut blobs = Vec::new();for _ in 0..missing_bytes {- blobs.push(reader.next().ok_or(Error::UnexpectedEndOfBytes)??);+ blobs.push(u8_reader_parser(&mut reader)?);}Ok(Self {diff --git a/crates/zimba/src/wiki.rs b/crates/zimba/src/wiki.rsindex 54b41433..11e5fe60 100644--- a/crates/zimba/src/wiki.rs+++ b/crates/zimba/src/wiki.rs@@ -1,5 +1,5 @@// Stract is an open source web search engine.-// Copyright (C) 2023 Stract ApS+// Copyright (C) 2024 Stract ApS//// This program is free software: you can redistribute it and/or modify// it under the terms of the GNU Affero General Public License as