Skip to content

web search done right

add proptests to some potentially problematic offsets into strings

Mikkel Denkeraabd774e3404
diff --git a/crates/core/src/entity_index/entity.rs b/crates/core/src/entity_index/entity.rs
index 5ae39ee1..3a7f6842 100644
--- a/crates/core/src/entity_index/entity.rs
+++ b/crates/core/src/entity_index/entity.rs
@@ -165,10 +165,7 @@ impl EntitySnippetFragment {
impl EntitySnippet {
pub fn from_span(span: &Span, truncate_to: usize) -> Self {
let (s, maybe_ellipsis) = if span.text.len() > truncate_to {
- let mut truncate_to = truncate_to;
- while !span.text.is_char_boundary(truncate_to) {
- truncate_to -= 1;
- }
+ let truncate_to = crate::floor_char_boundary(&span.text, truncate_to);
(&span.text[0..truncate_to], "...")
} else {
(&*span.text, "")
diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rs
index 5d336a4d..0498c2a7 100644
--- a/crates/core/src/lib.rs
+++ b/crates/core/src/lib.rs
@@ -436,3 +436,35 @@ pub fn urlencode(s: &str) -> String {
percent_encoding::utf8_percent_encode(s, FRAGMENT).to_string()
}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use proptest::prelude::*;
+
+ proptest! {
+ #[test]
+ fn prop_ceil_char_boundary(s: String, index: usize) {
+ let index = if s.is_empty() {
+ 0
+ } else {
+ index % s.len()
+ };
+
+ let ceil = ceil_char_boundary(&s, index);
+ prop_assert!(s.is_char_boundary(ceil));
+ }
+
+ #[test]
+ fn prop_floor_char_boundary(s: String, index: usize) {
+ let index = if s.is_empty() {
+ 0
+ } else {
+ index % s.len()
+ };
+
+ let floor = floor_char_boundary(&s, index);
+ prop_assert!(s.is_char_boundary(floor));
+ }
+ }
+}
diff --git a/crates/core/src/web_spell/error_model.rs b/crates/core/src/web_spell/error_model.rs
index 79ae2d69..2b118dca 100644
--- a/crates/core/src/web_spell/error_model.rs
+++ b/crates/core/src/web_spell/error_model.rs
@@ -232,6 +232,7 @@ impl ErrorModel {
#[cfg(test)]
mod tests {
use super::*;
+ use proptest::prelude::*;
#[test]
fn test_possible_errors() {
@@ -260,4 +261,16 @@ mod tests {
Some(ErrorSequence(vec![ErrorType::Substitution('o', 'i')]))
);
}
+
+ proptest! {
+ #[test]
+ fn prop_possible_errors_boundaries(a: String, b: String) {
+ let errors = possible_errors(&a, &b);
+ if a == b {
+ prop_assert_eq!(errors, None);
+ } else {
+ prop_assert!(errors.is_some());
+ }
+ }
+ }
}
diff --git a/crates/core/src/webpage/just_text.rs b/crates/core/src/webpage/just_text.rs
index 6192f936..b5446bf3 100644
--- a/crates/core/src/webpage/just_text.rs
+++ b/crates/core/src/webpage/just_text.rs
@@ -38,26 +38,26 @@ impl<const N: usize> Preprocessor<N> {
NodeEdge::Start(node) => {
if let Some(element) = node.as_element() {
let element_name: &str = &element.name.local;
- if let Some((_, n)) = self
+ if let Some((_, open_tags)) = self
.removed_tags
.iter()
.zip(self.num_open_tags.iter_mut())
.find(|(name, _)| **name == element_name)
{
- *n += 1;
+ *open_tags += 1;
}
}
}
NodeEdge::End(node) => {
if let Some(element) = node.as_element() {
let element_name: &str = &element.name.local;
- if let Some((_, n)) = self
+ if let Some((_, open_tags)) = self
.removed_tags
.iter()
.zip(self.num_open_tags.iter_mut())
.find(|(name, _)| **name == element_name)
{
- *n -= 1;
+ *open_tags -= 1;
}
}
}
diff --git a/crates/optics-lsp/Cargo.lock b/crates/optics-lsp/Cargo.lock
index 5c978ae5..72f30d53 100644
--- a/crates/optics-lsp/Cargo.lock
+++ b/crates/optics-lsp/Cargo.lock
@@ -155,12 +155,12 @@ checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5"
[[package]]
name = "errno"
-version = "0.3.5"
+version = "0.3.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ac3e13f66a2f95e32a39eaa81f6b95d42878ca0e1db0c7543723dfe12557e860"
+checksum = "534c5cf6194dfab3db3242765c03bbe257cf92f22b38f6bc0c58d59108a820ba"
dependencies = [
"libc",
- "windows-sys 0.48.0",
+ "windows-sys 0.52.0",
]
[[package]]
@@ -191,8 +191,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c05aeb6a22b8f62540c194aac980f2115af067bfe15a0734d7277a768d396b31"
dependencies = [
"cfg-if",
+ "js-sys",
"libc",
"wasi",
+ "wasm-bindgen",
]
[[package]]
@@ -329,17 +331,29 @@ dependencies = [
"regex",
]
+[[package]]
+name = "lazy_static"
+version = "1.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"
+
[[package]]
name = "libc"
-version = "0.2.149"
+version = "0.2.155"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a08173bc88b7955d1b3145aa561539096c421ac8debde8cbc3612ec635fee29b"
+checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c"
+
+[[package]]
+name = "libm"
+version = "0.2.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4ec2a862134d2a7d32d7983ddcdd1c4923530833c9f2ea1a44fc5fa473989058"
[[package]]
name = "linux-raw-sys"
-version = "0.4.10"
+version = "0.4.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "da2479e8c062e40bf0066ffa0bc823de0a9368974af99c9f6df941d2c231e03f"
+checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89"
[[package]]
name = "lock_api"
@@ -417,6 +431,16 @@ version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e4a24736216ec316047a1fc4252e27dabb04218aa4a3f37c6e7ddbf1f9782b54"
+[[package]]
+name = "num-traits"
+version = "0.2.19"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841"
+dependencies = [
+ "autocfg",
+ "libm",
+]
+
[[package]]
name = "once_cell"
version = "1.17.1"
@@ -443,10 +467,13 @@ name = "optics-lsp"
version = "0.1.0"
dependencies = [
"console_error_panic_hook",
+ "getrandom",
"itertools 0.12.1",
"js-sys",
"lsp-types",
"optics",
+ "proptest",
+ "proptest-derive",
"serde",
"serde-wasm-bindgen",
"thiserror",
@@ -507,6 +534,12 @@ version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5be167a7af36ee22fe3115051bc51f6e6c7054c9348e28deb4f49bd6f705a315"
+[[package]]
+name = "ppv-lite86"
+version = "0.2.17"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de"
+
[[package]]
name = "precomputed-hash"
version = "0.1.1"
@@ -546,6 +579,35 @@ dependencies = [
"unicode-ident",
]
+[[package]]
+name = "proptest"
+version = "1.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b4c2511913b88df1637da85cc8d96ec8e43a3f8bb8ccb71ee1ac240d6f3df58d"
+dependencies = [
+ "bit-set",
+ "bit-vec",
+ "bitflags 2.4.1",
+ "lazy_static",
+ "num-traits",
+ "rand",
+ "rand_chacha",
+ "rand_xorshift",
+ "regex-syntax 0.8.4",
+ "unarray",
+]
+
+[[package]]
+name = "proptest-derive"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9cf16337405ca084e9c78985114633b6827711d22b9e6ef6c6c0d665eb3f0b6e"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 1.0.109",
+]
+
[[package]]
name = "quote"
version = "1.0.35"
@@ -555,6 +617,45 @@ dependencies = [
"proc-macro2",
]
+[[package]]
+name = "rand"
+version = "0.8.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404"
+dependencies = [
+ "libc",
+ "rand_chacha",
+ "rand_core",
+]
+
+[[package]]
+name = "rand_chacha"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88"
+dependencies = [
+ "ppv-lite86",
+ "rand_core",
+]
+
+[[package]]
+name = "rand_core"
+version = "0.6.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c"
+dependencies = [
+ "getrandom",
+]
+
+[[package]]
+name = "rand_xorshift"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d25bf25ec5ae4a3f1b92f929810509a2f53d7dca2f50b794ff57e3face536c8f"
+dependencies = [
+ "rand_core",
+]
+
[[package]]
name = "redox_syscall"
version = "0.2.16"
@@ -598,17 +699,23 @@ version = "0.7.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da"
+[[package]]
+name = "regex-syntax"
+version = "0.8.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b"
+
[[package]]
name = "rustix"
-version = "0.38.19"
+version = "0.38.34"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "745ecfa778e66b2b63c88a61cb36e0eea109e803b0b86bf9879fbc77c70e86ed"
+checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f"
dependencies = [
"bitflags 2.4.1",
"errno",
"libc",
"linux-raw-sys",
- "windows-sys 0.48.0",
+ "windows-sys 0.52.0",
]
[[package]]
@@ -784,6 +891,12 @@ version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
+[[package]]
+name = "unarray"
+version = "0.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94"
+
[[package]]
name = "unicode-bidi"
version = "0.3.11"
@@ -966,6 +1079,15 @@ dependencies = [
"windows-targets 0.48.5",
]
+[[package]]
+name = "windows-sys"
+version = "0.52.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d"
+dependencies = [
+ "windows-targets 0.52.6",
+]
+
[[package]]
name = "windows-targets"
version = "0.42.1"
@@ -996,6 +1118,22 @@ dependencies = [
"windows_x86_64_msvc 0.48.5",
]
+[[package]]
+name = "windows-targets"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973"
+dependencies = [
+ "windows_aarch64_gnullvm 0.52.6",
+ "windows_aarch64_msvc 0.52.6",
+ "windows_i686_gnu 0.52.6",
+ "windows_i686_gnullvm",
+ "windows_i686_msvc 0.52.6",
+ "windows_x86_64_gnu 0.52.6",
+ "windows_x86_64_gnullvm 0.52.6",
+ "windows_x86_64_msvc 0.52.6",
+]
+
[[package]]
name = "windows_aarch64_gnullvm"
version = "0.42.1"
@@ -1008,6 +1146,12 @@ version = "0.48.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8"
+[[package]]
+name = "windows_aarch64_gnullvm"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3"
+
[[package]]
name = "windows_aarch64_msvc"
version = "0.42.1"
@@ -1020,6 +1164,12 @@ version = "0.48.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc"
+[[package]]
+name = "windows_aarch64_msvc"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469"
+
[[package]]
name = "windows_i686_gnu"
version = "0.42.1"
@@ -1032,6 +1182,18 @@ version = "0.48.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e"
+[[package]]
+name = "windows_i686_gnu"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b"
+
+[[package]]
+name = "windows_i686_gnullvm"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66"
+
[[package]]
name = "windows_i686_msvc"
version = "0.42.1"
@@ -1044,6 +1206,12 @@ version = "0.48.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406"
+[[package]]
+name = "windows_i686_msvc"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66"
+
[[package]]
name = "windows_x86_64_gnu"
version = "0.42.1"
@@ -1056,6 +1224,12 @@ version = "0.48.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e"
+[[package]]
+name = "windows_x86_64_gnu"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78"
+
[[package]]
name = "windows_x86_64_gnullvm"
version = "0.42.1"
@@ -1068,6 +1242,12 @@ version = "0.48.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc"
+[[package]]
+name = "windows_x86_64_gnullvm"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d"
+
[[package]]
name = "windows_x86_64_msvc"
version = "0.42.1"
@@ -1079,3 +1259,9 @@ name = "windows_x86_64_msvc"
version = "0.48.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538"
+
+[[package]]
+name = "windows_x86_64_msvc"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
diff --git a/crates/optics-lsp/Cargo.toml b/crates/optics-lsp/Cargo.toml
index d351ab43..c0eca7c0 100644
--- a/crates/optics-lsp/Cargo.toml
+++ b/crates/optics-lsp/Cargo.toml
@@ -1,21 +1,26 @@
[package]
+edition = "2021"
name = "optics-lsp"
version = "0.1.0"
-edition = "2021"
[lib]
crate-type = ["cdylib"]
[dependencies]
console_error_panic_hook = "0.1.7"
+getrandom = {version = "0.2.8", features = ["js"]}
itertools = "0.12.1"
js-sys = "0.3.60"
lsp-types = "0.95.0"
-serde = { version = "1.0.137", features = ["rc", "derive"] }
+serde = {version = "1.0.137", features = ["rc", "derive"]}
serde-wasm-bindgen = "0.6.3"
thiserror = "1.0.31"
wasm-bindgen = "0.2.83"
-optics = { path = "../optics" }
+optics = {path = "../optics"}
+
+[dev-dependencies]
+proptest = {version = "1.2.0", default-features = false, features = ["std", "bit-set"]}
+proptest-derive = "0.4.0"
[workspace]
diff --git a/crates/optics-lsp/src/lib.rs b/crates/optics-lsp/src/lib.rs
index 705d5b8b..d4cd7aab 100644
--- a/crates/optics-lsp/src/lib.rs
+++ b/crates/optics-lsp/src/lib.rs
@@ -158,8 +158,10 @@ impl OpticsBackend {
MarkedString::String(msg.to_string()),
]),
range: Some(Range {
- start: offset_to_pos(start, &file.source),
- end: offset_to_pos(end, &file.source),
+ start: offset_to_pos(start, &file.source)
+ .expect("start offset should be within bounds of source"),
+ end: offset_to_pos(end, &file.source)
+ .expect("end offset should be within bounds of source"),
}),
})
})
@@ -238,8 +240,10 @@ fn err_to_diagnostic(err: optics::Error, source: &str) -> Diagnostic {
Diagnostic {
range: Range {
- start: offset_to_pos(start, source),
- end: offset_to_pos(end, source),
+ start: offset_to_pos(start, source)
+ .expect("start offset should be within bounds of source"),
+ end: offset_to_pos(end, source)
+ .expect("end offset should be within bounds of source"),
},
severity: Some(DiagnosticSeverity::ERROR),
message,
@@ -252,8 +256,10 @@ fn err_to_diagnostic(err: optics::Error, source: &str) -> Diagnostic {
let message = format!("Unrecognized token \"{tok}\"");
Diagnostic {
range: Range {
- start: offset_to_pos(start, source),
- end: offset_to_pos(end, source),
+ start: offset_to_pos(start, source)
+ .expect("start offset should be within bounds of source"),
+ end: offset_to_pos(end, source)
+ .expect("end offset should be within bounds of source"),
},
severity: Some(DiagnosticSeverity::ERROR),
message,
@@ -266,8 +272,10 @@ fn err_to_diagnostic(err: optics::Error, source: &str) -> Diagnostic {
let message = format!("Failed to parse token \"{tok}\" as a number");
Diagnostic {
range: Range {
- start: offset_to_pos(start, source),
- end: offset_to_pos(end, source),
+ start: offset_to_pos(start, source)
+ .expect("start offset should be within bounds of source"),
+ end: offset_to_pos(end, source)
+ .expect("end offset should be within bounds of source"),
},
severity: Some(DiagnosticSeverity::ERROR),
message,
@@ -278,8 +286,10 @@ fn err_to_diagnostic(err: optics::Error, source: &str) -> Diagnostic {
let message = "We encountered an unknown error".to_string();
Diagnostic {
range: Range {
- start: offset_to_pos(start, source),
- end: offset_to_pos(end, source),
+ start: offset_to_pos(start, source)
+ .expect("start offset should be within bounds of source"),
+ end: offset_to_pos(end, source)
+ .expect("end offset should be within bounds of source"),
},
severity: Some(DiagnosticSeverity::ERROR),
message,
@@ -290,8 +300,10 @@ fn err_to_diagnostic(err: optics::Error, source: &str) -> Diagnostic {
let message = "One of your patterns are unsupported".to_string();
Diagnostic {
range: Range {
- start: offset_to_pos(0, source),
- end: offset_to_pos(source.len(), source),
+ start: offset_to_pos(0, source)
+ .expect("start offset should be within bounds of source"),
+ end: offset_to_pos(source.len(), source)
+ .expect("end offset should be within bounds of source"),
},
severity: Some(DiagnosticSeverity::ERROR),
message,
@@ -304,18 +316,22 @@ fn err_to_diagnostic(err: optics::Error, source: &str) -> Diagnostic {
}
}
-fn offset_to_pos(offset: usize, src: &str) -> Position {
+fn offset_to_pos(offset: usize, src: &str) -> Option<Position> {
+ if offset > src.len() || !src.is_char_boundary(offset) {
+ return None;
+ }
+
if src[..offset].is_empty() {
- return Position::new(0, 0);
+ return Some(Position::new(0, 0));
}
if src[..offset].ends_with('\n') {
let l = src[..offset].lines().count();
- Position::new(l as _, 0)
+ Some(Position::new(l as _, 0))
} else {
let l = src[..offset].lines().count() - 1;
let c = src[..offset].lines().last().unwrap_or_default().len();
- Position::new(l as _, c as _)
+ Some(Position::new(l as _, c as _))
}
}
@@ -328,7 +344,7 @@ fn position_to_byte_offset(pos: &Position, src: &str) -> Option<usize> {
if columns == 0 {
return true;
} else {
- columns -= 1
+ columns -= c.len_utf8() as u32;
}
} else if c == '\n' {
lines -= 1;
@@ -337,3 +353,24 @@ fn position_to_byte_offset(pos: &Position, src: &str) -> Option<usize> {
})
.map(|(idx, _)| idx)
}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use proptest::prelude::*;
+
+ proptest! {
+ #![proptest_config(ProptestConfig::with_cases(4096))]
+ #[test]
+ fn round_trip(src in ".*", byte_offset in 0..10_000usize) {
+
+ if let Some(pos) = offset_to_pos(byte_offset, &src) {
+ if let Some(b) = position_to_byte_offset(&pos, &src) {
+ prop_assert_eq!(byte_offset, b);
+ } else if src.is_char_boundary(byte_offset) {
+ prop_assert!(byte_offset >= src.len());
+ }
+ }
+ }
+ }
+}