Skip to content

web search done right

move tantivy dependencies up to workspace for consistent versioning between crates

Mikkel Denkerf46abd05110b
diff --git a/Cargo.lock b/Cargo.lock
index ecb1b8ca..555408c4 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -3306,7 +3306,7 @@ name = "optics"
version = "0.1.0"
dependencies = [
"bincode",
- "itertools 0.11.0",
+ "itertools 0.13.0",
"lalrpop",
"lalrpop-util",
"logos",
@@ -4274,7 +4274,7 @@ name = "robotstxt"
version = "0.1.0"
dependencies = [
"anyhow",
- "itertools 0.11.0",
+ "itertools 0.13.0",
"nom",
"percent-encoding",
"proptest",
@@ -4913,7 +4913,7 @@ dependencies = [
"async-stream",
"axum",
"axum-macros",
- "base64 0.21.7",
+ "base64 0.22.1",
"bincode",
"bitflags 2.6.0",
"bitvec",
@@ -4945,7 +4945,7 @@ dependencies = [
"image",
"indicatif 0.17.8",
"insta",
- "itertools 0.11.0",
+ "itertools 0.13.0",
"kuchiki",
"log",
"logos",
@@ -5204,7 +5204,6 @@ dependencies = [
"byteorder 1.5.0",
"census",
"crc32fast",
- "criterion",
"crossbeam-channel",
"downcast-rs",
"fail",
@@ -5247,7 +5246,6 @@ dependencies = [
"thiserror",
"time",
"uuid",
- "winapi",
"zipf",
"zstd 0.13.1",
]
diff --git a/Cargo.toml b/Cargo.toml
index 990f0963..e9ae94dc 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -25,83 +25,110 @@ debug = true
[workspace.dependencies]
ahash = "0.8.11"
+aho-corasick = "1.0"
anyhow = {version = "1.0.72", features = ["backtrace"]}
+arc-swap = "1.5.0"
async-channel = "1.8.0"
async-stream = "0.3.3"
+async-trait = "0.1"
axum = "0.7.2"
axum-macros = "0.4.0"
-base64 = "0.21.4"
+base64 = "0.22.0"
bincode = {git = "https://github.com/bincode-org/bincode", rev = "100685b", features = [
"serde",
]}
bindgen = "0.69.2"
+binggan = "0.8.0"
bitflags = "2.5.0"
+bitpacking = {version = "0.9.2", default-features = false, features = [
+ "bitpacker4x",
+ "bitpacker1x",
+]}
bitvec = {version = "1.0.1", features = ["serde"]}
bytecount = "0.6.7"
bytemuck = {version = "1.13.1", features = ["derive"]}
+byteorder = "1.4.3"
candle-core = "0.3.3"
candle-nn = "0.3.3"
candle-transformers = "0.3.3"
cc = {version = "1", features = ["parallel"]}
+census = "0.4.2"
chardetng = "0.1.17"
chitchat = "0.5.0"
chrono = {version = "0.4.23", features = ["serde"]}
clap = {version = "4.4.6", features = ["derive"]}
cmake = "0.1"
+crc32fast = "1.3.2"
criterion = "0.5.1"
crossbeam-channel = "0.5.6"
csv = "1.1.6"
dashmap = {version = "5.4.0", features = ["rayon"]}
deadpool = "0.12.1"
+downcast-rs = "1.2.0"
encoding_rs = "0.8.31"
enum_dispatch = "0.3.12"
eventsource-stream = "0.2.3"
+fail = {version = "0.5.0", features = ["failpoints"]}
+fastdivide = "0.4.0"
fend-core = "1.4.8"
flate2 = "1.0.28"
fnv = "1.0.3"
+fs4 = {version = "0.8.0"}
fst = {version = "0.4.7", features = ["levenshtein"]}
futures = "0.3.21"
+futures-util = {version = "0.3.28"}
half = {version = "2.2.1", features = ["serde"]}
hashbrown = {version = "0.14.0", features = ["serde"]}
+htmlescape = "0.3.1"
http = "1.0.0"
image = "0.25.1"
indicatif = {version = "0.17.7", features = ["rayon"]}
insta = "1.31"
-itertools = "0.11.0"
+itertools = "0.13.0"
js-sys = "0.3.60"
lalrpop = {version = "0.20.0", features = ["lexer"]}
lalrpop-util = {version = "0.20.0", features = ["lexer"]}
+levenshtein_automata = "0.2.1"
libc = "0.2.142"
log = {version = "0.4", features = ["release_max_level_info"]}
logos = "0.13.0"
+lru = "0.12.0"
lz-str = "0.2.1"
lz4_flex = "0.11.1"
lzma = "0.2.2"
maplit = "1.0.2"
+matches = "0.1.9"
md5 = "0.7.0"
+measure_time = "0.8.2"
memmap2 = "0.9.0"
mime = "0.3.17"
min-max-heap = "1.3.0"
+more-asserts = "0.3.1"
+murmurhash32 = "0.3"
nom = "7.1.3"
num-bigint = "0.4.1"
num-rational = "0.4.1"
once_cell = "1.13.1"
+oneshot = "0.1.7"
openraft = {version = "0.9.1", features = [
"storage-v2",
"serde",
"tracing-log",
]}
ownedbytes = {path = "crates/ownedbytes"}
+paste = "1.0.11"
percent-encoding = "2.3.0"
-postcard = {version = "1.0.8", features = ["experimental-derive", "alloc"]}
+postcard = {version = "1.0.8", features = ["experimental-derive", "alloc", "use-std"]}
+pretty_assertions = "1.2.1"
proptest = "1.2.0"
proptest-derive = "0.4.0"
publicsuffix = "2.2.3"
quick-xml = "0.30.0"
rand = "0.8.5"
+rand_distr = "0.4.3"
rayon = "1.5.3"
redb = "2.0.0"
-regex = "1.6.0"
+regex = {version = "1.6.0", features = ["std", "unicode"]}
reqwest = {version = "0.11.16", features = ["blocking", "stream", "json"]}
ring = "0.17.3"
rio_api = "0.8.4"
@@ -121,11 +148,14 @@ serde = {version = "1.0.137", features = ["rc", "derive"]}
serde-wasm-bindgen = "0.6.3"
serde_json = "1.0.81"
serde_urlencoded = "0.7.1"
+smallvec = "1.8.0"
stable_deref_trait = "1.2.0"
strum = {version = "0.26.2", features = ["derive"]}
tantivy = {path = "crates/tantivy"}
+tempfile = {version = "3.3.0"}
thiserror = "1.0.31"
tikv-jemallocator = "0.5"
+time = {version = "0.3.10", features = ["serde-well-known", "macros"]}
tokenizers = "0.13.2"
tokio = {version = "1.23.1", features = ["full"]}
tokio-stream = "0.1.11"
@@ -138,11 +168,12 @@ tracing-test = "0.2.4"
url = {version = "2.4.0", features = ["serde"]}
utoipa = {version = "4.2.3", features = ["axum_extras"]}
utoipa-swagger-ui = {version = "7.0.0", features = ["axum"]}
-uuid = "1.1.2"
+uuid = {version = "1.1.2", features = ["serde", "v4"]}
wasm-bindgen = "0.2.83"
whatlang = {version = "0.16.0", features = ["serde"]}
xxhash-rust = {version = "0.8.10", features = ["xxh3", "const_xxh3"]}
-zstd = "0.13"
+zipf = "7.0.0"
+zstd = {version = "0.13", features = ["experimental"]}
[profile.test.package]
flate2.opt-level = 3
diff --git a/crates/core/src/rake.rs b/crates/core/src/rake.rs
index bd35b18f..05cda90c 100644
--- a/crates/core/src/rake.rs
+++ b/crates/core/src/rake.rs
@@ -48,7 +48,7 @@ fn phrases<'a>(
.0
.split_whitespace()
.map(|word| word.replace(|c: char| matches!(c, ',' | '.'), ""))
- .group_by(|word| !stopwords.contains(word))
+ .chunk_by(|word| !stopwords.contains(word))
.into_iter()
.filter_map(move |(is_keyword, words)| {
if is_keyword {
diff --git a/crates/optics-lsp/Cargo.lock b/crates/optics-lsp/Cargo.lock
index 1fa43c0b..5c978ae5 100644
--- a/crates/optics-lsp/Cargo.lock
+++ b/crates/optics-lsp/Cargo.lock
@@ -266,18 +266,18 @@ dependencies = [
[[package]]
name = "itertools"
-version = "0.11.0"
+version = "0.12.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57"
+checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569"
dependencies = [
"either",
]
[[package]]
name = "itertools"
-version = "0.12.1"
+version = "0.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569"
+checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186"
dependencies = [
"either",
]
@@ -428,7 +428,7 @@ name = "optics"
version = "0.1.0"
dependencies = [
"bincode",
- "itertools 0.11.0",
+ "itertools 0.13.0",
"lalrpop",
"lalrpop-util",
"logos",
diff --git a/crates/tantivy/Cargo.toml b/crates/tantivy/Cargo.toml
index cc9f40cb..86e823ab 100644
--- a/crates/tantivy/Cargo.toml
+++ b/crates/tantivy/Cargo.toml
@@ -6,87 +6,70 @@ documentation = "https://docs.rs/tantivy/"
edition = "2021"
exclude = ["benches/*.json", "benches/*.txt"]
keywords = ["search", "information", "retrieval"]
-license = "MIT"
+license = "AGPL-3.0"
name = "tantivy"
publish = false
readme = "README.md"
version = "0.23.0"
[dependencies]
-aho-corasick = "1.0"
-arc-swap = "1.5.0"
-async-trait = "0.1"
-base64 = "0.22.0"
-bitpacking = {version = "0.9.2", default-features = false, features = [
- "bitpacker4x",
- "bitpacker1x",
-]}
-byteorder = "1.4.3"
-census = "0.4.2"
-crc32fast = "1.3.2"
-crossbeam-channel = "0.5.4"
-downcast-rs = "1.2.0"
-fail = {version = "0.5.0", optional = true}
-fastdivide = "0.4.0"
-fnv = "1.0.7"
-fs4 = {version = "0.8.0", optional = true}
+aho-corasick.workspace = true
+arc-swap.workspace = true
+async-trait.workspace = true
+base64.workspace = true
+bitpacking.workspace = true
+byteorder.workspace = true
+census.workspace = true
+crc32fast.workspace = true
+crossbeam-channel.workspace = true
+downcast-rs.workspace = true
+fail = {workspace = true, optional = true}
+fastdivide.workspace = true
+fnv.workspace = true
+fs4 = {workspace = true, optional = true}
fst.workspace = true
-futures-util = {version = "0.3.28", optional = true}
-htmlescape = "0.3.1"
-itertools = "0.13.0"
-levenshtein_automata = "0.2.1"
-log = "0.4.16"
-lru = "0.12.0"
-lz4_flex = {version = "0.11", default-features = false, optional = true}
-measure_time = "0.8.2"
-memmap2 = {version = "0.9.0", optional = true}
-murmurhash32 = "0.3"
-nom = "7"
-once_cell = "1.10.0"
-oneshot = "0.1.7"
+futures-util = {workspace = true, optional = true}
+htmlescape.workspace = true
+itertools.workspace = true
+levenshtein_automata.workspace = true
+log.workspace = true
+lru.workspace = true
+lz4_flex = {workspace = true, optional = true}
+measure_time.workspace = true
+memmap2 = {workspace = true, optional = true}
+murmurhash32.workspace = true
+nom.workspace = true
+once_cell.workspace = true
+oneshot.workspace = true
ownedbytes.workspace = true
-rayon = "1.5.2"
-regex = {version = "1.5.5", default-features = false, features = [
- "std",
- "unicode",
-]}
-rust-stemmers = "1.2.0"
-rustc-hash = "1.1.0"
-serde = {version = "1.0.136", features = ["derive"]}
-serde_json = "1.0.79"
-smallvec = "1.8.0"
-tempfile = {version = "3.3.0", optional = true}
-thiserror = "1.0.30"
-time = {version = "0.3.10", features = ["serde-well-known"]}
-uuid = {version = "1.0.0", features = ["v4", "serde"]}
-zstd = {version = "0.13", features = ["experimental"]}
-
-[target.'cfg(windows)'.dependencies]
-winapi = "0.3.9"
+rayon.workspace = true
+regex.workspace = true
+rust-stemmers.workspace = true
+rustc-hash.workspace = true
+serde.workspace = true
+serde_json.workspace = true
+smallvec.workspace = true
+tempfile = {workspace = true, optional = true}
+thiserror.workspace = true
+time.workspace = true
+uuid.workspace = true
+zstd.workspace = true
[dev-dependencies]
-binggan = "0.8.0"
-futures = "0.3.21"
-maplit = "1.0.2"
-matches = "0.1.9"
-more-asserts = "0.3.1"
-paste = "1.0.11"
-postcard = {version = "1.0.4", features = [
- "use-std",
-], default-features = false}
-pretty_assertions = "1.2.1"
-proptest = "1.0.0"
-rand = "0.8.5"
-rand_distr = "0.4.3"
-time = {version = "0.3.10", features = ["serde-well-known", "macros"]}
-zipf = "7.0.0"
-
-[target.'cfg(not(windows))'.dev-dependencies]
-criterion = {version = "0.5", default-features = false}
-
-[dev-dependencies.fail]
-features = ["failpoints"]
-version = "0.5.0"
+binggan.workspace = true
+fail.workspace = true
+futures.workspace = true
+maplit.workspace = true
+matches.workspace = true
+more-asserts.workspace = true
+paste.workspace = true
+postcard.workspace = true
+pretty_assertions.workspace = true
+proptest.workspace = true
+rand.workspace = true
+rand_distr.workspace = true
+time.workspace = true
+zipf.workspace = true
[features]
default = ["mmap", "stopwords", "lz4-compression"]